diff --git "a/checkpoint-20000/trainer_state.json" "b/checkpoint-20000/trainer_state.json" new file mode 100644--- /dev/null +++ "b/checkpoint-20000/trainer_state.json" @@ -0,0 +1,144432 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.39718790960003175, + "global_step": 20000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0, + "learning_rate": 9.92063492063492e-07, + "loss": 10.5793, + "theoretical_loss": 20.84128112621979, + "tokens_seen": 65536 + }, + { + "epoch": 0.0, + "learning_rate": 1.984126984126984e-06, + "loss": 10.5603, + "theoretical_loss": 17.594664429001284, + "tokens_seen": 131072 + }, + { + "epoch": 0.0, + "learning_rate": 2.9761904761904763e-06, + "loss": 10.5098, + "theoretical_loss": 15.967940417509208, + "tokens_seen": 196608 + }, + { + "epoch": 0.0, + "learning_rate": 3.968253968253968e-06, + "loss": 10.3655, + "theoretical_loss": 14.92078008066121, + "tokens_seen": 262144 + }, + { + "epoch": 0.0, + "learning_rate": 4.96031746031746e-06, + "loss": 10.2066, + "theoretical_loss": 14.164680262435617, + "tokens_seen": 327680 + }, + { + "epoch": 0.0, + "learning_rate": 5.9523809523809525e-06, + "loss": 9.9617, + "theoretical_loss": 13.581024797222863, + "tokens_seen": 393216 + }, + { + "epoch": 0.0, + "learning_rate": 6.944444444444444e-06, + "loss": 9.7808, + "theoretical_loss": 13.11027232607383, + "tokens_seen": 458752 + }, + { + "epoch": 0.0, + "learning_rate": 7.936507936507936e-06, + "loss": 9.5885, + "theoretical_loss": 12.718592950155966, + "tokens_seen": 524288 + }, + { + "epoch": 0.0, + "learning_rate": 8.928571428571428e-06, + "loss": 9.441, + "theoretical_loss": 12.385055788546264, + "tokens_seen": 589824 + }, + { + "epoch": 0.0, + "learning_rate": 9.92063492063492e-06, + "loss": 9.2732, + "theoretical_loss": 12.09587593170772, + "tokens_seen": 655360 + }, + { + "epoch": 0.0, + "learning_rate": 1.0912698412698412e-05, + "loss": 9.1123, + "theoretical_loss": 11.841529555774319, + "tokens_seen": 720896 + }, + { + "epoch": 0.0, + "learning_rate": 1.1904761904761905e-05, + "loss": 9.1967, + "theoretical_loss": 11.615182533379372, + "tokens_seen": 786432 + }, + { + "epoch": 0.0, + "learning_rate": 1.2896825396825396e-05, + "loss": 8.9707, + "theoretical_loss": 11.41177746485943, + "tokens_seen": 851968 + }, + { + "epoch": 0.0, + "learning_rate": 1.3888888888888888e-05, + "loss": 8.8491, + "theoretical_loss": 11.227475026784514, + "tokens_seen": 917504 + }, + { + "epoch": 0.0, + "learning_rate": 1.4880952380952381e-05, + "loss": 8.8375, + "theoretical_loss": 11.059297242650555, + "tokens_seen": 983040 + }, + { + "epoch": 0.0, + "learning_rate": 1.5873015873015872e-05, + "loss": 8.7712, + "theoretical_loss": 10.90489141112959, + "tokens_seen": 1048576 + }, + { + "epoch": 0.0, + "learning_rate": 1.6865079365079364e-05, + "loss": 8.6895, + "theoretical_loss": 10.762369064735367, + "tokens_seen": 1114112 + }, + { + "epoch": 0.0, + "learning_rate": 1.7857142857142855e-05, + "loss": 8.5644, + "theoretical_loss": 10.63019320090292, + "tokens_seen": 1179648 + }, + { + "epoch": 0.0, + "learning_rate": 1.884920634920635e-05, + "loss": 8.6177, + "theoretical_loss": 10.507097497029282, + "tokens_seen": 1245184 + }, + { + "epoch": 0.0, + "learning_rate": 1.984126984126984e-05, + "loss": 8.5199, + "theoretical_loss": 10.392027268435971, + "tokens_seen": 1310720 + }, + { + "epoch": 0.0, + "learning_rate": 2.0833333333333333e-05, + "loss": 8.4647, + "theoretical_loss": 10.284095545569386, + "tokens_seen": 1376256 + }, + { + "epoch": 0.0, + "learning_rate": 2.1825396825396824e-05, + "loss": 8.4134, + "theoretical_loss": 10.182549877942659, + "tokens_seen": 1441792 + }, + { + "epoch": 0.0, + "learning_rate": 2.2817460317460315e-05, + "loss": 8.2444, + "theoretical_loss": 10.08674688640524, + "tokens_seen": 1507328 + }, + { + "epoch": 0.0, + "learning_rate": 2.380952380952381e-05, + "loss": 8.1851, + "theoretical_loss": 9.99613250351292, + "tokens_seen": 1572864 + }, + { + "epoch": 0.0, + "learning_rate": 2.48015873015873e-05, + "loss": 8.1728, + "theoretical_loss": 9.91022645106575, + "tokens_seen": 1638400 + }, + { + "epoch": 0.0, + "learning_rate": 2.5793650793650793e-05, + "loss": 8.1049, + "theoretical_loss": 9.828609916213201, + "tokens_seen": 1703936 + }, + { + "epoch": 0.0, + "learning_rate": 2.6785714285714284e-05, + "loss": 8.1199, + "theoretical_loss": 9.750915671580087, + "tokens_seen": 1769472 + }, + { + "epoch": 0.0, + "learning_rate": 2.7777777777777776e-05, + "loss": 7.967, + "theoretical_loss": 9.676820083754187, + "tokens_seen": 1835008 + }, + { + "epoch": 0.0, + "learning_rate": 2.876984126984127e-05, + "loss": 7.7455, + "theoretical_loss": 9.606036595818281, + "tokens_seen": 1900544 + }, + { + "epoch": 0.0, + "learning_rate": 2.9761904761904762e-05, + "loss": 8.021, + "theoretical_loss": 9.538310371437493, + "tokens_seen": 1966080 + }, + { + "epoch": 0.0, + "learning_rate": 3.075396825396825e-05, + "loss": 7.808, + "theoretical_loss": 9.473413862304575, + "tokens_seen": 2031616 + }, + { + "epoch": 0.0, + "learning_rate": 3.1746031746031745e-05, + "loss": 7.7084, + "theoretical_loss": 9.411143115583098, + "tokens_seen": 2097152 + }, + { + "epoch": 0.0, + "learning_rate": 3.273809523809524e-05, + "loss": 7.764, + "theoretical_loss": 9.351314678906622, + "tokens_seen": 2162688 + }, + { + "epoch": 0.0, + "learning_rate": 3.373015873015873e-05, + "loss": 7.7169, + "theoretical_loss": 9.293762991332915, + "tokens_seen": 2228224 + }, + { + "epoch": 0.0, + "learning_rate": 3.472222222222222e-05, + "loss": 7.6829, + "theoretical_loss": 9.238338172117398, + "tokens_seen": 2293760 + }, + { + "epoch": 0.0, + "learning_rate": 3.571428571428571e-05, + "loss": 7.455, + "theoretical_loss": 9.184904137180933, + "tokens_seen": 2359296 + }, + { + "epoch": 0.0, + "learning_rate": 3.670634920634921e-05, + "loss": 7.5554, + "theoretical_loss": 9.133336987085992, + "tokens_seen": 2424832 + }, + { + "epoch": 0.0, + "learning_rate": 3.76984126984127e-05, + "loss": 7.6023, + "theoretical_loss": 9.083523621206375, + "tokens_seen": 2490368 + }, + { + "epoch": 0.0, + "learning_rate": 3.8690476190476195e-05, + "loss": 7.5086, + "theoretical_loss": 9.03536054131647, + "tokens_seen": 2555904 + }, + { + "epoch": 0.0, + "learning_rate": 3.968253968253968e-05, + "loss": 7.4053, + "theoretical_loss": 8.988752814581996, + "tokens_seen": 2621440 + }, + { + "epoch": 0.0, + "learning_rate": 4.067460317460318e-05, + "loss": 7.4336, + "theoretical_loss": 8.943613171313253, + "tokens_seen": 2686976 + }, + { + "epoch": 0.0, + "learning_rate": 4.1666666666666665e-05, + "loss": 7.2777, + "theoretical_loss": 8.899861217150864, + "tokens_seen": 2752512 + }, + { + "epoch": 0.0, + "learning_rate": 4.265873015873016e-05, + "loss": 7.3678, + "theoretical_loss": 8.85742274282614, + "tokens_seen": 2818048 + }, + { + "epoch": 0.0, + "learning_rate": 4.365079365079365e-05, + "loss": 7.3581, + "theoretical_loss": 8.816229117451055, + "tokens_seen": 2883584 + }, + { + "epoch": 0.0, + "learning_rate": 4.464285714285714e-05, + "loss": 7.2097, + "theoretical_loss": 8.776216753584098, + "tokens_seen": 2949120 + }, + { + "epoch": 0.0, + "learning_rate": 4.563492063492063e-05, + "loss": 7.1038, + "theoretical_loss": 8.737326634193474, + "tokens_seen": 3014656 + }, + { + "epoch": 0.0, + "learning_rate": 4.6626984126984126e-05, + "loss": 7.1872, + "theoretical_loss": 8.699503893181488, + "tokens_seen": 3080192 + }, + { + "epoch": 0.0, + "learning_rate": 4.761904761904762e-05, + "loss": 7.0083, + "theoretical_loss": 8.662697442408115, + "tokens_seen": 3145728 + }, + { + "epoch": 0.0, + "learning_rate": 4.8611111111111115e-05, + "loss": 7.125, + "theoretical_loss": 8.626859639208966, + "tokens_seen": 3211264 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 7634, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 6.988857746124268, + "objective/train/theoretical_loss": 8.591945989283708, + "objective/train/tokens_used": 3276800, + "theoretical_loss": 8.591945989283708, + "tokens_seen": 3276800 + }, + { + "epoch": 0.0, + "learning_rate": 4.96031746031746e-05, + "loss": 7.0107, + "theoretical_loss": 8.591945989283708, + "tokens_seen": 3276800 + }, + { + "epoch": 0.0, + "learning_rate": 5.05952380952381e-05, + "loss": 6.8827, + "theoretical_loss": 8.557914880567758, + "tokens_seen": 3342336 + }, + { + "epoch": 0.0, + "learning_rate": 5.1587301587301586e-05, + "loss": 6.958, + "theoretical_loss": 8.524727344318643, + "tokens_seen": 3407872 + }, + { + "epoch": 0.0, + "learning_rate": 5.257936507936508e-05, + "loss": 6.7792, + "theoretical_loss": 8.492346840169828, + "tokens_seen": 3473408 + }, + { + "epoch": 0.0, + "learning_rate": 5.357142857142857e-05, + "loss": 6.7257, + "theoretical_loss": 8.460739062345422, + "tokens_seen": 3538944 + }, + { + "epoch": 0.0, + "learning_rate": 5.4563492063492063e-05, + "loss": 6.8709, + "theoretical_loss": 8.429871764603359, + "tokens_seen": 3604480 + }, + { + "epoch": 0.0, + "learning_rate": 5.555555555555555e-05, + "loss": 6.7992, + "theoretical_loss": 8.399714601792851, + "tokens_seen": 3670016 + }, + { + "epoch": 0.0, + "learning_rate": 5.6547619047619046e-05, + "loss": 6.7397, + "theoretical_loss": 8.370238986183718, + "tokens_seen": 3735552 + }, + { + "epoch": 0.0, + "learning_rate": 5.753968253968254e-05, + "loss": 6.7248, + "theoretical_loss": 8.34141795695797, + "tokens_seen": 3801088 + }, + { + "epoch": 0.0, + "learning_rate": 5.8531746031746036e-05, + "loss": 6.524, + "theoretical_loss": 8.313226061453822, + "tokens_seen": 3866624 + }, + { + "epoch": 0.0, + "learning_rate": 5.9523809523809524e-05, + "loss": 6.531, + "theoretical_loss": 8.285639246924504, + "tokens_seen": 3932160 + }, + { + "epoch": 0.0, + "learning_rate": 6.051587301587302e-05, + "loss": 6.4757, + "theoretical_loss": 8.258634761722755, + "tokens_seen": 3997696 + }, + { + "epoch": 0.0, + "learning_rate": 6.15079365079365e-05, + "loss": 6.4186, + "theoretical_loss": 8.232191064950612, + "tokens_seen": 4063232 + }, + { + "epoch": 0.0, + "learning_rate": 6.25e-05, + "loss": 6.3928, + "theoretical_loss": 8.206287743725687, + "tokens_seen": 4128768 + }, + { + "epoch": 0.0, + "learning_rate": 6.349206349206349e-05, + "loss": 6.3013, + "theoretical_loss": 8.180905437312258, + "tokens_seen": 4194304 + }, + { + "epoch": 0.0, + "learning_rate": 6.448412698412699e-05, + "loss": 6.3149, + "theoretical_loss": 8.15602576745014, + "tokens_seen": 4259840 + }, + { + "epoch": 0.0, + "learning_rate": 6.547619047619048e-05, + "loss": 6.2491, + "theoretical_loss": 8.13163127428835, + "tokens_seen": 4325376 + }, + { + "epoch": 0.0, + "learning_rate": 6.646825396825397e-05, + "loss": 6.207, + "theoretical_loss": 8.107705357395282, + "tokens_seen": 4390912 + }, + { + "epoch": 0.0, + "learning_rate": 6.746031746031745e-05, + "loss": 6.072, + "theoretical_loss": 8.084232221374057, + "tokens_seen": 4456448 + }, + { + "epoch": 0.0, + "learning_rate": 6.845238095238096e-05, + "loss": 6.215, + "theoretical_loss": 8.06119682566165, + "tokens_seen": 4521984 + }, + { + "epoch": 0.0, + "learning_rate": 6.944444444444444e-05, + "loss": 6.1129, + "theoretical_loss": 8.038584838134476, + "tokens_seen": 4587520 + }, + { + "epoch": 0.0, + "learning_rate": 7.043650793650793e-05, + "loss": 5.9845, + "theoretical_loss": 8.016382592182019, + "tokens_seen": 4653056 + }, + { + "epoch": 0.0, + "learning_rate": 7.142857142857142e-05, + "loss": 6.0914, + "theoretical_loss": 7.994577046944442, + "tokens_seen": 4718592 + }, + { + "epoch": 0.0, + "learning_rate": 7.242063492063492e-05, + "loss": 5.9917, + "theoretical_loss": 7.973155750440629, + "tokens_seen": 4784128 + }, + { + "epoch": 0.0, + "learning_rate": 7.341269841269842e-05, + "loss": 5.9863, + "theoretical_loss": 7.95210680534016, + "tokens_seen": 4849664 + }, + { + "epoch": 0.0, + "learning_rate": 7.440476190476191e-05, + "loss": 5.9589, + "theoretical_loss": 7.931418837156708, + "tokens_seen": 4915200 + }, + { + "epoch": 0.0, + "learning_rate": 7.53968253968254e-05, + "loss": 5.8754, + "theoretical_loss": 7.911080964661844, + "tokens_seen": 4980736 + }, + { + "epoch": 0.0, + "learning_rate": 7.63888888888889e-05, + "loss": 5.9413, + "theoretical_loss": 7.891082772337269, + "tokens_seen": 5046272 + }, + { + "epoch": 0.0, + "learning_rate": 7.738095238095239e-05, + "loss": 5.9595, + "theoretical_loss": 7.871414284700578, + "tokens_seen": 5111808 + }, + { + "epoch": 0.0, + "learning_rate": 7.837301587301588e-05, + "loss": 5.7771, + "theoretical_loss": 7.852065942354954, + "tokens_seen": 5177344 + }, + { + "epoch": 0.0, + "learning_rate": 7.936507936507937e-05, + "loss": 5.8181, + "theoretical_loss": 7.833028579626806, + "tokens_seen": 5242880 + }, + { + "epoch": 0.0, + "learning_rate": 8.035714285714287e-05, + "loss": 5.784, + "theoretical_loss": 7.814293403667714, + "tokens_seen": 5308416 + }, + { + "epoch": 0.0, + "learning_rate": 8.134920634920635e-05, + "loss": 5.8234, + "theoretical_loss": 7.795851974908005, + "tokens_seen": 5373952 + }, + { + "epoch": 0.0, + "learning_rate": 8.234126984126984e-05, + "loss": 5.6448, + "theoretical_loss": 7.77769618875926, + "tokens_seen": 5439488 + }, + { + "epoch": 0.0, + "learning_rate": 8.333333333333333e-05, + "loss": 5.7859, + "theoretical_loss": 7.7598182584719595, + "tokens_seen": 5505024 + }, + { + "epoch": 0.0, + "learning_rate": 8.432539682539683e-05, + "loss": 5.5257, + "theoretical_loss": 7.742210699062561, + "tokens_seen": 5570560 + }, + { + "epoch": 0.0, + "learning_rate": 8.531746031746032e-05, + "loss": 5.4901, + "theoretical_loss": 7.724866312231575, + "tokens_seen": 5636096 + }, + { + "epoch": 0.0, + "learning_rate": 8.630952380952381e-05, + "loss": 5.6892, + "theoretical_loss": 7.707778172200822, + "tokens_seen": 5701632 + }, + { + "epoch": 0.0, + "learning_rate": 8.73015873015873e-05, + "loss": 5.5293, + "theoretical_loss": 7.6909396124039695, + "tokens_seen": 5767168 + }, + { + "epoch": 0.0, + "learning_rate": 8.82936507936508e-05, + "loss": 5.4723, + "theoretical_loss": 7.67434421296992, + "tokens_seen": 5832704 + }, + { + "epoch": 0.0, + "learning_rate": 8.928571428571429e-05, + "loss": 5.4827, + "theoretical_loss": 7.657985788943502, + "tokens_seen": 5898240 + }, + { + "epoch": 0.0, + "learning_rate": 9.027777777777777e-05, + "loss": 5.4353, + "theoretical_loss": 7.641858379192337, + "tokens_seen": 5963776 + }, + { + "epoch": 0.0, + "learning_rate": 9.126984126984126e-05, + "loss": 5.5443, + "theoretical_loss": 7.625956235952921, + "tokens_seen": 6029312 + }, + { + "epoch": 0.0, + "learning_rate": 9.226190476190476e-05, + "loss": 5.3914, + "theoretical_loss": 7.610273814972529, + "tokens_seen": 6094848 + }, + { + "epoch": 0.0, + "learning_rate": 9.325396825396825e-05, + "loss": 5.5417, + "theoretical_loss": 7.594805766207012, + "tokens_seen": 6160384 + }, + { + "epoch": 0.0, + "learning_rate": 9.424603174603175e-05, + "loss": 5.4499, + "theoretical_loss": 7.57954692503759, + "tokens_seen": 6225920 + }, + { + "epoch": 0.0, + "learning_rate": 9.523809523809524e-05, + "loss": 5.4622, + "theoretical_loss": 7.5644923039725604, + "tokens_seen": 6291456 + }, + { + "epoch": 0.0, + "learning_rate": 9.623015873015874e-05, + "loss": 5.4681, + "theoretical_loss": 7.549637084802403, + "tokens_seen": 6356992 + }, + { + "epoch": 0.0, + "learning_rate": 9.722222222222223e-05, + "loss": 5.4838, + "theoretical_loss": 7.534976611179138, + "tokens_seen": 6422528 + }, + { + "epoch": 0.0, + "learning_rate": 9.821428571428572e-05, + "loss": 5.4365, + "theoretical_loss": 7.52050638159289, + "tokens_seen": 6488064 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 8801, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 5.350468635559082, + "objective/train/theoretical_loss": 7.506222042720672, + "objective/train/tokens_used": 6553600, + "theoretical_loss": 7.506222042720672, + "tokens_seen": 6553600 + }, + { + "epoch": 0.0, + "learning_rate": 9.92063492063492e-05, + "loss": 5.3935, + "theoretical_loss": 7.506222042720672, + "tokens_seen": 6553600 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010019841269841271, + "loss": 5.4434, + "theoretical_loss": 7.492119383124103, + "tokens_seen": 6619136 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001011904761904762, + "loss": 5.3941, + "theoretical_loss": 7.478194327274557, + "tokens_seen": 6684672 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010218253968253968, + "loss": 5.2929, + "theoretical_loss": 7.46444292988566, + "tokens_seen": 6750208 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010317460317460317, + "loss": 5.2363, + "theoretical_loss": 7.450861370534546, + "tokens_seen": 6815744 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010416666666666667, + "loss": 5.4293, + "theoretical_loss": 7.437445948554516, + "tokens_seen": 6881280 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010515873015873016, + "loss": 5.2499, + "theoretical_loss": 7.424193078182986, + "tokens_seen": 6946816 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010615079365079365, + "loss": 5.2549, + "theoretical_loss": 7.411099283949674, + "tokens_seen": 7012352 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010714285714285714, + "loss": 5.2183, + "theoretical_loss": 7.3981611962910145, + "tokens_seen": 7077888 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010813492063492064, + "loss": 5.2086, + "theoretical_loss": 7.38537554737773, + "tokens_seen": 7143424 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010912698412698413, + "loss": 5.1708, + "theoretical_loss": 7.372739167143329, + "tokens_seen": 7208960 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011011904761904761, + "loss": 5.166, + "theoretical_loss": 7.36024897950216, + "tokens_seen": 7274496 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001111111111111111, + "loss": 5.203, + "theoretical_loss": 7.347901998746318, + "tokens_seen": 7340032 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001121031746031746, + "loss": 5.1519, + "theoretical_loss": 7.335695326111465, + "tokens_seen": 7405568 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011309523809523809, + "loss": 5.2466, + "theoretical_loss": 7.323626146502205, + "tokens_seen": 7471104 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011408730158730158, + "loss": 5.2486, + "theoretical_loss": 7.311691725368293, + "tokens_seen": 7536640 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011507936507936508, + "loss": 5.1916, + "theoretical_loss": 7.299889405723473, + "tokens_seen": 7602176 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011607142857142858, + "loss": 5.0617, + "theoretical_loss": 7.288216605299264, + "tokens_seen": 7667712 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011706349206349207, + "loss": 5.0773, + "theoretical_loss": 7.276670813826511, + "tokens_seen": 7733248 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011805555555555556, + "loss": 5.0869, + "theoretical_loss": 7.265249590437903, + "tokens_seen": 7798784 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011904761904761905, + "loss": 5.1283, + "theoretical_loss": 7.253950561185138, + "tokens_seen": 7864320 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012003968253968255, + "loss": 5.0027, + "theoretical_loss": 7.242771416664761, + "tokens_seen": 7929856 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012103174603174604, + "loss": 5.1616, + "theoretical_loss": 7.231709909747042, + "tokens_seen": 7995392 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012202380952380953, + "loss": 5.0014, + "theoretical_loss": 7.220763853402656, + "tokens_seen": 8060928 + }, + { + "epoch": 0.0, + "learning_rate": 0.000123015873015873, + "loss": 5.0359, + "theoretical_loss": 7.209931118622157, + "tokens_seen": 8126464 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001240079365079365, + "loss": 5.1071, + "theoretical_loss": 7.199209632423606, + "tokens_seen": 8192000 + }, + { + "epoch": 0.0, + "learning_rate": 0.000125, + "loss": 5.059, + "theoretical_loss": 7.188597375943913, + "tokens_seen": 8257536 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001259920634920635, + "loss": 4.992, + "theoretical_loss": 7.178092382609779, + "tokens_seen": 8323072 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012698412698412698, + "loss": 5.0964, + "theoretical_loss": 7.1676927363842795, + "tokens_seen": 8388608 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012797619047619048, + "loss": 5.1229, + "theoretical_loss": 7.157396570085435, + "tokens_seen": 8454144 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012896825396825398, + "loss": 5.1571, + "theoretical_loss": 7.147202063773257, + "tokens_seen": 8519680 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012996031746031748, + "loss": 5.0557, + "theoretical_loss": 7.137107443201986, + "tokens_seen": 8585216 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013095238095238096, + "loss": 4.8387, + "theoretical_loss": 7.127110978334411, + "tokens_seen": 8650752 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013194444444444446, + "loss": 5.1021, + "theoretical_loss": 7.117210981915334, + "tokens_seen": 8716288 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013293650793650793, + "loss": 4.9019, + "theoretical_loss": 7.10740580810139, + "tokens_seen": 8781824 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013392857142857144, + "loss": 4.9421, + "theoretical_loss": 7.097693851144607, + "tokens_seen": 8847360 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001349206349206349, + "loss": 4.961, + "theoretical_loss": 7.088073544127209, + "tokens_seen": 8912896 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001359126984126984, + "loss": 5.0525, + "theoretical_loss": 7.07854335774531, + "tokens_seen": 8978432 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001369047619047619, + "loss": 4.9376, + "theoretical_loss": 7.069101799139266, + "tokens_seen": 9043968 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013789682539682541, + "loss": 4.8755, + "theoretical_loss": 7.0597474107685745, + "tokens_seen": 9109504 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001388888888888889, + "loss": 4.8863, + "theoretical_loss": 7.050478769329319, + "tokens_seen": 9175040 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001398809523809524, + "loss": 4.8716, + "theoretical_loss": 7.0412944847122505, + "tokens_seen": 9240576 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014087301587301586, + "loss": 4.9693, + "theoretical_loss": 7.032193198999712, + "tokens_seen": 9306112 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014186507936507937, + "loss": 4.9202, + "theoretical_loss": 7.023173585499698, + "tokens_seen": 9371648 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014285714285714284, + "loss": 4.9288, + "theoretical_loss": 7.014234347815421, + "tokens_seen": 9437184 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014384920634920634, + "loss": 4.83, + "theoretical_loss": 7.00537421894883, + "tokens_seen": 9502720 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014484126984126984, + "loss": 4.6482, + "theoretical_loss": 6.996591960436656, + "tokens_seen": 9568256 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014583333333333335, + "loss": 4.7924, + "theoretical_loss": 6.9878863615175355, + "tokens_seen": 9633792 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014682539682539685, + "loss": 4.6763, + "theoretical_loss": 6.97925623832895, + "tokens_seen": 9699328 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014781746031746032, + "loss": 4.7696, + "theoretical_loss": 6.97070043313267, + "tokens_seen": 9764864 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 9872, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 4.768758773803711, + "objective/train/theoretical_loss": 6.962217813567541, + "objective/train/tokens_used": 9830400, + "theoretical_loss": 6.962217813567541, + "tokens_seen": 9830400 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014880952380952382, + "loss": 4.7784, + "theoretical_loss": 6.962217813567541, + "tokens_seen": 9830400 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001498015873015873, + "loss": 4.6501, + "theoretical_loss": 6.953807271928458, + "tokens_seen": 9895936 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001507936507936508, + "loss": 4.7478, + "theoretical_loss": 6.945467724470439, + "tokens_seen": 9961472 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015178571428571427, + "loss": 4.7687, + "theoretical_loss": 6.937198110736788, + "tokens_seen": 10027008 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001527777777777778, + "loss": 4.7686, + "theoretical_loss": 6.928997392910329, + "tokens_seen": 10092544 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015376984126984128, + "loss": 4.6178, + "theoretical_loss": 6.920864555186801, + "tokens_seen": 10158080 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015476190476190478, + "loss": 4.7489, + "theoretical_loss": 6.912798603169512, + "tokens_seen": 10223616 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015575396825396825, + "loss": 4.5276, + "theoretical_loss": 6.904798563284391, + "tokens_seen": 10289152 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015674603174603175, + "loss": 4.7731, + "theoretical_loss": 6.896863482214638, + "tokens_seen": 10354688 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015773809523809523, + "loss": 4.8336, + "theoretical_loss": 6.888992426354192, + "tokens_seen": 10420224 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015873015873015873, + "loss": 4.9735, + "theoretical_loss": 6.881184481279271, + "tokens_seen": 10485760 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001597222222222222, + "loss": 4.8357, + "theoretical_loss": 6.873438751237283, + "tokens_seen": 10551296 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016071428571428573, + "loss": 4.7233, + "theoretical_loss": 6.865754358652436, + "tokens_seen": 10616832 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001617063492063492, + "loss": 4.6979, + "theoretical_loss": 6.85813044364739, + "tokens_seen": 10682368 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001626984126984127, + "loss": 4.7943, + "theoretical_loss": 6.85056616358036, + "tokens_seen": 10747904 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016369047619047618, + "loss": 4.7761, + "theoretical_loss": 6.843060692597036, + "tokens_seen": 10813440 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016468253968253969, + "loss": 4.7188, + "theoretical_loss": 6.835613221196822, + "tokens_seen": 10878976 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016567460317460316, + "loss": 4.7359, + "theoretical_loss": 6.8282229558127865, + "tokens_seen": 10944512 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016666666666666666, + "loss": 4.8324, + "theoretical_loss": 6.820889118404866, + "tokens_seen": 11010048 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016765873015873016, + "loss": 4.8003, + "theoretical_loss": 6.813610946065789, + "tokens_seen": 11075584 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016865079365079366, + "loss": 4.6263, + "theoretical_loss": 6.806387690639279, + "tokens_seen": 11141120 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016964285714285717, + "loss": 4.7358, + "theoretical_loss": 6.799218618350064, + "tokens_seen": 11206656 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017063492063492064, + "loss": 4.5761, + "theoretical_loss": 6.792103009445266, + "tokens_seen": 11272192 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017162698412698414, + "loss": 4.8088, + "theoretical_loss": 6.7850401578467645, + "tokens_seen": 11337728 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017261904761904762, + "loss": 4.6377, + "theoretical_loss": 6.778029370814127, + "tokens_seen": 11403264 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017361111111111112, + "loss": 4.6211, + "theoretical_loss": 6.771069968617732, + "tokens_seen": 11468800 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001746031746031746, + "loss": 4.735, + "theoretical_loss": 6.764161284221716, + "tokens_seen": 11534336 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001755952380952381, + "loss": 4.6208, + "theoretical_loss": 6.757302662976402, + "tokens_seen": 11599872 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001765873015873016, + "loss": 4.6961, + "theoretical_loss": 6.750493462319867, + "tokens_seen": 11665408 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001775793650793651, + "loss": 4.738, + "theoretical_loss": 6.74373305148833, + "tokens_seen": 11730944 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017857142857142857, + "loss": 4.6999, + "theoretical_loss": 6.737020811235054, + "tokens_seen": 11796480 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017956349206349207, + "loss": 4.6253, + "theoretical_loss": 6.730356133557465, + "tokens_seen": 11862016 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018055555555555555, + "loss": 4.4414, + "theoretical_loss": 6.7237384214322, + "tokens_seen": 11927552 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018154761904761905, + "loss": 4.7582, + "theoretical_loss": 6.717167088557821, + "tokens_seen": 11993088 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018253968253968252, + "loss": 4.5558, + "theoretical_loss": 6.710641559104915, + "tokens_seen": 12058624 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018353174603174602, + "loss": 4.6598, + "theoretical_loss": 6.704161267473356, + "tokens_seen": 12124160 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018452380952380953, + "loss": 4.6167, + "theoretical_loss": 6.697725658056456, + "tokens_seen": 12189696 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018551587301587303, + "loss": 4.6307, + "theoretical_loss": 6.691334185011799, + "tokens_seen": 12255232 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001865079365079365, + "loss": 4.7105, + "theoretical_loss": 6.68498631203852, + "tokens_seen": 12320768 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001875, + "loss": 4.6805, + "theoretical_loss": 6.678681512160812, + "tokens_seen": 12386304 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001884920634920635, + "loss": 4.7547, + "theoretical_loss": 6.672419267517472, + "tokens_seen": 12451840 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018948412698412698, + "loss": 4.6918, + "theoretical_loss": 6.666199069157267, + "tokens_seen": 12517376 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019047619047619048, + "loss": 4.5493, + "theoretical_loss": 6.660020416839945, + "tokens_seen": 12582912 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019146825396825398, + "loss": 4.6568, + "theoretical_loss": 6.65388281884269, + "tokens_seen": 12648448 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019246031746031748, + "loss": 4.5495, + "theoretical_loss": 6.647785791771864, + "tokens_seen": 12713984 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019345238095238096, + "loss": 4.5651, + "theoretical_loss": 6.641728860379852, + "tokens_seen": 12779520 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019444444444444446, + "loss": 4.7186, + "theoretical_loss": 6.635711557386854, + "tokens_seen": 12845056 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019543650793650793, + "loss": 4.6509, + "theoretical_loss": 6.629733423307451, + "tokens_seen": 12910592 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019642857142857144, + "loss": 4.6096, + "theoretical_loss": 6.6237940062818215, + "tokens_seen": 12976128 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001974206349206349, + "loss": 4.5959, + "theoretical_loss": 6.617892861911426, + "tokens_seen": 13041664 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 11219, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 4.550532341003418, + "objective/train/theoretical_loss": 6.612029553099055, + "objective/train/tokens_used": 13107200, + "theoretical_loss": 6.612029553099055, + "tokens_seen": 13107200 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001984126984126984, + "loss": 4.55, + "theoretical_loss": 6.612029553099055, + "tokens_seen": 13107200 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019940476190476191, + "loss": 4.6515, + "theoretical_loss": 6.606203649893072, + "tokens_seen": 13172736 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020039682539682542, + "loss": 4.5975, + "theoretical_loss": 6.600414729335739, + "tokens_seen": 13238272 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002013888888888889, + "loss": 4.4879, + "theoretical_loss": 6.5946623753155045, + "tokens_seen": 13303808 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002023809523809524, + "loss": 4.5965, + "theoretical_loss": 6.588946178423104, + "tokens_seen": 13369344 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020337301587301587, + "loss": 4.5372, + "theoretical_loss": 6.583265735811386, + "tokens_seen": 13434880 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020436507936507937, + "loss": 4.4473, + "theoretical_loss": 6.577620651058732, + "tokens_seen": 13500416 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020535714285714284, + "loss": 4.6504, + "theoretical_loss": 6.572010534035971, + "tokens_seen": 13565952 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020634920634920634, + "loss": 4.619, + "theoretical_loss": 6.566435000776667, + "tokens_seen": 13631488 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020734126984126985, + "loss": 4.5437, + "theoretical_loss": 6.5608936733507095, + "tokens_seen": 13697024 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020833333333333335, + "loss": 4.6223, + "theoretical_loss": 6.555386179741061, + "tokens_seen": 13762560 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020932539682539685, + "loss": 4.4505, + "theoretical_loss": 6.549912153723619, + "tokens_seen": 13828096 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021031746031746032, + "loss": 4.5368, + "theoretical_loss": 6.544471234750057, + "tokens_seen": 13893632 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021130952380952382, + "loss": 4.5878, + "theoretical_loss": 6.539063067833591, + "tokens_seen": 13959168 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002123015873015873, + "loss": 4.4868, + "theoretical_loss": 6.533687303437551, + "tokens_seen": 14024704 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002132936507936508, + "loss": 4.5297, + "theoretical_loss": 6.528343597366709, + "tokens_seen": 14090240 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021428571428571427, + "loss": 4.6548, + "theoretical_loss": 6.523031610661259, + "tokens_seen": 14155776 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002152777777777778, + "loss": 4.5422, + "theoretical_loss": 6.517751009493386, + "tokens_seen": 14221312 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021626984126984128, + "loss": 4.3978, + "theoretical_loss": 6.51250146506634, + "tokens_seen": 14286848 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021726190476190478, + "loss": 4.558, + "theoretical_loss": 6.507282653515952, + "tokens_seen": 14352384 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021825396825396825, + "loss": 4.4198, + "theoretical_loss": 6.5020942558145105, + "tokens_seen": 14417920 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021924603174603176, + "loss": 4.4131, + "theoretical_loss": 6.496935957676949, + "tokens_seen": 14483456 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022023809523809523, + "loss": 4.539, + "theoretical_loss": 6.491807449469257, + "tokens_seen": 14548992 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022123015873015873, + "loss": 4.4655, + "theoretical_loss": 6.486708426119071, + "tokens_seen": 14614528 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002222222222222222, + "loss": 4.4907, + "theoretical_loss": 6.481638587028371, + "tokens_seen": 14680064 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022321428571428573, + "loss": 4.3812, + "theoretical_loss": 6.4765976359882345, + "tokens_seen": 14745600 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002242063492063492, + "loss": 4.3331, + "theoretical_loss": 6.47158528109558, + "tokens_seen": 14811136 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002251984126984127, + "loss": 4.6371, + "theoretical_loss": 6.466601234671855, + "tokens_seen": 14876672 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022619047619047618, + "loss": 4.6048, + "theoretical_loss": 6.461645213183605, + "tokens_seen": 14942208 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022718253968253969, + "loss": 4.4, + "theoretical_loss": 6.45671693716488, + "tokens_seen": 15007744 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022817460317460316, + "loss": 4.442, + "theoretical_loss": 6.451816131141427, + "tokens_seen": 15073280 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022916666666666666, + "loss": 4.5292, + "theoretical_loss": 6.4469425235566105, + "tokens_seen": 15138816 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023015873015873016, + "loss": 4.5233, + "theoretical_loss": 6.442095846699032, + "tokens_seen": 15204352 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023115079365079367, + "loss": 4.5149, + "theoretical_loss": 6.437275836631781, + "tokens_seen": 15269888 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023214285714285717, + "loss": 4.5043, + "theoretical_loss": 6.432482233123297, + "tokens_seen": 15335424 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023313492063492064, + "loss": 4.4439, + "theoretical_loss": 6.427714779579773, + "tokens_seen": 15400960 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023412698412698414, + "loss": 4.2917, + "theoretical_loss": 6.422973222979093, + "tokens_seen": 15466496 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023511904761904762, + "loss": 4.341, + "theoretical_loss": 6.418257313806215, + "tokens_seen": 15532032 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023611111111111112, + "loss": 4.4741, + "theoretical_loss": 6.413566805990028, + "tokens_seen": 15597568 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002371031746031746, + "loss": 4.3301, + "theoretical_loss": 6.408901456841564, + "tokens_seen": 15663104 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002380952380952381, + "loss": 4.5135, + "theoretical_loss": 6.404261026993607, + "tokens_seen": 15728640 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002390873015873016, + "loss": 4.5372, + "theoretical_loss": 6.399645280341607, + "tokens_seen": 15794176 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002400793650793651, + "loss": 4.3196, + "theoretical_loss": 6.395053983985889, + "tokens_seen": 15859712 + }, + { + "epoch": 0.0, + "learning_rate": 0.00024107142857142857, + "loss": 4.3191, + "theoretical_loss": 6.390486908175121, + "tokens_seen": 15925248 + }, + { + "epoch": 0.0, + "learning_rate": 0.00024206349206349207, + "loss": 4.4536, + "theoretical_loss": 6.385943826251012, + "tokens_seen": 15990784 + }, + { + "epoch": 0.0, + "learning_rate": 0.00024305555555555555, + "loss": 4.5204, + "theoretical_loss": 6.381424514594194, + "tokens_seen": 16056320 + }, + { + "epoch": 0.0, + "learning_rate": 0.00024404761904761905, + "loss": 4.2461, + "theoretical_loss": 6.376928752571271, + "tokens_seen": 16121856 + }, + { + "epoch": 0.0, + "learning_rate": 0.00024503968253968255, + "loss": 4.441, + "theoretical_loss": 6.372456322483007, + "tokens_seen": 16187392 + }, + { + "epoch": 0.0, + "learning_rate": 0.000246031746031746, + "loss": 4.4616, + "theoretical_loss": 6.368007009513611, + "tokens_seen": 16252928 + }, + { + "epoch": 0.0, + "learning_rate": 0.00024702380952380955, + "loss": 4.4605, + "theoretical_loss": 6.363580601681101, + "tokens_seen": 16318464 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 12340, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 4.5626678466796875, + "objective/train/theoretical_loss": 6.359176889788724, + "objective/train/tokens_used": 16384000, + "theoretical_loss": 6.359176889788724, + "tokens_seen": 16384000 + }, + { + "epoch": 0.0, + "learning_rate": 0.000248015873015873, + "loss": 4.4998, + "theoretical_loss": 6.359176889788724, + "tokens_seen": 16384000 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002490079365079365, + "loss": 4.3389, + "theoretical_loss": 6.354795667377392, + "tokens_seen": 16449536 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025, + "loss": 4.3868, + "theoretical_loss": 6.350436730679111, + "tokens_seen": 16515072 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002509920634920635, + "loss": 4.3452, + "theoretical_loss": 6.346099878571405, + "tokens_seen": 16580608 + }, + { + "epoch": 0.01, + "learning_rate": 0.000251984126984127, + "loss": 4.3472, + "theoretical_loss": 6.341784912532662, + "tokens_seen": 16646144 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025297619047619046, + "loss": 4.3821, + "theoretical_loss": 6.337491636598426, + "tokens_seen": 16711680 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025396825396825396, + "loss": 4.2802, + "theoretical_loss": 6.33321985731858, + "tokens_seen": 16777216 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025496031746031746, + "loss": 4.3582, + "theoretical_loss": 6.328969383715416, + "tokens_seen": 16842752 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025595238095238096, + "loss": 4.4243, + "theoretical_loss": 6.324740027242559, + "tokens_seen": 16908288 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002569444444444444, + "loss": 4.3511, + "theoretical_loss": 6.32053160174474, + "tokens_seen": 16973824 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025793650793650796, + "loss": 4.5508, + "theoretical_loss": 6.316343923418371, + "tokens_seen": 17039360 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025892857142857146, + "loss": 4.3529, + "theoretical_loss": 6.312176810772938, + "tokens_seen": 17104896 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025992063492063497, + "loss": 4.1879, + "theoretical_loss": 6.308030084593148, + "tokens_seen": 17170432 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002609126984126984, + "loss": 4.3327, + "theoretical_loss": 6.30390356790187, + "tokens_seen": 17235968 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002619047619047619, + "loss": 4.299, + "theoretical_loss": 6.2997970859237835, + "tokens_seen": 17301504 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002628968253968254, + "loss": 4.5473, + "theoretical_loss": 6.295710466049778, + "tokens_seen": 17367040 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002638888888888889, + "loss": 4.4112, + "theoretical_loss": 6.29164353780205, + "tokens_seen": 17432576 + }, + { + "epoch": 0.01, + "learning_rate": 0.00026488095238095237, + "loss": 4.1185, + "theoretical_loss": 6.287596132799887, + "tokens_seen": 17498112 + }, + { + "epoch": 0.01, + "learning_rate": 0.00026587301587301587, + "loss": 4.2535, + "theoretical_loss": 6.283568084726138, + "tokens_seen": 17563648 + }, + { + "epoch": 0.01, + "learning_rate": 0.00026686507936507937, + "loss": 4.4254, + "theoretical_loss": 6.27955922929433, + "tokens_seen": 17629184 + }, + { + "epoch": 0.01, + "learning_rate": 0.00026785714285714287, + "loss": 4.307, + "theoretical_loss": 6.2755694042164425, + "tokens_seen": 17694720 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002688492063492063, + "loss": 4.3317, + "theoretical_loss": 6.271598449171293, + "tokens_seen": 17760256 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002698412698412698, + "loss": 4.2982, + "theoretical_loss": 6.267646205773557, + "tokens_seen": 17825792 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002708333333333333, + "loss": 4.2089, + "theoretical_loss": 6.263712517543373, + "tokens_seen": 17891328 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002718253968253968, + "loss": 4.3226, + "theoretical_loss": 6.25979722987654, + "tokens_seen": 17956864 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002728174603174603, + "loss": 4.3918, + "theoretical_loss": 6.255900190015284, + "tokens_seen": 18022400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002738095238095238, + "loss": 4.3674, + "theoretical_loss": 6.252021247019596, + "tokens_seen": 18087936 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002748015873015873, + "loss": 4.0737, + "theoretical_loss": 6.2481602517390975, + "tokens_seen": 18153472 + }, + { + "epoch": 0.01, + "learning_rate": 0.00027579365079365083, + "loss": 4.3916, + "theoretical_loss": 6.244317056785462, + "tokens_seen": 18219008 + }, + { + "epoch": 0.01, + "learning_rate": 0.00027678571428571433, + "loss": 4.217, + "theoretical_loss": 6.240491516505343, + "tokens_seen": 18284544 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002777777777777778, + "loss": 4.2583, + "theoretical_loss": 6.236683486953819, + "tokens_seen": 18350080 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002787698412698413, + "loss": 4.1121, + "theoretical_loss": 6.232892825868342, + "tokens_seen": 18415616 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002797619047619048, + "loss": 4.2927, + "theoretical_loss": 6.229119392643171, + "tokens_seen": 18481152 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002807539682539683, + "loss": 4.1183, + "theoretical_loss": 6.225363048304281, + "tokens_seen": 18546688 + }, + { + "epoch": 0.01, + "learning_rate": 0.00028174603174603173, + "loss": 4.1073, + "theoretical_loss": 6.221623655484739, + "tokens_seen": 18612224 + }, + { + "epoch": 0.01, + "learning_rate": 0.00028273809523809523, + "loss": 4.1997, + "theoretical_loss": 6.217901078400542, + "tokens_seen": 18677760 + }, + { + "epoch": 0.01, + "learning_rate": 0.00028373015873015873, + "loss": 4.3263, + "theoretical_loss": 6.214195182826902, + "tokens_seen": 18743296 + }, + { + "epoch": 0.01, + "learning_rate": 0.00028472222222222223, + "loss": 4.13, + "theoretical_loss": 6.2105058360749545, + "tokens_seen": 18808832 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002857142857142857, + "loss": 4.1547, + "theoretical_loss": 6.206832906968911, + "tokens_seen": 18874368 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002867063492063492, + "loss": 4.1076, + "theoretical_loss": 6.203176265823624, + "tokens_seen": 18939904 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002876984126984127, + "loss": 4.2702, + "theoretical_loss": 6.1995357844225545, + "tokens_seen": 19005440 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002886904761904762, + "loss": 4.2407, + "theoretical_loss": 6.195911335996151, + "tokens_seen": 19070976 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002896825396825397, + "loss": 4.2923, + "theoretical_loss": 6.192302795200606, + "tokens_seen": 19136512 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002906746031746032, + "loss": 4.2202, + "theoretical_loss": 6.188710038097013, + "tokens_seen": 19202048 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002916666666666667, + "loss": 4.1672, + "theoretical_loss": 6.185132942130884, + "tokens_seen": 19267584 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002926587301587302, + "loss": 4.2244, + "theoretical_loss": 6.181571386112038, + "tokens_seen": 19333120 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002936507936507937, + "loss": 4.2665, + "theoretical_loss": 6.178025250194852, + "tokens_seen": 19398656 + }, + { + "epoch": 0.01, + "learning_rate": 0.00029464285714285714, + "loss": 4.2907, + "theoretical_loss": 6.174494415858863, + "tokens_seen": 19464192 + }, + { + "epoch": 0.01, + "learning_rate": 0.00029563492063492064, + "loss": 4.0705, + "theoretical_loss": 6.170978765889705, + "tokens_seen": 19529728 + }, + { + "epoch": 0.01, + "learning_rate": 0.00029662698412698414, + "loss": 4.2082, + "theoretical_loss": 6.167478184360402, + "tokens_seen": 19595264 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 13257, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 4.230472564697266, + "objective/train/theoretical_loss": 6.163992556612967, + "objective/train/tokens_used": 19660800, + "theoretical_loss": 6.163992556612967, + "tokens_seen": 19660800 + }, + { + "epoch": 0.01, + "learning_rate": 0.00029761904761904765, + "loss": 4.2174, + "theoretical_loss": 6.163992556612967, + "tokens_seen": 19660800 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002986111111111111, + "loss": 3.9924, + "theoretical_loss": 6.160521769240346, + "tokens_seen": 19726336 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002996031746031746, + "loss": 4.2239, + "theoretical_loss": 6.157065710068663, + "tokens_seen": 19791872 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003005952380952381, + "loss": 4.3329, + "theoretical_loss": 6.1536242681397795, + "tokens_seen": 19857408 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003015873015873016, + "loss": 4.0925, + "theoretical_loss": 6.150197333694162, + "tokens_seen": 19922944 + }, + { + "epoch": 0.01, + "learning_rate": 0.00030257936507936505, + "loss": 4.2802, + "theoretical_loss": 6.146784798154039, + "tokens_seen": 19988480 + }, + { + "epoch": 0.01, + "learning_rate": 0.00030357142857142855, + "loss": 4.0422, + "theoretical_loss": 6.143386554106854, + "tokens_seen": 20054016 + }, + { + "epoch": 0.01, + "learning_rate": 0.00030456349206349205, + "loss": 4.0467, + "theoretical_loss": 6.140002495289, + "tokens_seen": 20119552 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003055555555555556, + "loss": 4.1696, + "theoretical_loss": 6.136632516569836, + "tokens_seen": 20185088 + }, + { + "epoch": 0.01, + "learning_rate": 0.00030654761904761905, + "loss": 4.0606, + "theoretical_loss": 6.133276513935977, + "tokens_seen": 20250624 + }, + { + "epoch": 0.01, + "learning_rate": 0.00030753968253968255, + "loss": 4.1634, + "theoretical_loss": 6.129934384475845, + "tokens_seen": 20316160 + }, + { + "epoch": 0.01, + "learning_rate": 0.00030853174603174605, + "loss": 4.0397, + "theoretical_loss": 6.126606026364497, + "tokens_seen": 20381696 + }, + { + "epoch": 0.01, + "learning_rate": 0.00030952380952380956, + "loss": 4.2085, + "theoretical_loss": 6.123291338848694, + "tokens_seen": 20447232 + }, + { + "epoch": 0.01, + "learning_rate": 0.000310515873015873, + "loss": 4.1928, + "theoretical_loss": 6.119990222232234, + "tokens_seen": 20512768 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003115079365079365, + "loss": 4.1389, + "theoretical_loss": 6.116702577861526, + "tokens_seen": 20578304 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003125, + "loss": 4.3023, + "theoretical_loss": 6.113428308111404, + "tokens_seen": 20643840 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003134920634920635, + "loss": 4.0229, + "theoretical_loss": 6.110167316371188, + "tokens_seen": 20709376 + }, + { + "epoch": 0.01, + "learning_rate": 0.000314484126984127, + "loss": 4.0715, + "theoretical_loss": 6.106919507030966, + "tokens_seen": 20774912 + }, + { + "epoch": 0.01, + "learning_rate": 0.00031547619047619046, + "loss": 4.0989, + "theoretical_loss": 6.103684785468113, + "tokens_seen": 20840448 + }, + { + "epoch": 0.01, + "learning_rate": 0.00031646825396825396, + "loss": 4.1795, + "theoretical_loss": 6.100463058034027, + "tokens_seen": 20905984 + }, + { + "epoch": 0.01, + "learning_rate": 0.00031746031746031746, + "loss": 4.2181, + "theoretical_loss": 6.097254232041089, + "tokens_seen": 20971520 + }, + { + "epoch": 0.01, + "learning_rate": 0.00031845238095238096, + "loss": 4.0827, + "theoretical_loss": 6.094058215749833, + "tokens_seen": 21037056 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003194444444444444, + "loss": 4.1648, + "theoretical_loss": 6.090874918356328, + "tokens_seen": 21102592 + }, + { + "epoch": 0.01, + "learning_rate": 0.00032043650793650796, + "loss": 4.1457, + "theoretical_loss": 6.087704249979775, + "tokens_seen": 21168128 + }, + { + "epoch": 0.01, + "learning_rate": 0.00032142857142857147, + "loss": 4.0665, + "theoretical_loss": 6.084546121650289, + "tokens_seen": 21233664 + }, + { + "epoch": 0.01, + "learning_rate": 0.00032242063492063497, + "loss": 4.2027, + "theoretical_loss": 6.081400445296899, + "tokens_seen": 21299200 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003234126984126984, + "loss": 4.129, + "theoretical_loss": 6.078267133735724, + "tokens_seen": 21364736 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003244047619047619, + "loss": 4.0394, + "theoretical_loss": 6.075146100658356, + "tokens_seen": 21430272 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003253968253968254, + "loss": 3.9669, + "theoretical_loss": 6.072037260620421, + "tokens_seen": 21495808 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003263888888888889, + "loss": 4.045, + "theoretical_loss": 6.0689405290303275, + "tokens_seen": 21561344 + }, + { + "epoch": 0.01, + "learning_rate": 0.00032738095238095237, + "loss": 4.0533, + "theoretical_loss": 6.065855822138193, + "tokens_seen": 21626880 + }, + { + "epoch": 0.01, + "learning_rate": 0.00032837301587301587, + "loss": 4.0483, + "theoretical_loss": 6.06278305702495, + "tokens_seen": 21692416 + }, + { + "epoch": 0.01, + "learning_rate": 0.00032936507936507937, + "loss": 4.0916, + "theoretical_loss": 6.05972215159162, + "tokens_seen": 21757952 + }, + { + "epoch": 0.01, + "learning_rate": 0.00033035714285714287, + "loss": 4.0242, + "theoretical_loss": 6.056673024548766, + "tokens_seen": 21823488 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003313492063492063, + "loss": 4.1523, + "theoretical_loss": 6.053635595406105, + "tokens_seen": 21889024 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003323412698412698, + "loss": 3.9993, + "theoretical_loss": 6.050609784462289, + "tokens_seen": 21954560 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003333333333333333, + "loss": 3.9836, + "theoretical_loss": 6.047595512794841, + "tokens_seen": 22020096 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003343253968253968, + "loss": 3.9448, + "theoretical_loss": 6.044592702250255, + "tokens_seen": 22085632 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003353174603174603, + "loss": 4.0473, + "theoretical_loss": 6.041601275434251, + "tokens_seen": 22151168 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003363095238095238, + "loss": 3.7588, + "theoretical_loss": 6.038621155702174, + "tokens_seen": 22216704 + }, + { + "epoch": 0.01, + "learning_rate": 0.00033730158730158733, + "loss": 4.1745, + "theoretical_loss": 6.0356522671495485, + "tokens_seen": 22282240 + }, + { + "epoch": 0.01, + "learning_rate": 0.00033829365079365083, + "loss": 4.0328, + "theoretical_loss": 6.032694534602784, + "tokens_seen": 22347776 + }, + { + "epoch": 0.01, + "learning_rate": 0.00033928571428571433, + "loss": 4.0154, + "theoretical_loss": 6.029747883610009, + "tokens_seen": 22413312 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003402777777777778, + "loss": 4.0547, + "theoretical_loss": 6.026812240432067, + "tokens_seen": 22478848 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003412698412698413, + "loss": 4.1954, + "theoretical_loss": 6.023887532033628, + "tokens_seen": 22544384 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003422619047619048, + "loss": 3.9873, + "theoretical_loss": 6.020973686074459, + "tokens_seen": 22609920 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003432539682539683, + "loss": 4.0026, + "theoretical_loss": 6.018070630900809, + "tokens_seen": 22675456 + }, + { + "epoch": 0.01, + "learning_rate": 0.00034424603174603173, + "loss": 4.0039, + "theoretical_loss": 6.015178295536933, + "tokens_seen": 22740992 + }, + { + "epoch": 0.01, + "learning_rate": 0.00034523809523809523, + "loss": 4.0788, + "theoretical_loss": 6.012296609676753, + "tokens_seen": 22806528 + }, + { + "epoch": 0.01, + "learning_rate": 0.00034623015873015873, + "loss": 3.9454, + "theoretical_loss": 6.009425503675624, + "tokens_seen": 22872064 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 14591, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 4.174915313720703, + "objective/train/theoretical_loss": 6.006564908542249, + "objective/train/tokens_used": 22937600, + "theoretical_loss": 6.006564908542249, + "tokens_seen": 22937600 + }, + { + "epoch": 0.01, + "learning_rate": 0.00034722222222222224, + "loss": 4.0212, + "theoretical_loss": 6.006564908542249, + "tokens_seen": 22937600 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003482142857142857, + "loss": 4.0426, + "theoretical_loss": 6.003714755930698, + "tokens_seen": 23003136 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003492063492063492, + "loss": 3.9484, + "theoretical_loss": 6.000874978132554, + "tokens_seen": 23068672 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003501984126984127, + "loss": 4.0467, + "theoretical_loss": 5.998045508069175, + "tokens_seen": 23134208 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003511904761904762, + "loss": 4.0756, + "theoretical_loss": 5.995226279284073, + "tokens_seen": 23199744 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003521825396825397, + "loss": 4.0251, + "theoretical_loss": 5.992417225935405, + "tokens_seen": 23265280 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003531746031746032, + "loss": 3.7968, + "theoretical_loss": 5.989618282788578, + "tokens_seen": 23330816 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003541666666666667, + "loss": 3.9415, + "theoretical_loss": 5.9868293852089565, + "tokens_seen": 23396352 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003551587301587302, + "loss": 3.8995, + "theoretical_loss": 5.984050469154685, + "tokens_seen": 23461888 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003561507936507937, + "loss": 3.9422, + "theoretical_loss": 5.981281471169619, + "tokens_seen": 23527424 + }, + { + "epoch": 0.01, + "learning_rate": 0.00035714285714285714, + "loss": 4.0012, + "theoretical_loss": 5.978522328376346, + "tokens_seen": 23592960 + }, + { + "epoch": 0.01, + "learning_rate": 0.00035813492063492064, + "loss": 4.1092, + "theoretical_loss": 5.975772978469321, + "tokens_seen": 23658496 + }, + { + "epoch": 0.01, + "learning_rate": 0.00035912698412698415, + "loss": 3.9273, + "theoretical_loss": 5.973033359708099, + "tokens_seen": 23724032 + }, + { + "epoch": 0.01, + "learning_rate": 0.00036011904761904765, + "loss": 3.9452, + "theoretical_loss": 5.9703034109106685, + "tokens_seen": 23789568 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003611111111111111, + "loss": 3.8854, + "theoretical_loss": 5.967583071446869, + "tokens_seen": 23855104 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003621031746031746, + "loss": 3.9328, + "theoretical_loss": 5.9648722812319255, + "tokens_seen": 23920640 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003630952380952381, + "loss": 3.9025, + "theoretical_loss": 5.962170980720055, + "tokens_seen": 23986176 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003640873015873016, + "loss": 3.9325, + "theoretical_loss": 5.959479110898181, + "tokens_seen": 24051712 + }, + { + "epoch": 0.01, + "learning_rate": 0.00036507936507936505, + "loss": 4.0022, + "theoretical_loss": 5.956796613279727, + "tokens_seen": 24117248 + }, + { + "epoch": 0.01, + "learning_rate": 0.00036607142857142855, + "loss": 3.9687, + "theoretical_loss": 5.954123429898508, + "tokens_seen": 24182784 + }, + { + "epoch": 0.01, + "learning_rate": 0.00036706349206349205, + "loss": 3.828, + "theoretical_loss": 5.951459503302701, + "tokens_seen": 24248320 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003680555555555556, + "loss": 3.9295, + "theoretical_loss": 5.948804776548908, + "tokens_seen": 24313856 + }, + { + "epoch": 0.01, + "learning_rate": 0.00036904761904761905, + "loss": 3.8142, + "theoretical_loss": 5.9461591931963, + "tokens_seen": 24379392 + }, + { + "epoch": 0.01, + "learning_rate": 0.00037003968253968255, + "loss": 3.829, + "theoretical_loss": 5.943522697300843, + "tokens_seen": 24444928 + }, + { + "epoch": 0.01, + "learning_rate": 0.00037103174603174606, + "loss": 3.8388, + "theoretical_loss": 5.940895233409613, + "tokens_seen": 24510464 + }, + { + "epoch": 0.01, + "learning_rate": 0.00037202380952380956, + "loss": 3.8946, + "theoretical_loss": 5.938276746555175, + "tokens_seen": 24576000 + }, + { + "epoch": 0.01, + "learning_rate": 0.000373015873015873, + "loss": 3.8477, + "theoretical_loss": 5.9356671822500635, + "tokens_seen": 24641536 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003740079365079365, + "loss": 3.9653, + "theoretical_loss": 5.933066486481318, + "tokens_seen": 24707072 + }, + { + "epoch": 0.01, + "learning_rate": 0.000375, + "loss": 3.8734, + "theoretical_loss": 5.930474605705113, + "tokens_seen": 24772608 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003759920634920635, + "loss": 4.0398, + "theoretical_loss": 5.927891486841449, + "tokens_seen": 24838144 + }, + { + "epoch": 0.01, + "learning_rate": 0.000376984126984127, + "loss": 3.9074, + "theoretical_loss": 5.925317077268927, + "tokens_seen": 24903680 + }, + { + "epoch": 0.01, + "learning_rate": 0.00037797619047619046, + "loss": 3.9102, + "theoretical_loss": 5.92275132481959, + "tokens_seen": 24969216 + }, + { + "epoch": 0.01, + "learning_rate": 0.00037896825396825396, + "loss": 3.9845, + "theoretical_loss": 5.9201941777738405, + "tokens_seen": 25034752 + }, + { + "epoch": 0.01, + "learning_rate": 0.00037996031746031746, + "loss": 3.8048, + "theoretical_loss": 5.917645584855421, + "tokens_seen": 25100288 + }, + { + "epoch": 0.01, + "learning_rate": 0.00038095238095238096, + "loss": 3.9786, + "theoretical_loss": 5.915105495226474, + "tokens_seen": 25165824 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003819444444444444, + "loss": 3.9157, + "theoretical_loss": 5.912573858482657, + "tokens_seen": 25231360 + }, + { + "epoch": 0.01, + "learning_rate": 0.00038293650793650797, + "loss": 3.9112, + "theoretical_loss": 5.910050624648335, + "tokens_seen": 25296896 + }, + { + "epoch": 0.01, + "learning_rate": 0.00038392857142857147, + "loss": 3.9547, + "theoretical_loss": 5.907535744171835, + "tokens_seen": 25362432 + }, + { + "epoch": 0.01, + "learning_rate": 0.00038492063492063497, + "loss": 3.9285, + "theoretical_loss": 5.905029167920765, + "tokens_seen": 25427968 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003859126984126984, + "loss": 3.7176, + "theoretical_loss": 5.902530847177394, + "tokens_seen": 25493504 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003869047619047619, + "loss": 3.8348, + "theoretical_loss": 5.900040733634098, + "tokens_seen": 25559040 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003878968253968254, + "loss": 3.8284, + "theoretical_loss": 5.897558779388872, + "tokens_seen": 25624576 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003888888888888889, + "loss": 3.7625, + "theoretical_loss": 5.895084936940887, + "tokens_seen": 25690112 + }, + { + "epoch": 0.01, + "learning_rate": 0.00038988095238095237, + "loss": 3.8807, + "theoretical_loss": 5.892619159186127, + "tokens_seen": 25755648 + }, + { + "epoch": 0.01, + "learning_rate": 0.00039087301587301587, + "loss": 3.8577, + "theoretical_loss": 5.890161399413071, + "tokens_seen": 25821184 + }, + { + "epoch": 0.01, + "learning_rate": 0.00039186507936507937, + "loss": 3.8169, + "theoretical_loss": 5.887711611298428, + "tokens_seen": 25886720 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003928571428571429, + "loss": 3.9106, + "theoretical_loss": 5.885269748902955, + "tokens_seen": 25952256 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003938492063492063, + "loss": 3.8295, + "theoretical_loss": 5.882835766667295, + "tokens_seen": 26017792 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003948412698412698, + "loss": 3.8442, + "theoretical_loss": 5.880409619407899, + "tokens_seen": 26083328 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003958333333333333, + "loss": 3.8974, + "theoretical_loss": 5.877991262312986, + "tokens_seen": 26148864 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 15466, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 3.919363498687744, + "objective/train/theoretical_loss": 5.875580650938565, + "objective/train/tokens_used": 26214400, + "theoretical_loss": 5.875580650938565, + "tokens_seen": 26214400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003968253968253968, + "loss": 3.9646, + "theoretical_loss": 5.875580650938565, + "tokens_seen": 26214400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003978174603174603, + "loss": 3.8902, + "theoretical_loss": 5.873177741204501, + "tokens_seen": 26279936 + }, + { + "epoch": 0.01, + "learning_rate": 0.00039880952380952383, + "loss": 3.7995, + "theoretical_loss": 5.8707824893906455, + "tokens_seen": 26345472 + }, + { + "epoch": 0.01, + "learning_rate": 0.00039980158730158733, + "loss": 3.717, + "theoretical_loss": 5.868394852133001, + "tokens_seen": 26411008 + }, + { + "epoch": 0.01, + "learning_rate": 0.00040079365079365083, + "loss": 3.7985, + "theoretical_loss": 5.866014786419951, + "tokens_seen": 26476544 + }, + { + "epoch": 0.01, + "learning_rate": 0.00040178571428571433, + "loss": 3.7711, + "theoretical_loss": 5.86364224958853, + "tokens_seen": 26542080 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004027777777777778, + "loss": 3.8253, + "theoretical_loss": 5.861277199320743, + "tokens_seen": 26607616 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004037698412698413, + "loss": 3.727, + "theoretical_loss": 5.858919593639941, + "tokens_seen": 26673152 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004047619047619048, + "loss": 3.9276, + "theoretical_loss": 5.856569390907229, + "tokens_seen": 26738688 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004057539682539683, + "loss": 4.0368, + "theoretical_loss": 5.854226549817936, + "tokens_seen": 26804224 + }, + { + "epoch": 0.01, + "learning_rate": 0.00040674603174603173, + "loss": 3.8356, + "theoretical_loss": 5.851891029398114, + "tokens_seen": 26869760 + }, + { + "epoch": 0.01, + "learning_rate": 0.00040773809523809523, + "loss": 3.8716, + "theoretical_loss": 5.8495627890011015, + "tokens_seen": 26935296 + }, + { + "epoch": 0.01, + "learning_rate": 0.00040873015873015874, + "loss": 3.7607, + "theoretical_loss": 5.847241788304114, + "tokens_seen": 27000832 + }, + { + "epoch": 0.01, + "learning_rate": 0.00040972222222222224, + "loss": 4.0169, + "theoretical_loss": 5.844927987304884, + "tokens_seen": 27066368 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004107142857142857, + "loss": 3.7878, + "theoretical_loss": 5.842621346318348, + "tokens_seen": 27131904 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004117063492063492, + "loss": 3.7259, + "theoretical_loss": 5.84032182597337, + "tokens_seen": 27197440 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004126984126984127, + "loss": 3.796, + "theoretical_loss": 5.8380293872095095, + "tokens_seen": 27262976 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004136904761904762, + "loss": 3.7734, + "theoretical_loss": 5.835743991273832, + "tokens_seen": 27328512 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004146825396825397, + "loss": 3.7416, + "theoretical_loss": 5.833465599717752, + "tokens_seen": 27394048 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004156746031746032, + "loss": 3.8185, + "theoretical_loss": 5.831194174393929, + "tokens_seen": 27459584 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004166666666666667, + "loss": 3.7381, + "theoretical_loss": 5.828929677453187, + "tokens_seen": 27525120 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004176587301587302, + "loss": 3.8399, + "theoretical_loss": 5.826672071341488, + "tokens_seen": 27590656 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004186507936507937, + "loss": 3.7886, + "theoretical_loss": 5.824421318796932, + "tokens_seen": 27656192 + }, + { + "epoch": 0.01, + "learning_rate": 0.00041964285714285714, + "loss": 3.74, + "theoretical_loss": 5.822177382846799, + "tokens_seen": 27721728 + }, + { + "epoch": 0.01, + "learning_rate": 0.00042063492063492065, + "loss": 3.8009, + "theoretical_loss": 5.8199402268046265, + "tokens_seen": 27787264 + }, + { + "epoch": 0.01, + "learning_rate": 0.00042162698412698415, + "loss": 3.7421, + "theoretical_loss": 5.817709814267329, + "tokens_seen": 27852800 + }, + { + "epoch": 0.01, + "learning_rate": 0.00042261904761904765, + "loss": 3.7592, + "theoretical_loss": 5.815486109112342, + "tokens_seen": 27918336 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004236111111111111, + "loss": 3.7261, + "theoretical_loss": 5.813269075494812, + "tokens_seen": 27983872 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004246031746031746, + "loss": 3.7071, + "theoretical_loss": 5.811058677844817, + "tokens_seen": 28049408 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004255952380952381, + "loss": 3.8082, + "theoretical_loss": 5.808854880864618, + "tokens_seen": 28114944 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004265873015873016, + "loss": 3.7002, + "theoretical_loss": 5.8066576495259525, + "tokens_seen": 28180480 + }, + { + "epoch": 0.01, + "learning_rate": 0.00042757936507936505, + "loss": 3.7413, + "theoretical_loss": 5.804466949067352, + "tokens_seen": 28246016 + }, + { + "epoch": 0.01, + "learning_rate": 0.00042857142857142855, + "loss": 3.7295, + "theoretical_loss": 5.8022827449914995, + "tokens_seen": 28311552 + }, + { + "epoch": 0.01, + "learning_rate": 0.00042956349206349205, + "loss": 3.7315, + "theoretical_loss": 5.800105003062612, + "tokens_seen": 28377088 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004305555555555556, + "loss": 3.8065, + "theoretical_loss": 5.797933689303866, + "tokens_seen": 28442624 + }, + { + "epoch": 0.01, + "learning_rate": 0.00043154761904761905, + "loss": 3.7531, + "theoretical_loss": 5.795768769994842, + "tokens_seen": 28508160 + }, + { + "epoch": 0.01, + "learning_rate": 0.00043253968253968256, + "loss": 3.8029, + "theoretical_loss": 5.793610211669004, + "tokens_seen": 28573696 + }, + { + "epoch": 0.01, + "learning_rate": 0.00043353174603174606, + "loss": 3.866, + "theoretical_loss": 5.79145798111122, + "tokens_seen": 28639232 + }, + { + "epoch": 0.01, + "learning_rate": 0.00043452380952380956, + "loss": 3.6966, + "theoretical_loss": 5.789312045355292, + "tokens_seen": 28704768 + }, + { + "epoch": 0.01, + "learning_rate": 0.000435515873015873, + "loss": 3.601, + "theoretical_loss": 5.787172371681535, + "tokens_seen": 28770304 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004365079365079365, + "loss": 3.6692, + "theoretical_loss": 5.785038927614373, + "tokens_seen": 28835840 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004375, + "loss": 3.7387, + "theoretical_loss": 5.782911680919971, + "tokens_seen": 28901376 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004384920634920635, + "loss": 3.7303, + "theoretical_loss": 5.78079059960389, + "tokens_seen": 28966912 + }, + { + "epoch": 0.01, + "learning_rate": 0.000439484126984127, + "loss": 3.7274, + "theoretical_loss": 5.77867565190877, + "tokens_seen": 29032448 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044047619047619046, + "loss": 3.8177, + "theoretical_loss": 5.776566806312052, + "tokens_seen": 29097984 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044146825396825396, + "loss": 3.6899, + "theoretical_loss": 5.774464031523705, + "tokens_seen": 29163520 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044246031746031746, + "loss": 3.7763, + "theoretical_loss": 5.772367296484001, + "tokens_seen": 29229056 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044345238095238096, + "loss": 3.7005, + "theoretical_loss": 5.770276570361309, + "tokens_seen": 29294592 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004444444444444444, + "loss": 3.7144, + "theoretical_loss": 5.768191822549908, + "tokens_seen": 29360128 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044543650793650797, + "loss": 3.7047, + "theoretical_loss": 5.766113022667838, + "tokens_seen": 29425664 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 16709, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 3.606027603149414, + "objective/train/theoretical_loss": 5.764040140554766, + "objective/train/tokens_used": 29491200, + "theoretical_loss": 5.764040140554766, + "tokens_seen": 29491200 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044642857142857147, + "loss": 3.5723, + "theoretical_loss": 5.764040140554766, + "tokens_seen": 29491200 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044742063492063497, + "loss": 3.7567, + "theoretical_loss": 5.761973146269881, + "tokens_seen": 29556736 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004484126984126984, + "loss": 3.5865, + "theoretical_loss": 5.759912010089819, + "tokens_seen": 29622272 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004494047619047619, + "loss": 3.5789, + "theoretical_loss": 5.757856702506597, + "tokens_seen": 29687808 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004503968253968254, + "loss": 3.7197, + "theoretical_loss": 5.755807194225595, + "tokens_seen": 29753344 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004513888888888889, + "loss": 3.5679, + "theoretical_loss": 5.753763456163529, + "tokens_seen": 29818880 + }, + { + "epoch": 0.01, + "learning_rate": 0.00045238095238095237, + "loss": 3.7968, + "theoretical_loss": 5.751725459446487, + "tokens_seen": 29884416 + }, + { + "epoch": 0.01, + "learning_rate": 0.00045337301587301587, + "loss": 3.5949, + "theoretical_loss": 5.7496931754079466, + "tokens_seen": 29949952 + }, + { + "epoch": 0.01, + "learning_rate": 0.00045436507936507937, + "loss": 3.6958, + "theoretical_loss": 5.747666575586849, + "tokens_seen": 30015488 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004553571428571429, + "loss": 3.812, + "theoretical_loss": 5.7456456317256785, + "tokens_seen": 30081024 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004563492063492063, + "loss": 3.6283, + "theoretical_loss": 5.743630315768563, + "tokens_seen": 30146560 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004573412698412698, + "loss": 3.7527, + "theoretical_loss": 5.741620599859402, + "tokens_seen": 30212096 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004583333333333333, + "loss": 3.4671, + "theoretical_loss": 5.739616456340021, + "tokens_seen": 30277632 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004593253968253968, + "loss": 3.6428, + "theoretical_loss": 5.737617857748329, + "tokens_seen": 30343168 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046031746031746033, + "loss": 3.6583, + "theoretical_loss": 5.73562477681652, + "tokens_seen": 30408704 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046130952380952383, + "loss": 3.4444, + "theoretical_loss": 5.733637186469274, + "tokens_seen": 30474240 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046230158730158733, + "loss": 3.5912, + "theoretical_loss": 5.731655059821993, + "tokens_seen": 30539776 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046329365079365083, + "loss": 3.6914, + "theoretical_loss": 5.729678370179052, + "tokens_seen": 30605312 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046428571428571433, + "loss": 3.6253, + "theoretical_loss": 5.727707091032062, + "tokens_seen": 30670848 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004652777777777778, + "loss": 3.7508, + "theoretical_loss": 5.725741196058175, + "tokens_seen": 30736384 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004662698412698413, + "loss": 3.6997, + "theoretical_loss": 5.723780659118377, + "tokens_seen": 30801920 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004672619047619048, + "loss": 3.551, + "theoretical_loss": 5.721825454255825, + "tokens_seen": 30867456 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004682539682539683, + "loss": 3.6614, + "theoretical_loss": 5.71987555569419, + "tokens_seen": 30932992 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046924603174603173, + "loss": 3.7331, + "theoretical_loss": 5.717930937836029, + "tokens_seen": 30998528 + }, + { + "epoch": 0.01, + "learning_rate": 0.00047023809523809523, + "loss": 3.5684, + "theoretical_loss": 5.715991575261159, + "tokens_seen": 31064064 + }, + { + "epoch": 0.01, + "learning_rate": 0.00047123015873015874, + "loss": 3.5283, + "theoretical_loss": 5.714057442725068, + "tokens_seen": 31129600 + }, + { + "epoch": 0.01, + "learning_rate": 0.00047222222222222224, + "loss": 3.7344, + "theoretical_loss": 5.712128515157324, + "tokens_seen": 31195136 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004732142857142857, + "loss": 3.4604, + "theoretical_loss": 5.710204767660022, + "tokens_seen": 31260672 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004742063492063492, + "loss": 3.5904, + "theoretical_loss": 5.708286175506232, + "tokens_seen": 31326208 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004751984126984127, + "loss": 3.6531, + "theoretical_loss": 5.706372714138472, + "tokens_seen": 31391744 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004761904761904762, + "loss": 3.4382, + "theoretical_loss": 5.704464359167195, + "tokens_seen": 31457280 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004771825396825397, + "loss": 3.6047, + "theoretical_loss": 5.702561086369294, + "tokens_seen": 31522816 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004781746031746032, + "loss": 3.6999, + "theoretical_loss": 5.7006628716866246, + "tokens_seen": 31588352 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004791666666666667, + "loss": 3.6113, + "theoretical_loss": 5.698769691224539, + "tokens_seen": 31653888 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004801587301587302, + "loss": 3.5066, + "theoretical_loss": 5.696881521250441, + "tokens_seen": 31719424 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004811507936507937, + "loss": 3.5783, + "theoretical_loss": 5.694998338192356, + "tokens_seen": 31784960 + }, + { + "epoch": 0.01, + "learning_rate": 0.00048214285714285715, + "loss": 3.684, + "theoretical_loss": 5.693120118637513, + "tokens_seen": 31850496 + }, + { + "epoch": 0.01, + "learning_rate": 0.00048313492063492065, + "loss": 3.626, + "theoretical_loss": 5.691246839330946, + "tokens_seen": 31916032 + }, + { + "epoch": 0.01, + "learning_rate": 0.00048412698412698415, + "loss": 3.6495, + "theoretical_loss": 5.689378477174106, + "tokens_seen": 31981568 + }, + { + "epoch": 0.01, + "learning_rate": 0.00048511904761904765, + "loss": 3.4647, + "theoretical_loss": 5.687515009223498, + "tokens_seen": 32047104 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004861111111111111, + "loss": 3.6183, + "theoretical_loss": 5.685656412689318, + "tokens_seen": 32112640 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004871031746031746, + "loss": 3.5805, + "theoretical_loss": 5.683802664934115, + "tokens_seen": 32178176 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004880952380952381, + "loss": 3.4505, + "theoretical_loss": 5.681953743471466, + "tokens_seen": 32243712 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004890873015873016, + "loss": 3.5045, + "theoretical_loss": 5.680109625964663, + "tokens_seen": 32309248 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004900793650793651, + "loss": 3.4528, + "theoretical_loss": 5.678270290225415, + "tokens_seen": 32374784 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004910714285714286, + "loss": 3.5019, + "theoretical_loss": 5.676435714212562, + "tokens_seen": 32440320 + }, + { + "epoch": 0.01, + "learning_rate": 0.000492063492063492, + "loss": 3.3658, + "theoretical_loss": 5.674605876030808, + "tokens_seen": 32505856 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004930555555555556, + "loss": 3.4662, + "theoretical_loss": 5.672780753929461, + "tokens_seen": 32571392 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004940476190476191, + "loss": 3.525, + "theoretical_loss": 5.67096032630119, + "tokens_seen": 32636928 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004950396825396826, + "loss": 3.4717, + "theoretical_loss": 5.669144571680797, + "tokens_seen": 32702464 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 17558, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 3.467946767807007, + "objective/train/theoretical_loss": 5.667333468743994, + "objective/train/tokens_used": 32768000, + "theoretical_loss": 5.667333468743994, + "tokens_seen": 32768000 + }, + { + "epoch": 0.01, + "learning_rate": 0.000496031746031746, + "loss": 3.3898, + "theoretical_loss": 5.667333468743994, + "tokens_seen": 32768000 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004970238095238095, + "loss": 3.5133, + "theoretical_loss": 5.665526996306205, + "tokens_seen": 32833536 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498015873015873, + "loss": 3.3887, + "theoretical_loss": 5.663725133321369, + "tokens_seen": 32899072 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990079365079365, + "loss": 3.5714, + "theoretical_loss": 5.661927858880761, + "tokens_seen": 32964608 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005, + "loss": 3.7606, + "theoretical_loss": 5.6601351522118275, + "tokens_seen": 33030144 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999899699097292, + "loss": 3.5544, + "theoretical_loss": 5.65834699267703, + "tokens_seen": 33095680 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999799398194584, + "loss": 3.5567, + "theoretical_loss": 5.6565633597727025, + "tokens_seen": 33161216 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999699097291876, + "loss": 3.4876, + "theoretical_loss": 5.65478423312792, + "tokens_seen": 33226752 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999598796389167, + "loss": 3.5821, + "theoretical_loss": 5.65300959250338, + "tokens_seen": 33292288 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499949849548646, + "loss": 3.7036, + "theoretical_loss": 5.651239417790293, + "tokens_seen": 33357824 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999398194583751, + "loss": 3.5064, + "theoretical_loss": 5.649473689009292, + "tokens_seen": 33423360 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999297893681044, + "loss": 3.5474, + "theoretical_loss": 5.6477123863093395, + "tokens_seen": 33488896 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999197592778335, + "loss": 3.4653, + "theoretical_loss": 5.645955489966661, + "tokens_seen": 33554432 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999097291875627, + "loss": 3.399, + "theoretical_loss": 5.644202980383678, + "tokens_seen": 33619968 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998996990972919, + "loss": 3.5306, + "theoretical_loss": 5.6424548380879616, + "tokens_seen": 33685504 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998896690070211, + "loss": 3.4728, + "theoretical_loss": 5.640711043731184, + "tokens_seen": 33751040 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998796389167503, + "loss": 3.4433, + "theoretical_loss": 5.6389715780880945, + "tokens_seen": 33816576 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998696088264795, + "loss": 3.5626, + "theoretical_loss": 5.6372364220555, + "tokens_seen": 33882112 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998595787362087, + "loss": 3.5993, + "theoretical_loss": 5.635505556651251, + "tokens_seen": 33947648 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998495486459378, + "loss": 3.3909, + "theoretical_loss": 5.633778963013247, + "tokens_seen": 34013184 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499839518555667, + "loss": 3.4752, + "theoretical_loss": 5.632056622398448, + "tokens_seen": 34078720 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998294884653962, + "loss": 3.6171, + "theoretical_loss": 5.63033851618189, + "tokens_seen": 34144256 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998194583751254, + "loss": 3.5379, + "theoretical_loss": 5.628624625855725, + "tokens_seen": 34209792 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998094282848546, + "loss": 3.3854, + "theoretical_loss": 5.626914933028255, + "tokens_seen": 34275328 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997993981945837, + "loss": 3.6787, + "theoretical_loss": 5.625209419422983, + "tokens_seen": 34340864 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997893681043129, + "loss": 3.4415, + "theoretical_loss": 5.623508066877678, + "tokens_seen": 34406400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997793380140421, + "loss": 3.4248, + "theoretical_loss": 5.621810857343441, + "tokens_seen": 34471936 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997693079237714, + "loss": 3.5621, + "theoretical_loss": 5.620117772883784, + "tokens_seen": 34537472 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997592778335005, + "loss": 3.3828, + "theoretical_loss": 5.618428795673718, + "tokens_seen": 34603008 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997492477432298, + "loss": 3.4984, + "theoretical_loss": 5.616743907998853, + "tokens_seen": 34668544 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997392176529588, + "loss": 3.3977, + "theoretical_loss": 5.6150630922545, + "tokens_seen": 34734080 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997291875626881, + "loss": 3.395, + "theoretical_loss": 5.613386330944792, + "tokens_seen": 34799616 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997191574724173, + "loss": 3.3643, + "theoretical_loss": 5.6117136066818, + "tokens_seen": 34865152 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997091273821465, + "loss": 3.5862, + "theoretical_loss": 5.610044902184672, + "tokens_seen": 34930688 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996990972918757, + "loss": 3.5718, + "theoretical_loss": 5.608380200278775, + "tokens_seen": 34996224 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996890672016048, + "loss": 3.5601, + "theoretical_loss": 5.606719483894837, + "tokens_seen": 35061760 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499679037111334, + "loss": 3.5172, + "theoretical_loss": 5.605062736068113, + "tokens_seen": 35127296 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996690070210632, + "loss": 3.4024, + "theoretical_loss": 5.603409939937547, + "tokens_seen": 35192832 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996589769307924, + "loss": 3.6152, + "theoretical_loss": 5.601761078744952, + "tokens_seen": 35258368 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996489468405216, + "loss": 3.3564, + "theoretical_loss": 5.600116135834183, + "tokens_seen": 35323904 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996389167502507, + "loss": 3.4248, + "theoretical_loss": 5.598475094650334, + "tokens_seen": 35389440 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996288866599799, + "loss": 3.4177, + "theoretical_loss": 5.59683793873894, + "tokens_seen": 35454976 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996188565697091, + "loss": 3.3269, + "theoretical_loss": 5.595204651745176, + "tokens_seen": 35520512 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996088264794383, + "loss": 3.4723, + "theoretical_loss": 5.593575217413074, + "tokens_seen": 35586048 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995987963891675, + "loss": 3.4936, + "theoretical_loss": 5.591949619584746, + "tokens_seen": 35651584 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995887662988968, + "loss": 3.4967, + "theoretical_loss": 5.590327842199611, + "tokens_seen": 35717120 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995787362086258, + "loss": 3.4435, + "theoretical_loss": 5.588709869293634, + "tokens_seen": 35782656 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995687061183551, + "loss": 3.3488, + "theoretical_loss": 5.5870956849985705, + "tokens_seen": 35848192 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995586760280842, + "loss": 3.3003, + "theoretical_loss": 5.585485273541212, + "tokens_seen": 35913728 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995486459378135, + "loss": 3.5302, + "theoretical_loss": 5.583878619242652, + "tokens_seen": 35979264 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 18719, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 3.4601385593414307, + "objective/train/theoretical_loss": 5.582275706517549, + "objective/train/tokens_used": 36044800, + "theoretical_loss": 5.582275706517549, + "tokens_seen": 36044800 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995386158475427, + "loss": 3.4621, + "theoretical_loss": 5.582275706517549, + "tokens_seen": 36044800 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995285857572718, + "loss": 3.4038, + "theoretical_loss": 5.580676519873396, + "tokens_seen": 36110336 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499518555667001, + "loss": 3.3501, + "theoretical_loss": 5.579081043909808, + "tokens_seen": 36175872 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995085255767302, + "loss": 3.383, + "theoretical_loss": 5.577489263317796, + "tokens_seen": 36241408 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994984954864594, + "loss": 3.4061, + "theoretical_loss": 5.5759011628790764, + "tokens_seen": 36306944 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994884653961886, + "loss": 3.5433, + "theoretical_loss": 5.574316727465358, + "tokens_seen": 36372480 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994784353059178, + "loss": 3.4512, + "theoretical_loss": 5.572735942037655, + "tokens_seen": 36438016 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994684052156469, + "loss": 3.4229, + "theoretical_loss": 5.571158791645597, + "tokens_seen": 36503552 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994583751253761, + "loss": 3.43, + "theoretical_loss": 5.569585261426752, + "tokens_seen": 36569088 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994483450351053, + "loss": 3.3882, + "theoretical_loss": 5.568015336605953, + "tokens_seen": 36634624 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994383149448345, + "loss": 3.3435, + "theoretical_loss": 5.566449002494627, + "tokens_seen": 36700160 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994282848545637, + "loss": 3.3699, + "theoretical_loss": 5.564886244490136, + "tokens_seen": 36765696 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994182547642928, + "loss": 3.3147, + "theoretical_loss": 5.563327048075124, + "tokens_seen": 36831232 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994082246740221, + "loss": 3.4415, + "theoretical_loss": 5.5617713988168616, + "tokens_seen": 36896768 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993981945837512, + "loss": 3.23, + "theoretical_loss": 5.560219282366614, + "tokens_seen": 36962304 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993881644934805, + "loss": 3.3128, + "theoretical_loss": 5.558670684458994, + "tokens_seen": 37027840 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993781344032096, + "loss": 3.5197, + "theoretical_loss": 5.557125590911339, + "tokens_seen": 37093376 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993681043129389, + "loss": 3.2435, + "theoretical_loss": 5.5555839876230815, + "tokens_seen": 37158912 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499358074222668, + "loss": 3.4773, + "theoretical_loss": 5.55404586057513, + "tokens_seen": 37224448 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993480441323972, + "loss": 3.5102, + "theoretical_loss": 5.5525111958292594, + "tokens_seen": 37289984 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993380140421264, + "loss": 3.3631, + "theoretical_loss": 5.550979979527501, + "tokens_seen": 37355520 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993279839518556, + "loss": 3.4903, + "theoretical_loss": 5.549452197891538, + "tokens_seen": 37421056 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993179538615848, + "loss": 3.2782, + "theoretical_loss": 5.547927837222119, + "tokens_seen": 37486592 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993079237713139, + "loss": 3.3585, + "theoretical_loss": 5.546406883898451, + "tokens_seen": 37552128 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992978936810431, + "loss": 3.3905, + "theoretical_loss": 5.544889324377628, + "tokens_seen": 37617664 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992878635907723, + "loss": 3.3584, + "theoretical_loss": 5.543375145194048, + "tokens_seen": 37683200 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992778335005015, + "loss": 3.4642, + "theoretical_loss": 5.541864332958831, + "tokens_seen": 37748736 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992678034102307, + "loss": 3.4594, + "theoretical_loss": 5.540356874359257, + "tokens_seen": 37814272 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992577733199598, + "loss": 3.3889, + "theoretical_loss": 5.538852756158201, + "tokens_seen": 37879808 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499247743229689, + "loss": 3.4217, + "theoretical_loss": 5.537351965193569, + "tokens_seen": 37945344 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992377131394183, + "loss": 3.3643, + "theoretical_loss": 5.535854488377751, + "tokens_seen": 38010880 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992276830491475, + "loss": 3.3478, + "theoretical_loss": 5.534360312697064, + "tokens_seen": 38076416 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992176529588767, + "loss": 3.2531, + "theoretical_loss": 5.532869425211214, + "tokens_seen": 38141952 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992076228686059, + "loss": 3.274, + "theoretical_loss": 5.531381813052757, + "tokens_seen": 38207488 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499197592778335, + "loss": 3.2343, + "theoretical_loss": 5.529897463426559, + "tokens_seen": 38273024 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991875626880642, + "loss": 3.3217, + "theoretical_loss": 5.528416363609276, + "tokens_seen": 38338560 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991775325977934, + "loss": 3.4237, + "theoretical_loss": 5.526938500948817, + "tokens_seen": 38404096 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991675025075226, + "loss": 3.2912, + "theoretical_loss": 5.525463862863836, + "tokens_seen": 38469632 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991574724172518, + "loss": 3.3005, + "theoretical_loss": 5.523992436843209, + "tokens_seen": 38535168 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499147442326981, + "loss": 3.3535, + "theoretical_loss": 5.522524210445528, + "tokens_seen": 38600704 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991374122367101, + "loss": 3.2041, + "theoretical_loss": 5.521059171298592, + "tokens_seen": 38666240 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991273821464393, + "loss": 3.2325, + "theoretical_loss": 5.519597307098907, + "tokens_seen": 38731776 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991173520561685, + "loss": 3.3404, + "theoretical_loss": 5.518138605611188, + "tokens_seen": 38797312 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991073219658977, + "loss": 3.4651, + "theoretical_loss": 5.516683054667868, + "tokens_seen": 38862848 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499097291875627, + "loss": 3.4364, + "theoretical_loss": 5.515230642168607, + "tokens_seen": 38928384 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499087261785356, + "loss": 3.3549, + "theoretical_loss": 5.513781356079811, + "tokens_seen": 38993920 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990772316950853, + "loss": 3.359, + "theoretical_loss": 5.512335184434152, + "tokens_seen": 39059456 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990672016048144, + "loss": 3.1655, + "theoretical_loss": 5.5108921153300905, + "tokens_seen": 39124992 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990571715145437, + "loss": 3.271, + "theoretical_loss": 5.5094521369314045, + "tokens_seen": 39190528 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990471414242729, + "loss": 3.386, + "theoretical_loss": 5.5080152374667275, + "tokens_seen": 39256064 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 19585, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 3.43406343460083, + "objective/train/theoretical_loss": 5.506581405229079, + "objective/train/tokens_used": 39321600, + "theoretical_loss": 5.506581405229079, + "tokens_seen": 39321600 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499037111334002, + "loss": 3.3639, + "theoretical_loss": 5.506581405229079, + "tokens_seen": 39321600 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990270812437312, + "loss": 3.3511, + "theoretical_loss": 5.505150628575409, + "tokens_seen": 39387136 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990170511534604, + "loss": 3.3222, + "theoretical_loss": 5.503722895926143, + "tokens_seen": 39452672 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990070210631896, + "loss": 3.2525, + "theoretical_loss": 5.50229819576473, + "tokens_seen": 39518208 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989969909729188, + "loss": 3.3258, + "theoretical_loss": 5.500876516637199, + "tokens_seen": 39583744 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498986960882648, + "loss": 3.3465, + "theoretical_loss": 5.49945784715171, + "tokens_seen": 39649280 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989769307923771, + "loss": 3.2204, + "theoretical_loss": 5.498042175978123, + "tokens_seen": 39714816 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989669007021063, + "loss": 3.2699, + "theoretical_loss": 5.496629491847553, + "tokens_seen": 39780352 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989568706118355, + "loss": 3.1906, + "theoretical_loss": 5.495219783551947, + "tokens_seen": 39845888 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989468405215647, + "loss": 3.265, + "theoretical_loss": 5.493813039943654, + "tokens_seen": 39911424 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989368104312939, + "loss": 3.3007, + "theoretical_loss": 5.492409249934996, + "tokens_seen": 39976960 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498926780341023, + "loss": 3.2557, + "theoretical_loss": 5.491008402497855, + "tokens_seen": 40042496 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989167502507523, + "loss": 3.1805, + "theoretical_loss": 5.489610486663252, + "tokens_seen": 40108032 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989067201604814, + "loss": 3.0619, + "theoretical_loss": 5.488215491520933, + "tokens_seen": 40173568 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988966900702107, + "loss": 3.316, + "theoretical_loss": 5.486823406218962, + "tokens_seen": 40239104 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988866599799398, + "loss": 3.286, + "theoretical_loss": 5.485434219963315, + "tokens_seen": 40304640 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988766298896691, + "loss": 3.3864, + "theoretical_loss": 5.484047922017476, + "tokens_seen": 40370176 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988665997993982, + "loss": 3.1815, + "theoretical_loss": 5.482664501702036, + "tokens_seen": 40435712 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988565697091274, + "loss": 3.2832, + "theoretical_loss": 5.481283948394303, + "tokens_seen": 40501248 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988465396188566, + "loss": 3.1921, + "theoretical_loss": 5.479906251527901, + "tokens_seen": 40566784 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988365095285858, + "loss": 3.2684, + "theoretical_loss": 5.478531400592393, + "tokens_seen": 40632320 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498826479438315, + "loss": 2.9848, + "theoretical_loss": 5.4771593851328815, + "tokens_seen": 40697856 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988164493480441, + "loss": 3.2648, + "theoretical_loss": 5.475790194749637, + "tokens_seen": 40763392 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988064192577733, + "loss": 3.1568, + "theoretical_loss": 5.474423819097713, + "tokens_seen": 40828928 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987963891675025, + "loss": 3.3457, + "theoretical_loss": 5.473060247886572, + "tokens_seen": 40894464 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987863590772317, + "loss": 3.2702, + "theoretical_loss": 5.471699470879715, + "tokens_seen": 40960000 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987763289869609, + "loss": 3.2031, + "theoretical_loss": 5.470341477894303, + "tokens_seen": 41025536 + }, + { + "epoch": 0.01, + "learning_rate": 0.00049876629889669, + "loss": 3.1032, + "theoretical_loss": 5.468986258800805, + "tokens_seen": 41091072 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987562688064192, + "loss": 3.2353, + "theoretical_loss": 5.4676338035226175, + "tokens_seen": 41156608 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987462387161484, + "loss": 3.1287, + "theoretical_loss": 5.466284102035719, + "tokens_seen": 41222144 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987362086258777, + "loss": 3.0774, + "theoretical_loss": 5.464937144368307, + "tokens_seen": 41287680 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987261785356068, + "loss": 3.1591, + "theoretical_loss": 5.463592920600435, + "tokens_seen": 41353216 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987161484453361, + "loss": 3.303, + "theoretical_loss": 5.46225142086368, + "tokens_seen": 41418752 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987061183550651, + "loss": 3.3303, + "theoretical_loss": 5.460912635340775, + "tokens_seen": 41484288 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986960882647944, + "loss": 3.0398, + "theoretical_loss": 5.459576554265276, + "tokens_seen": 41549824 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986860581745236, + "loss": 3.245, + "theoretical_loss": 5.458243167921212, + "tokens_seen": 41615360 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986760280842528, + "loss": 3.2602, + "theoretical_loss": 5.456912466642748, + "tokens_seen": 41680896 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498665997993982, + "loss": 3.1023, + "theoretical_loss": 5.455584440813853, + "tokens_seen": 41746432 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986559679037111, + "loss": 3.1868, + "theoretical_loss": 5.454259080867951, + "tokens_seen": 41811968 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986459378134403, + "loss": 3.3109, + "theoretical_loss": 5.4529363772876085, + "tokens_seen": 41877504 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986359077231695, + "loss": 3.2151, + "theoretical_loss": 5.451616320604193, + "tokens_seen": 41943040 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986258776328987, + "loss": 3.2158, + "theoretical_loss": 5.450298901397547, + "tokens_seen": 42008576 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986158475426279, + "loss": 3.2152, + "theoretical_loss": 5.448984110295672, + "tokens_seen": 42074112 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498605817452357, + "loss": 3.1239, + "theoretical_loss": 5.447671937974404, + "tokens_seen": 42139648 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985957873620862, + "loss": 3.1448, + "theoretical_loss": 5.446362375157092, + "tokens_seen": 42205184 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985857572718154, + "loss": 3.1766, + "theoretical_loss": 5.445055412614287, + "tokens_seen": 42270720 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985757271815446, + "loss": 3.2248, + "theoretical_loss": 5.443751041163428, + "tokens_seen": 42336256 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985656970912738, + "loss": 3.1846, + "theoretical_loss": 5.442449251668533, + "tokens_seen": 42401792 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985556670010031, + "loss": 3.1763, + "theoretical_loss": 5.441150035039886, + "tokens_seen": 42467328 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985456369107321, + "loss": 3.1093, + "theoretical_loss": 5.439853382233738, + "tokens_seen": 42532864 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 20818, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 3.2105917930603027, + "objective/train/theoretical_loss": 5.438559284252003, + "objective/train/tokens_used": 42598400, + "theoretical_loss": 5.438559284252003, + "tokens_seen": 42598400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985356068204614, + "loss": 3.2079, + "theoretical_loss": 5.438559284252003, + "tokens_seen": 42598400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985255767301905, + "loss": 3.0898, + "theoretical_loss": 5.437267732141949, + "tokens_seen": 42663936 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985155466399198, + "loss": 3.1634, + "theoretical_loss": 5.435978716995917, + "tokens_seen": 42729472 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498505516549649, + "loss": 3.2145, + "theoretical_loss": 5.434692229951009, + "tokens_seen": 42795008 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984954864593782, + "loss": 3.1525, + "theoretical_loss": 5.433408262188802, + "tokens_seen": 42860544 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984854563691073, + "loss": 3.0955, + "theoretical_loss": 5.432126804935059, + "tokens_seen": 42926080 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984754262788365, + "loss": 3.1633, + "theoretical_loss": 5.430847849459438, + "tokens_seen": 42991616 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984653961885657, + "loss": 3.1462, + "theoretical_loss": 5.429571387075203, + "tokens_seen": 43057152 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984553660982949, + "loss": 3.1308, + "theoretical_loss": 5.428297409138947, + "tokens_seen": 43122688 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984453360080241, + "loss": 3.1652, + "theoretical_loss": 5.427025907050307, + "tokens_seen": 43188224 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984353059177532, + "loss": 3.2837, + "theoretical_loss": 5.425756872251682, + "tokens_seen": 43253760 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984252758274825, + "loss": 3.0943, + "theoretical_loss": 5.42449029622796, + "tokens_seen": 43319296 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984152457372116, + "loss": 3.0748, + "theoretical_loss": 5.423226170506243, + "tokens_seen": 43384832 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984052156469409, + "loss": 3.182, + "theoretical_loss": 5.421964486655567, + "tokens_seen": 43450368 + }, + { + "epoch": 0.01, + "learning_rate": 0.00049839518555667, + "loss": 2.9998, + "theoretical_loss": 5.420705236286646, + "tokens_seen": 43515904 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983851554663993, + "loss": 3.1283, + "theoretical_loss": 5.41944841105159, + "tokens_seen": 43581440 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983751253761284, + "loss": 3.1802, + "theoretical_loss": 5.418194002643643, + "tokens_seen": 43646976 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983650952858576, + "loss": 3.0812, + "theoretical_loss": 5.416942002796926, + "tokens_seen": 43712512 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983550651955868, + "loss": 3.1278, + "theoretical_loss": 5.415692403286162, + "tokens_seen": 43778048 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498345035105316, + "loss": 3.1487, + "theoretical_loss": 5.414445195926426, + "tokens_seen": 43843584 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983350050150452, + "loss": 2.8856, + "theoretical_loss": 5.413200372572884, + "tokens_seen": 43909120 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983249749247743, + "loss": 3.2621, + "theoretical_loss": 5.411957925120538, + "tokens_seen": 43974656 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983149448345035, + "loss": 3.0968, + "theoretical_loss": 5.410717845503969, + "tokens_seen": 44040192 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983049147442327, + "loss": 3.1076, + "theoretical_loss": 5.40948012569709, + "tokens_seen": 44105728 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982948846539619, + "loss": 3.1407, + "theoretical_loss": 5.408244757712893, + "tokens_seen": 44171264 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982848545636911, + "loss": 3.0824, + "theoretical_loss": 5.407011733603199, + "tokens_seen": 44236800 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982748244734202, + "loss": 3.0406, + "theoretical_loss": 5.4057810454584185, + "tokens_seen": 44302336 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982647943831494, + "loss": 2.9378, + "theoretical_loss": 5.404552685407301, + "tokens_seen": 44367872 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982547642928786, + "loss": 3.1573, + "theoretical_loss": 5.403326645616698, + "tokens_seen": 44433408 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982447342026079, + "loss": 3.0197, + "theoretical_loss": 5.402102918291316, + "tokens_seen": 44498944 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498234704112337, + "loss": 3.1508, + "theoretical_loss": 5.400881495673488, + "tokens_seen": 44564480 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982246740220663, + "loss": 3.0965, + "theoretical_loss": 5.3996623700429245, + "tokens_seen": 44630016 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982146439317953, + "loss": 3.1374, + "theoretical_loss": 5.398445533716492, + "tokens_seen": 44695552 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982046138415246, + "loss": 3.0028, + "theoretical_loss": 5.3972309790479684, + "tokens_seen": 44761088 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981945837512538, + "loss": 3.1009, + "theoretical_loss": 5.39601869842782, + "tokens_seen": 44826624 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498184553660983, + "loss": 3.1442, + "theoretical_loss": 5.394808684282968, + "tokens_seen": 44892160 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981745235707122, + "loss": 3.0966, + "theoretical_loss": 5.393600929076564, + "tokens_seen": 44957696 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981644934804413, + "loss": 3.1862, + "theoretical_loss": 5.392395425307758, + "tokens_seen": 45023232 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981544633901705, + "loss": 3.0641, + "theoretical_loss": 5.391192165511484, + "tokens_seen": 45088768 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981444332998997, + "loss": 3.2417, + "theoretical_loss": 5.389991142258227, + "tokens_seen": 45154304 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981344032096289, + "loss": 3.1994, + "theoretical_loss": 5.38879234815381, + "tokens_seen": 45219840 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981243731193581, + "loss": 3.0273, + "theoretical_loss": 5.387595775839175, + "tokens_seen": 45285376 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981143430290873, + "loss": 3.0648, + "theoretical_loss": 5.386401417990159, + "tokens_seen": 45350912 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981043129388164, + "loss": 3.0055, + "theoretical_loss": 5.385209267317286, + "tokens_seen": 45416448 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004980942828485456, + "loss": 2.9476, + "theoretical_loss": 5.3840193165655466, + "tokens_seen": 45481984 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004980842527582748, + "loss": 2.9455, + "theoretical_loss": 5.382831558514191, + "tokens_seen": 45547520 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498074222668004, + "loss": 3.0993, + "theoretical_loss": 5.3816459859765144, + "tokens_seen": 45613056 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004980641925777333, + "loss": 2.9975, + "theoretical_loss": 5.380462591799651, + "tokens_seen": 45678592 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004980541624874623, + "loss": 3.0398, + "theoretical_loss": 5.379281368864362, + "tokens_seen": 45744128 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004980441323971916, + "loss": 3.1611, + "theoretical_loss": 5.378102310084834, + "tokens_seen": 45809664 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 21809, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 3.2171735763549805, + "objective/train/theoretical_loss": 5.376925408408475, + "objective/train/tokens_used": 45875200, + "theoretical_loss": 5.376925408408475, + "tokens_seen": 45875200 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004980341023069207, + "loss": 3.1751, + "theoretical_loss": 5.376925408408475, + "tokens_seen": 45875200 + }, + { + "epoch": 0.01, + "learning_rate": 0.00049802407221665, + "loss": 2.9994, + "theoretical_loss": 5.375750656815705, + "tokens_seen": 45940736 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004980140421263792, + "loss": 3.1872, + "theoretical_loss": 5.37457804831976, + "tokens_seen": 46006272 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004980040120361084, + "loss": 3.126, + "theoretical_loss": 5.373407575966493, + "tokens_seen": 46071808 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979939819458375, + "loss": 2.9081, + "theoretical_loss": 5.372239232834173, + "tokens_seen": 46137344 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979839518555667, + "loss": 3.0754, + "theoretical_loss": 5.3710730120332855, + "tokens_seen": 46202880 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979739217652959, + "loss": 3.0943, + "theoretical_loss": 5.369908906706346, + "tokens_seen": 46268416 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979638916750251, + "loss": 3.1266, + "theoretical_loss": 5.368746910027696, + "tokens_seen": 46333952 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979538615847543, + "loss": 3.0593, + "theoretical_loss": 5.3675870152033145, + "tokens_seen": 46399488 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979438314944834, + "loss": 3.1459, + "theoretical_loss": 5.36642921547063, + "tokens_seen": 46465024 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979338014042126, + "loss": 3.0515, + "theoretical_loss": 5.365273504098326, + "tokens_seen": 46530560 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979237713139418, + "loss": 3.1229, + "theoretical_loss": 5.364119874386157, + "tokens_seen": 46596096 + }, + { + "epoch": 0.01, + "learning_rate": 0.000497913741223671, + "loss": 2.9873, + "theoretical_loss": 5.362968319664756, + "tokens_seen": 46661632 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979037111334002, + "loss": 3.0413, + "theoretical_loss": 5.361818833295455, + "tokens_seen": 46727168 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978936810431293, + "loss": 2.9379, + "theoretical_loss": 5.3606714086701, + "tokens_seen": 46792704 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978836509528586, + "loss": 3.0642, + "theoretical_loss": 5.359526039210861, + "tokens_seen": 46858240 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978736208625877, + "loss": 2.9736, + "theoretical_loss": 5.358382718370063, + "tokens_seen": 46923776 + }, + { + "epoch": 0.01, + "learning_rate": 0.000497863590772317, + "loss": 2.9311, + "theoretical_loss": 5.357241439629992, + "tokens_seen": 46989312 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978535606820461, + "loss": 3.1402, + "theoretical_loss": 5.3561021965027305, + "tokens_seen": 47054848 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978435305917754, + "loss": 3.255, + "theoretical_loss": 5.354964982529967, + "tokens_seen": 47120384 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978335005015045, + "loss": 2.9952, + "theoretical_loss": 5.353829791282831, + "tokens_seen": 47185920 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978234704112337, + "loss": 2.9198, + "theoretical_loss": 5.35269661636171, + "tokens_seen": 47251456 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978134403209629, + "loss": 2.9246, + "theoretical_loss": 5.35156545139608, + "tokens_seen": 47316992 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978034102306921, + "loss": 2.9681, + "theoretical_loss": 5.350436290044332, + "tokens_seen": 47382528 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977933801404213, + "loss": 2.9943, + "theoretical_loss": 5.3493091259936, + "tokens_seen": 47448064 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977833500501504, + "loss": 3.0843, + "theoretical_loss": 5.348183952959596, + "tokens_seen": 47513600 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977733199598796, + "loss": 3.0508, + "theoretical_loss": 5.347060764686436, + "tokens_seen": 47579136 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977632898696088, + "loss": 3.0892, + "theoretical_loss": 5.345939554946472, + "tokens_seen": 47644672 + }, + { + "epoch": 0.01, + "learning_rate": 0.000497753259779338, + "loss": 2.9154, + "theoretical_loss": 5.344820317540132, + "tokens_seen": 47710208 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977432296890672, + "loss": 2.8809, + "theoretical_loss": 5.343703046295751, + "tokens_seen": 47775744 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977331995987965, + "loss": 3.0878, + "theoretical_loss": 5.342587735069408, + "tokens_seen": 47841280 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977231695085255, + "loss": 3.0759, + "theoretical_loss": 5.3414743777447615, + "tokens_seen": 47906816 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977131394182548, + "loss": 2.9697, + "theoretical_loss": 5.340362968232892, + "tokens_seen": 47972352 + }, + { + "epoch": 0.01, + "learning_rate": 0.000497703109327984, + "loss": 2.9672, + "theoretical_loss": 5.339253500472138, + "tokens_seen": 48037888 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976930792377132, + "loss": 3.0365, + "theoretical_loss": 5.338145968427941, + "tokens_seen": 48103424 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976830491474424, + "loss": 3.023, + "theoretical_loss": 5.337040366092687, + "tokens_seen": 48168960 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976730190571715, + "loss": 2.9992, + "theoretical_loss": 5.335936687485542, + "tokens_seen": 48234496 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976629889669007, + "loss": 2.9193, + "theoretical_loss": 5.334834926652308, + "tokens_seen": 48300032 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976529588766299, + "loss": 3.07, + "theoretical_loss": 5.33373507766526, + "tokens_seen": 48365568 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976429287863591, + "loss": 2.9053, + "theoretical_loss": 5.332637134622999, + "tokens_seen": 48431104 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976328986960883, + "loss": 3.0217, + "theoretical_loss": 5.331541091650294, + "tokens_seen": 48496640 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976228686058175, + "loss": 3.0504, + "theoretical_loss": 5.330446942897931, + "tokens_seen": 48562176 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976128385155466, + "loss": 2.8328, + "theoretical_loss": 5.32935468254257, + "tokens_seen": 48627712 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976028084252758, + "loss": 2.9865, + "theoretical_loss": 5.328264304786586, + "tokens_seen": 48693248 + }, + { + "epoch": 0.01, + "learning_rate": 0.000497592778335005, + "loss": 2.9885, + "theoretical_loss": 5.327175803857929, + "tokens_seen": 48758784 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975827482447342, + "loss": 2.9374, + "theoretical_loss": 5.326089174009972, + "tokens_seen": 48824320 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975727181544635, + "loss": 3.0534, + "theoretical_loss": 5.325004409521368, + "tokens_seen": 48889856 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975626880641925, + "loss": 2.9515, + "theoretical_loss": 5.323921504695903, + "tokens_seen": 48955392 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975526579739218, + "loss": 3.0032, + "theoretical_loss": 5.322840453862356, + "tokens_seen": 49020928 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975426278836509, + "loss": 2.9493, + "theoretical_loss": 5.3217612513743475, + "tokens_seen": 49086464 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 22989, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 3.039201259613037, + "objective/train/theoretical_loss": 5.320683891610208, + "objective/train/tokens_used": 49152000, + "theoretical_loss": 5.320683891610208, + "tokens_seen": 49152000 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975325977933802, + "loss": 3.077, + "theoretical_loss": 5.320683891610208, + "tokens_seen": 49152000 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975225677031094, + "loss": 2.9953, + "theoretical_loss": 5.3196083689728315, + "tokens_seen": 49217536 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975125376128386, + "loss": 2.9259, + "theoretical_loss": 5.318534677889536, + "tokens_seen": 49283072 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975025075225677, + "loss": 2.9706, + "theoretical_loss": 5.317462812811925, + "tokens_seen": 49348608 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004974924774322969, + "loss": 3.0255, + "theoretical_loss": 5.316392768215751, + "tokens_seen": 49414144 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004974824473420261, + "loss": 2.8164, + "theoretical_loss": 5.315324538600778, + "tokens_seen": 49479680 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004974724172517553, + "loss": 2.8274, + "theoretical_loss": 5.31425811849064, + "tokens_seen": 49545216 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004974623871614845, + "loss": 2.8914, + "theoretical_loss": 5.313193502432718, + "tokens_seen": 49610752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004974523570712136, + "loss": 3.1267, + "theoretical_loss": 5.312130684997992, + "tokens_seen": 49676288 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004974423269809428, + "loss": 3.0967, + "theoretical_loss": 5.31106966078092, + "tokens_seen": 49741824 + }, + { + "epoch": 0.02, + "learning_rate": 0.000497432296890672, + "loss": 2.854, + "theoretical_loss": 5.310010424399295, + "tokens_seen": 49807360 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004974222668004012, + "loss": 2.9149, + "theoretical_loss": 5.308952970494124, + "tokens_seen": 49872896 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004974122367101304, + "loss": 2.9691, + "theoretical_loss": 5.307897293729489, + "tokens_seen": 49938432 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004974022066198595, + "loss": 2.9186, + "theoretical_loss": 5.306843388792423, + "tokens_seen": 50003968 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973921765295888, + "loss": 2.9594, + "theoretical_loss": 5.305791250392778, + "tokens_seen": 50069504 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973821464393179, + "loss": 2.9624, + "theoretical_loss": 5.304740873263098, + "tokens_seen": 50135040 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973721163490472, + "loss": 2.9037, + "theoretical_loss": 5.303692252158496, + "tokens_seen": 50200576 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973620862587763, + "loss": 2.8752, + "theoretical_loss": 5.302645381856522, + "tokens_seen": 50266112 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973520561685056, + "loss": 3.0065, + "theoretical_loss": 5.30160025715704, + "tokens_seen": 50331648 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973420260782347, + "loss": 2.9275, + "theoretical_loss": 5.300556872882108, + "tokens_seen": 50397184 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973319959879639, + "loss": 2.8135, + "theoretical_loss": 5.299515223875848, + "tokens_seen": 50462720 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973219658976931, + "loss": 2.8587, + "theoretical_loss": 5.2984753050043265, + "tokens_seen": 50528256 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973119358074223, + "loss": 2.9236, + "theoretical_loss": 5.2974371111554355, + "tokens_seen": 50593792 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973019057171515, + "loss": 2.9835, + "theoretical_loss": 5.296400637238764, + "tokens_seen": 50659328 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972918756268806, + "loss": 2.9701, + "theoretical_loss": 5.295365878185488, + "tokens_seen": 50724864 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972818455366098, + "loss": 2.7918, + "theoretical_loss": 5.294332828948243, + "tokens_seen": 50790400 + }, + { + "epoch": 0.02, + "learning_rate": 0.000497271815446339, + "loss": 2.9327, + "theoretical_loss": 5.2933014845010105, + "tokens_seen": 50855936 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972617853560682, + "loss": 2.9467, + "theoretical_loss": 5.292271839838996, + "tokens_seen": 50921472 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972517552657974, + "loss": 3.0105, + "theoretical_loss": 5.291243889978516, + "tokens_seen": 50987008 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972417251755266, + "loss": 2.9723, + "theoretical_loss": 5.290217629956882, + "tokens_seen": 51052544 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972316950852557, + "loss": 2.959, + "theoretical_loss": 5.289193054832282, + "tokens_seen": 51118080 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972216649949849, + "loss": 2.8772, + "theoretical_loss": 5.288170159683668, + "tokens_seen": 51183616 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972116349047142, + "loss": 2.8817, + "theoretical_loss": 5.287148939610645, + "tokens_seen": 51249152 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972016048144433, + "loss": 2.9347, + "theoretical_loss": 5.28612938973335, + "tokens_seen": 51314688 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971915747241726, + "loss": 2.6982, + "theoretical_loss": 5.285111505192349, + "tokens_seen": 51380224 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971815446339017, + "loss": 2.804, + "theoretical_loss": 5.284095281148522, + "tokens_seen": 51445760 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971715145436309, + "loss": 2.8198, + "theoretical_loss": 5.283080712782951, + "tokens_seen": 51511296 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971614844533601, + "loss": 2.7931, + "theoretical_loss": 5.282067795296811, + "tokens_seen": 51576832 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971514543630893, + "loss": 3.0037, + "theoretical_loss": 5.28105652391126, + "tokens_seen": 51642368 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971414242728185, + "loss": 2.9444, + "theoretical_loss": 5.280046893867333, + "tokens_seen": 51707904 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971313941825477, + "loss": 3.0042, + "theoretical_loss": 5.279038900425832, + "tokens_seen": 51773440 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971213640922768, + "loss": 2.9707, + "theoretical_loss": 5.278032538867221, + "tokens_seen": 51838976 + }, + { + "epoch": 0.02, + "learning_rate": 0.000497111334002006, + "loss": 2.8256, + "theoretical_loss": 5.277027804491517, + "tokens_seen": 51904512 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971013039117352, + "loss": 2.8544, + "theoretical_loss": 5.2760246926181855, + "tokens_seen": 51970048 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970912738214644, + "loss": 2.9213, + "theoretical_loss": 5.275023198586037, + "tokens_seen": 52035584 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970812437311936, + "loss": 2.7525, + "theoretical_loss": 5.274023317753125, + "tokens_seen": 52101120 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970712136409227, + "loss": 2.8283, + "theoretical_loss": 5.273025045496631, + "tokens_seen": 52166656 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970611835506519, + "loss": 2.7698, + "theoretical_loss": 5.272028377212777, + "tokens_seen": 52232192 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970511534603811, + "loss": 2.8, + "theoretical_loss": 5.271033308316715, + "tokens_seen": 52297728 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970411233701103, + "loss": 2.7585, + "theoretical_loss": 5.270039834242425, + "tokens_seen": 52363264 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 24258, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 2.7275328636169434, + "objective/train/theoretical_loss": 5.269047950442619, + "objective/train/tokens_used": 52428800, + "theoretical_loss": 5.269047950442619, + "tokens_seen": 52428800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970310932798396, + "loss": 2.7971, + "theoretical_loss": 5.269047950442619, + "tokens_seen": 52428800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970210631895686, + "loss": 2.767, + "theoretical_loss": 5.2680576523886335, + "tokens_seen": 52494336 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970110330992979, + "loss": 2.775, + "theoretical_loss": 5.2670689355703395, + "tokens_seen": 52559872 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970010030090271, + "loss": 2.8153, + "theoretical_loss": 5.266081795496035, + "tokens_seen": 52625408 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969909729187563, + "loss": 2.8365, + "theoretical_loss": 5.265096227692354, + "tokens_seen": 52690944 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969809428284855, + "loss": 2.8178, + "theoretical_loss": 5.264112227704162, + "tokens_seen": 52756480 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969709127382147, + "loss": 2.7097, + "theoretical_loss": 5.263129791094466, + "tokens_seen": 52822016 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969608826479438, + "loss": 2.8371, + "theoretical_loss": 5.26214891344431, + "tokens_seen": 52887552 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496950852557673, + "loss": 2.8291, + "theoretical_loss": 5.26116959035269, + "tokens_seen": 52953088 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969408224674022, + "loss": 2.6521, + "theoretical_loss": 5.260191817436452, + "tokens_seen": 53018624 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969307923771314, + "loss": 2.937, + "theoretical_loss": 5.259215590330195, + "tokens_seen": 53084160 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969207622868606, + "loss": 2.746, + "theoretical_loss": 5.258240904686186, + "tokens_seen": 53149696 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969107321965897, + "loss": 2.9208, + "theoretical_loss": 5.25726775617426, + "tokens_seen": 53215232 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496900702106319, + "loss": 2.8539, + "theoretical_loss": 5.256296140481728, + "tokens_seen": 53280768 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968906720160481, + "loss": 2.9449, + "theoretical_loss": 5.255326053313292, + "tokens_seen": 53346304 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968806419257774, + "loss": 2.7077, + "theoretical_loss": 5.254357490390941, + "tokens_seen": 53411840 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968706118355065, + "loss": 2.6982, + "theoretical_loss": 5.253390447453873, + "tokens_seen": 53477376 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968605817452358, + "loss": 2.6889, + "theoretical_loss": 5.252424920258396, + "tokens_seen": 53542912 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968505516549649, + "loss": 2.854, + "theoretical_loss": 5.251460904577845, + "tokens_seen": 53608448 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968405215646941, + "loss": 2.6716, + "theoretical_loss": 5.250498396202486, + "tokens_seen": 53673984 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968304914744233, + "loss": 2.7023, + "theoretical_loss": 5.249537390939436, + "tokens_seen": 53739520 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968204613841525, + "loss": 2.9237, + "theoretical_loss": 5.248577884612564, + "tokens_seen": 53805056 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968104312938817, + "loss": 2.678, + "theoretical_loss": 5.247619873062416, + "tokens_seen": 53870592 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968004012036108, + "loss": 2.4866, + "theoretical_loss": 5.246663352146122, + "tokens_seen": 53936128 + }, + { + "epoch": 0.02, + "learning_rate": 0.00049679037111334, + "loss": 2.845, + "theoretical_loss": 5.245708317737305, + "tokens_seen": 54001664 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967803410230692, + "loss": 2.616, + "theoretical_loss": 5.244754765726007, + "tokens_seen": 54067200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967703109327984, + "loss": 2.7555, + "theoretical_loss": 5.243802692018596, + "tokens_seen": 54132736 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967602808425276, + "loss": 2.8425, + "theoretical_loss": 5.242852092537677, + "tokens_seen": 54198272 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967502507522568, + "loss": 2.7662, + "theoretical_loss": 5.241902963222023, + "tokens_seen": 54263808 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967402206619859, + "loss": 2.8764, + "theoretical_loss": 5.240955300026476, + "tokens_seen": 54329344 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967301905717151, + "loss": 2.7542, + "theoretical_loss": 5.240009098921874, + "tokens_seen": 54394880 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967201604814444, + "loss": 2.8301, + "theoretical_loss": 5.239064355894965, + "tokens_seen": 54460416 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967101303911735, + "loss": 2.7124, + "theoretical_loss": 5.238121066948324, + "tokens_seen": 54525952 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967001003009028, + "loss": 2.7829, + "theoretical_loss": 5.237179228100272, + "tokens_seen": 54591488 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496690070210632, + "loss": 2.7074, + "theoretical_loss": 5.2362388353848, + "tokens_seen": 54657024 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966800401203611, + "loss": 2.5904, + "theoretical_loss": 5.23529988485148, + "tokens_seen": 54722560 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966700100300903, + "loss": 2.7559, + "theoretical_loss": 5.2343623725653945, + "tokens_seen": 54788096 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966599799398195, + "loss": 2.8941, + "theoretical_loss": 5.233426294607049, + "tokens_seen": 54853632 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966499498495487, + "loss": 2.7096, + "theoretical_loss": 5.232491647072299, + "tokens_seen": 54919168 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966399197592779, + "loss": 2.5766, + "theoretical_loss": 5.231558426072269, + "tokens_seen": 54984704 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496629889669007, + "loss": 2.5925, + "theoretical_loss": 5.230626627733274, + "tokens_seen": 55050240 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966198595787362, + "loss": 2.7755, + "theoretical_loss": 5.2296962481967455, + "tokens_seen": 55115776 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966098294884654, + "loss": 2.7142, + "theoretical_loss": 5.228767283619151, + "tokens_seen": 55181312 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965997993981946, + "loss": 2.807, + "theoretical_loss": 5.227839730171918, + "tokens_seen": 55246848 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965897693079238, + "loss": 2.8469, + "theoretical_loss": 5.2269135840413625, + "tokens_seen": 55312384 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965797392176529, + "loss": 2.657, + "theoretical_loss": 5.225988841428606, + "tokens_seen": 55377920 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965697091273821, + "loss": 2.7139, + "theoretical_loss": 5.225065498549508, + "tokens_seen": 55443456 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965596790371113, + "loss": 2.7868, + "theoretical_loss": 5.224143551634588, + "tokens_seen": 55508992 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965496489468405, + "loss": 2.7157, + "theoretical_loss": 5.22322299692895, + "tokens_seen": 55574528 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965396188565698, + "loss": 2.7684, + "theoretical_loss": 5.222303830692211, + "tokens_seen": 55640064 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 25188, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 2.550832748413086, + "objective/train/theoretical_loss": 5.22138604919843, + "objective/train/tokens_used": 55705600, + "theoretical_loss": 5.22138604919843, + "tokens_seen": 55705600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965295887662988, + "loss": 2.696, + "theoretical_loss": 5.22138604919843, + "tokens_seen": 55705600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965195586760281, + "loss": 2.7652, + "theoretical_loss": 5.220469648736031, + "tokens_seen": 55771136 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965095285857573, + "loss": 2.9163, + "theoretical_loss": 5.219554625607731, + "tokens_seen": 55836672 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964994984954865, + "loss": 2.821, + "theoretical_loss": 5.218640976130474, + "tokens_seen": 55902208 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964894684052157, + "loss": 2.7402, + "theoretical_loss": 5.217728696635353, + "tokens_seen": 55967744 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964794383149449, + "loss": 2.6459, + "theoretical_loss": 5.216817783467543, + "tokens_seen": 56033280 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496469408224674, + "loss": 2.8592, + "theoretical_loss": 5.215908232986227, + "tokens_seen": 56098816 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964593781344032, + "loss": 2.5293, + "theoretical_loss": 5.215000041564533, + "tokens_seen": 56164352 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964493480441324, + "loss": 2.7222, + "theoretical_loss": 5.214093205589455, + "tokens_seen": 56229888 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964393179538616, + "loss": 2.8287, + "theoretical_loss": 5.213187721461791, + "tokens_seen": 56295424 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964292878635908, + "loss": 2.6058, + "theoretical_loss": 5.212283585596071, + "tokens_seen": 56360960 + }, + { + "epoch": 0.02, + "learning_rate": 0.00049641925777332, + "loss": 2.6385, + "theoretical_loss": 5.211380794420492, + "tokens_seen": 56426496 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964092276830491, + "loss": 2.592, + "theoretical_loss": 5.210479344376845, + "tokens_seen": 56492032 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963991975927783, + "loss": 2.7272, + "theoretical_loss": 5.209579231920451, + "tokens_seen": 56557568 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963891675025075, + "loss": 2.6793, + "theoretical_loss": 5.208680453520092, + "tokens_seen": 56623104 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963791374122367, + "loss": 2.6107, + "theoretical_loss": 5.2077830056579515, + "tokens_seen": 56688640 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963691073219659, + "loss": 2.6113, + "theoretical_loss": 5.206886884829535, + "tokens_seen": 56754176 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963590772316951, + "loss": 2.6811, + "theoretical_loss": 5.205992087543614, + "tokens_seen": 56819712 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963490471414242, + "loss": 2.7521, + "theoretical_loss": 5.205098610322162, + "tokens_seen": 56885248 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963390170511535, + "loss": 2.591, + "theoretical_loss": 5.204206449700279, + "tokens_seen": 56950784 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963289869608827, + "loss": 2.7288, + "theoretical_loss": 5.203315602226139, + "tokens_seen": 57016320 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963189568706119, + "loss": 2.7461, + "theoretical_loss": 5.2024260644609175, + "tokens_seen": 57081856 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496308926780341, + "loss": 2.4898, + "theoretical_loss": 5.201537832978731, + "tokens_seen": 57147392 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962988966900702, + "loss": 2.6704, + "theoretical_loss": 5.200650904366574, + "tokens_seen": 57212928 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962888665997994, + "loss": 2.6, + "theoretical_loss": 5.199765275224252, + "tokens_seen": 57278464 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962788365095286, + "loss": 2.5684, + "theoretical_loss": 5.198880942164323, + "tokens_seen": 57344000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962688064192578, + "loss": 2.5794, + "theoretical_loss": 5.197997901812036, + "tokens_seen": 57409536 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496258776328987, + "loss": 2.6323, + "theoretical_loss": 5.197116150805264, + "tokens_seen": 57475072 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962487462387161, + "loss": 2.6188, + "theoretical_loss": 5.196235685794446, + "tokens_seen": 57540608 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962387161484453, + "loss": 2.6798, + "theoretical_loss": 5.195356503442525, + "tokens_seen": 57606144 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962286860581746, + "loss": 2.6429, + "theoretical_loss": 5.194478600424889, + "tokens_seen": 57671680 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962186559679037, + "loss": 2.5711, + "theoretical_loss": 5.193601973429306, + "tokens_seen": 57737216 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496208625877633, + "loss": 2.5684, + "theoretical_loss": 5.192726619155867, + "tokens_seen": 57802752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961985957873621, + "loss": 2.7022, + "theoretical_loss": 5.191852534316928, + "tokens_seen": 57868288 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961885656970913, + "loss": 2.6462, + "theoretical_loss": 5.190979715637049, + "tokens_seen": 57933824 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961785356068205, + "loss": 2.5803, + "theoretical_loss": 5.190108159852929, + "tokens_seen": 57999360 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961685055165497, + "loss": 2.6922, + "theoretical_loss": 5.189237863713357, + "tokens_seen": 58064896 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961584754262789, + "loss": 2.6386, + "theoretical_loss": 5.188368823979152, + "tokens_seen": 58130432 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961484453360081, + "loss": 2.4882, + "theoretical_loss": 5.187501037423096, + "tokens_seen": 58195968 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961384152457372, + "loss": 2.5747, + "theoretical_loss": 5.186634500829886, + "tokens_seen": 58261504 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961283851554664, + "loss": 2.6547, + "theoretical_loss": 5.185769210996076, + "tokens_seen": 58327040 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961183550651956, + "loss": 2.6045, + "theoretical_loss": 5.184905164730012, + "tokens_seen": 58392576 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961083249749248, + "loss": 2.6032, + "theoretical_loss": 5.184042358851785, + "tokens_seen": 58458112 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496098294884654, + "loss": 2.5651, + "theoretical_loss": 5.1831807901931715, + "tokens_seen": 58523648 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960882647943831, + "loss": 2.5524, + "theoretical_loss": 5.18232045559757, + "tokens_seen": 58589184 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960782347041123, + "loss": 2.5858, + "theoretical_loss": 5.181461351919959, + "tokens_seen": 58654720 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960682046138415, + "loss": 2.4733, + "theoretical_loss": 5.180603476026832, + "tokens_seen": 58720256 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960581745235707, + "loss": 2.6014, + "theoretical_loss": 5.179746824796146, + "tokens_seen": 58785792 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960481444333, + "loss": 2.5305, + "theoretical_loss": 5.178891395117262, + "tokens_seen": 58851328 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496038114343029, + "loss": 2.4625, + "theoretical_loss": 5.178037183890902, + "tokens_seen": 58916864 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 26422, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 2.632338523864746, + "objective/train/theoretical_loss": 5.177184188029082, + "objective/train/tokens_used": 58982400, + "theoretical_loss": 5.177184188029082, + "tokens_seen": 58982400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960280842527583, + "loss": 2.5362, + "theoretical_loss": 5.177184188029082, + "tokens_seen": 58982400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960180541624875, + "loss": 2.531, + "theoretical_loss": 5.1763324044550645, + "tokens_seen": 59047936 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960080240722167, + "loss": 2.5727, + "theoretical_loss": 5.175481830103307, + "tokens_seen": 59113472 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959979939819459, + "loss": 2.5895, + "theoretical_loss": 5.174632461919406, + "tokens_seen": 59179008 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959879638916751, + "loss": 2.5081, + "theoretical_loss": 5.173784296860043, + "tokens_seen": 59244544 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959779338014042, + "loss": 2.5921, + "theoretical_loss": 5.172937331892934, + "tokens_seen": 59310080 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959679037111334, + "loss": 2.6694, + "theoretical_loss": 5.17209156399678, + "tokens_seen": 59375616 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959578736208626, + "loss": 2.556, + "theoretical_loss": 5.171246990161208, + "tokens_seen": 59441152 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959478435305918, + "loss": 2.4598, + "theoretical_loss": 5.17040360738673, + "tokens_seen": 59506688 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495937813440321, + "loss": 2.6095, + "theoretical_loss": 5.169561412684677, + "tokens_seen": 59572224 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959277833500501, + "loss": 2.5574, + "theoretical_loss": 5.168720403077165, + "tokens_seen": 59637760 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959177532597793, + "loss": 2.535, + "theoretical_loss": 5.167880575597031, + "tokens_seen": 59703296 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959077231695085, + "loss": 2.4034, + "theoretical_loss": 5.167041927287787, + "tokens_seen": 59768832 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958976930792377, + "loss": 2.6675, + "theoretical_loss": 5.166204455203575, + "tokens_seen": 59834368 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958876629889669, + "loss": 2.5857, + "theoretical_loss": 5.16536815640911, + "tokens_seen": 59899904 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495877632898696, + "loss": 2.598, + "theoretical_loss": 5.1645330279796315, + "tokens_seen": 59965440 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958676028084253, + "loss": 2.4901, + "theoretical_loss": 5.1636990670008585, + "tokens_seen": 60030976 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958575727181544, + "loss": 2.5614, + "theoretical_loss": 5.162866270568938, + "tokens_seen": 60096512 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958475426278837, + "loss": 2.539, + "theoretical_loss": 5.162034635790397, + "tokens_seen": 60162048 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958375125376129, + "loss": 2.5835, + "theoretical_loss": 5.1612041597820895, + "tokens_seen": 60227584 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958274824473421, + "loss": 2.6478, + "theoretical_loss": 5.160374839671159, + "tokens_seen": 60293120 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958174523570712, + "loss": 2.6727, + "theoretical_loss": 5.1595466725949795, + "tokens_seen": 60358656 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958074222668004, + "loss": 2.6281, + "theoretical_loss": 5.158719655701115, + "tokens_seen": 60424192 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957973921765296, + "loss": 2.5876, + "theoretical_loss": 5.157893786147269, + "tokens_seen": 60489728 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957873620862588, + "loss": 2.6353, + "theoretical_loss": 5.157069061101238, + "tokens_seen": 60555264 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495777331995988, + "loss": 2.6062, + "theoretical_loss": 5.156245477740866, + "tokens_seen": 60620800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957673019057172, + "loss": 2.5154, + "theoretical_loss": 5.155423033253997, + "tokens_seen": 60686336 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957572718154463, + "loss": 2.5679, + "theoretical_loss": 5.154601724838431, + "tokens_seen": 60751872 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957472417251755, + "loss": 2.5779, + "theoretical_loss": 5.153781549701872, + "tokens_seen": 60817408 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957372116349047, + "loss": 2.547, + "theoretical_loss": 5.15296250506189, + "tokens_seen": 60882944 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957271815446339, + "loss": 2.386, + "theoretical_loss": 5.152144588145871, + "tokens_seen": 60948480 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957171514543631, + "loss": 2.4629, + "theoretical_loss": 5.151327796190975, + "tokens_seen": 61014016 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957071213640923, + "loss": 2.3375, + "theoretical_loss": 5.1505121264440845, + "tokens_seen": 61079552 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956970912738214, + "loss": 2.3483, + "theoretical_loss": 5.149697576161769, + "tokens_seen": 61145088 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956870611835507, + "loss": 2.5531, + "theoretical_loss": 5.148884142610232, + "tokens_seen": 61210624 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956770310932798, + "loss": 2.4216, + "theoretical_loss": 5.148071823065274, + "tokens_seen": 61276160 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956670010030091, + "loss": 2.6158, + "theoretical_loss": 5.147260614812246, + "tokens_seen": 61341696 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956569709127383, + "loss": 2.3871, + "theoretical_loss": 5.146450515146002, + "tokens_seen": 61407232 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956469408224674, + "loss": 2.5341, + "theoretical_loss": 5.145641521370861, + "tokens_seen": 61472768 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956369107321966, + "loss": 2.5078, + "theoretical_loss": 5.144833630800562, + "tokens_seen": 61538304 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956268806419258, + "loss": 2.5313, + "theoretical_loss": 5.1440268407582215, + "tokens_seen": 61603840 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495616850551655, + "loss": 2.4293, + "theoretical_loss": 5.143221148576288, + "tokens_seen": 61669376 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956068204613842, + "loss": 2.4543, + "theoretical_loss": 5.142416551596505, + "tokens_seen": 61734912 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955967903711133, + "loss": 2.5561, + "theoretical_loss": 5.141613047169862, + "tokens_seen": 61800448 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955867602808425, + "loss": 2.5811, + "theoretical_loss": 5.14081063265656, + "tokens_seen": 61865984 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955767301905717, + "loss": 2.4285, + "theoretical_loss": 5.140009305425963, + "tokens_seen": 61931520 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955667001003009, + "loss": 2.5237, + "theoretical_loss": 5.13920906285656, + "tokens_seen": 61997056 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955566700100301, + "loss": 2.3714, + "theoretical_loss": 5.138409902335925, + "tokens_seen": 62062592 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955466399197592, + "loss": 2.5344, + "theoretical_loss": 5.137611821260673, + "tokens_seen": 62128128 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955366098294884, + "loss": 2.511, + "theoretical_loss": 5.13681481703642, + "tokens_seen": 62193664 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 27339, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 2.3425395488739014, + "objective/train/theoretical_loss": 5.136018887077743, + "objective/train/tokens_used": 62259200, + "theoretical_loss": 5.136018887077743, + "tokens_seen": 62259200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955265797392177, + "loss": 2.4486, + "theoretical_loss": 5.136018887077743, + "tokens_seen": 62259200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955165496489468, + "loss": 2.4376, + "theoretical_loss": 5.13522402880814, + "tokens_seen": 62324736 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955065195586761, + "loss": 2.4611, + "theoretical_loss": 5.13443023965999, + "tokens_seen": 62390272 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954964894684052, + "loss": 2.4631, + "theoretical_loss": 5.13363751707451, + "tokens_seen": 62455808 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954864593781344, + "loss": 2.4327, + "theoretical_loss": 5.132845858501721, + "tokens_seen": 62521344 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954764292878636, + "loss": 2.3909, + "theoretical_loss": 5.132055261400403, + "tokens_seen": 62586880 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954663991975928, + "loss": 2.3559, + "theoretical_loss": 5.131265723238059, + "tokens_seen": 62652416 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495456369107322, + "loss": 2.4335, + "theoretical_loss": 5.130477241490876, + "tokens_seen": 62717952 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954463390170512, + "loss": 2.3476, + "theoretical_loss": 5.129689813643683, + "tokens_seen": 62783488 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954363089267803, + "loss": 2.4264, + "theoretical_loss": 5.128903437189917, + "tokens_seen": 62849024 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954262788365095, + "loss": 2.4822, + "theoretical_loss": 5.128118109631581, + "tokens_seen": 62914560 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954162487462387, + "loss": 2.4765, + "theoretical_loss": 5.127333828479208, + "tokens_seen": 62980096 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954062186559679, + "loss": 2.3488, + "theoretical_loss": 5.126550591251821, + "tokens_seen": 63045632 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953961885656971, + "loss": 2.4784, + "theoretical_loss": 5.125768395476896, + "tokens_seen": 63111168 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953861584754263, + "loss": 2.4969, + "theoretical_loss": 5.124987238690329, + "tokens_seen": 63176704 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953761283851555, + "loss": 2.4529, + "theoretical_loss": 5.124207118436388, + "tokens_seen": 63242240 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953660982948846, + "loss": 2.4388, + "theoretical_loss": 5.123428032267688, + "tokens_seen": 63307776 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953560682046139, + "loss": 2.5485, + "theoretical_loss": 5.122649977745146, + "tokens_seen": 63373312 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953460381143431, + "loss": 2.4242, + "theoretical_loss": 5.121872952437947, + "tokens_seen": 63438848 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953360080240723, + "loss": 2.4384, + "theoretical_loss": 5.121096953923507, + "tokens_seen": 63504384 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953259779338014, + "loss": 2.5071, + "theoretical_loss": 5.1203219797874375, + "tokens_seen": 63569920 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953159478435306, + "loss": 2.3507, + "theoretical_loss": 5.119548027623509, + "tokens_seen": 63635456 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953059177532598, + "loss": 2.4938, + "theoretical_loss": 5.118775095033612, + "tokens_seen": 63700992 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495295887662989, + "loss": 2.4744, + "theoretical_loss": 5.118003179627729, + "tokens_seen": 63766528 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952858575727182, + "loss": 2.53, + "theoretical_loss": 5.11723227902389, + "tokens_seen": 63832064 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952758274824474, + "loss": 2.4601, + "theoretical_loss": 5.1164623908481435, + "tokens_seen": 63897600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952657973921765, + "loss": 2.3607, + "theoretical_loss": 5.115693512734515, + "tokens_seen": 63963136 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952557673019057, + "loss": 2.3691, + "theoretical_loss": 5.114925642324982, + "tokens_seen": 64028672 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952457372116349, + "loss": 2.3921, + "theoretical_loss": 5.114158777269429, + "tokens_seen": 64094208 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952357071213641, + "loss": 2.4451, + "theoretical_loss": 5.113392915225617, + "tokens_seen": 64159744 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952256770310933, + "loss": 2.3937, + "theoretical_loss": 5.112628053859153, + "tokens_seen": 64225280 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952156469408225, + "loss": 2.4867, + "theoretical_loss": 5.1118641908434475, + "tokens_seen": 64290816 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952056168505516, + "loss": 2.2713, + "theoretical_loss": 5.111101323859687, + "tokens_seen": 64356352 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951955867602809, + "loss": 2.4424, + "theoretical_loss": 5.1103394505968005, + "tokens_seen": 64421888 + }, + { + "epoch": 0.02, + "learning_rate": 0.00049518555667001, + "loss": 2.4482, + "theoretical_loss": 5.109578568751417, + "tokens_seen": 64487424 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951755265797393, + "loss": 2.4656, + "theoretical_loss": 5.108818676027845, + "tokens_seen": 64552960 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951654964894685, + "loss": 2.3863, + "theoretical_loss": 5.108059770138027, + "tokens_seen": 64618496 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951554663991976, + "loss": 2.3329, + "theoretical_loss": 5.107301848801519, + "tokens_seen": 64684032 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951454363089268, + "loss": 2.3253, + "theoretical_loss": 5.106544909745443, + "tokens_seen": 64749568 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495135406218656, + "loss": 2.3129, + "theoretical_loss": 5.105788950704467, + "tokens_seen": 64815104 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951253761283852, + "loss": 2.358, + "theoretical_loss": 5.105033969420764, + "tokens_seen": 64880640 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951153460381144, + "loss": 2.4364, + "theoretical_loss": 5.104279963643982, + "tokens_seen": 64946176 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951053159478435, + "loss": 2.3041, + "theoretical_loss": 5.103526931131217, + "tokens_seen": 65011712 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950952858575727, + "loss": 2.3957, + "theoretical_loss": 5.102774869646972, + "tokens_seen": 65077248 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950852557673019, + "loss": 2.4817, + "theoretical_loss": 5.102023776963131, + "tokens_seen": 65142784 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950752256770311, + "loss": 2.3818, + "theoretical_loss": 5.101273650858926, + "tokens_seen": 65208320 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950651955867603, + "loss": 2.3406, + "theoretical_loss": 5.100524489120902, + "tokens_seen": 65273856 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950551654964894, + "loss": 2.3908, + "theoretical_loss": 5.099776289542893, + "tokens_seen": 65339392 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950451354062186, + "loss": 2.2294, + "theoretical_loss": 5.099029049925985, + "tokens_seen": 65404928 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950351053159479, + "loss": 2.3517, + "theoretical_loss": 5.098282768078483, + "tokens_seen": 65470464 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 28596, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 2.4704413414001465, + "objective/train/theoretical_loss": 5.097537441815887, + "objective/train/tokens_used": 65536000, + "theoretical_loss": 5.097537441815887, + "tokens_seen": 65536000 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495025075225677, + "loss": 2.4691, + "theoretical_loss": 5.097537441815887, + "tokens_seen": 65536000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950150451354063, + "loss": 2.3064, + "theoretical_loss": 5.096793068960857, + "tokens_seen": 65601536 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950050150451354, + "loss": 2.4207, + "theoretical_loss": 5.096049647343182, + "tokens_seen": 65667072 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949949849548646, + "loss": 2.3543, + "theoretical_loss": 5.095307174799752, + "tokens_seen": 65732608 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949849548645938, + "loss": 2.4067, + "theoretical_loss": 5.094565649174524, + "tokens_seen": 65798144 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494974924774323, + "loss": 2.4599, + "theoretical_loss": 5.0938250683184965, + "tokens_seen": 65863680 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949648946840522, + "loss": 2.3649, + "theoretical_loss": 5.093085430089674, + "tokens_seen": 65929216 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949548645937814, + "loss": 2.3092, + "theoretical_loss": 5.092346732353046, + "tokens_seen": 65994752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949448345035105, + "loss": 2.4339, + "theoretical_loss": 5.091608972980545, + "tokens_seen": 66060288 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949348044132397, + "loss": 2.4251, + "theoretical_loss": 5.090872149851029, + "tokens_seen": 66125824 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949247743229689, + "loss": 2.3378, + "theoretical_loss": 5.090136260850245, + "tokens_seen": 66191360 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949147442326981, + "loss": 2.3252, + "theoretical_loss": 5.089401303870801, + "tokens_seen": 66256896 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949047141424273, + "loss": 2.4794, + "theoretical_loss": 5.088667276812138, + "tokens_seen": 66322432 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948946840521565, + "loss": 2.3645, + "theoretical_loss": 5.087934177580502, + "tokens_seen": 66387968 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948846539618856, + "loss": 2.395, + "theoretical_loss": 5.087202004088914, + "tokens_seen": 66453504 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948746238716148, + "loss": 2.1846, + "theoretical_loss": 5.086470754257139, + "tokens_seen": 66519040 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494864593781344, + "loss": 2.3721, + "theoretical_loss": 5.0857404260116645, + "tokens_seen": 66584576 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948545636910733, + "loss": 2.404, + "theoretical_loss": 5.085011017285662, + "tokens_seen": 66650112 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948445336008024, + "loss": 2.1748, + "theoretical_loss": 5.084282526018972, + "tokens_seen": 66715648 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948345035105316, + "loss": 2.3204, + "theoretical_loss": 5.083554950158062, + "tokens_seen": 66781184 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948244734202607, + "loss": 2.2603, + "theoretical_loss": 5.0828282876560085, + "tokens_seen": 66846720 + }, + { + "epoch": 0.02, + "learning_rate": 0.00049481444332999, + "loss": 2.2397, + "theoretical_loss": 5.0821025364724655, + "tokens_seen": 66912256 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948044132397192, + "loss": 2.4305, + "theoretical_loss": 5.081377694573638, + "tokens_seen": 66977792 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947943831494484, + "loss": 2.3758, + "theoretical_loss": 5.080653759932255, + "tokens_seen": 67043328 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947843530591776, + "loss": 2.3013, + "theoretical_loss": 5.079930730527538, + "tokens_seen": 67108864 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947743229689067, + "loss": 2.3517, + "theoretical_loss": 5.079208604345181, + "tokens_seen": 67174400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947642928786359, + "loss": 2.3321, + "theoretical_loss": 5.078487379377318, + "tokens_seen": 67239936 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947542627883651, + "loss": 2.3664, + "theoretical_loss": 5.0777670536225, + "tokens_seen": 67305472 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947442326980943, + "loss": 2.2932, + "theoretical_loss": 5.07704762508566, + "tokens_seen": 67371008 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947342026078235, + "loss": 2.2168, + "theoretical_loss": 5.076329091778101, + "tokens_seen": 67436544 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947241725175527, + "loss": 2.3191, + "theoretical_loss": 5.075611451717457, + "tokens_seen": 67502080 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947141424272818, + "loss": 2.3962, + "theoretical_loss": 5.0748947029276685, + "tokens_seen": 67567616 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947041123370111, + "loss": 2.4228, + "theoretical_loss": 5.074178843438963, + "tokens_seen": 67633152 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946940822467402, + "loss": 2.2538, + "theoretical_loss": 5.073463871287823, + "tokens_seen": 67698688 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946840521564695, + "loss": 2.2735, + "theoretical_loss": 5.07274978451696, + "tokens_seen": 67764224 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946740220661987, + "loss": 2.3794, + "theoretical_loss": 5.072036581175293, + "tokens_seen": 67829760 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946639919759278, + "loss": 2.3177, + "theoretical_loss": 5.071324259317921, + "tokens_seen": 67895296 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494653961885657, + "loss": 2.3378, + "theoretical_loss": 5.070612817006094, + "tokens_seen": 67960832 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946439317953862, + "loss": 2.4135, + "theoretical_loss": 5.06990225230719, + "tokens_seen": 68026368 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946339017051154, + "loss": 2.2338, + "theoretical_loss": 5.069192563294694, + "tokens_seen": 68091904 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946238716148446, + "loss": 2.3043, + "theoretical_loss": 5.068483748048166, + "tokens_seen": 68157440 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946138415245737, + "loss": 2.3105, + "theoretical_loss": 5.06777580465322, + "tokens_seen": 68222976 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946038114343029, + "loss": 2.356, + "theoretical_loss": 5.067068731201498, + "tokens_seen": 68288512 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945937813440321, + "loss": 2.1993, + "theoretical_loss": 5.066362525790646, + "tokens_seen": 68354048 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945837512537613, + "loss": 2.3042, + "theoretical_loss": 5.065657186524287, + "tokens_seen": 68419584 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945737211634905, + "loss": 2.2381, + "theoretical_loss": 5.064952711512001, + "tokens_seen": 68485120 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945636910732196, + "loss": 2.1977, + "theoretical_loss": 5.064249098869295, + "tokens_seen": 68550656 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945536609829488, + "loss": 2.229, + "theoretical_loss": 5.063546346717585, + "tokens_seen": 68616192 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945436308926781, + "loss": 2.3357, + "theoretical_loss": 5.062844453184166, + "tokens_seen": 68681728 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945336008024072, + "loss": 2.2092, + "theoretical_loss": 5.062143416402192, + "tokens_seen": 68747264 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 29617, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 2.3302438259124756, + "objective/train/theoretical_loss": 5.061443234510648, + "objective/train/tokens_used": 68812800, + "theoretical_loss": 5.061443234510648, + "tokens_seen": 68812800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945235707121365, + "loss": 2.3245, + "theoretical_loss": 5.061443234510648, + "tokens_seen": 68812800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945135406218656, + "loss": 2.3461, + "theoretical_loss": 5.060743905654332, + "tokens_seen": 68878336 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945035105315948, + "loss": 2.2669, + "theoretical_loss": 5.0600454279838285, + "tokens_seen": 68943872 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494493480441324, + "loss": 2.4241, + "theoretical_loss": 5.059347799655483, + "tokens_seen": 69009408 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944834503510532, + "loss": 2.3086, + "theoretical_loss": 5.05865101883138, + "tokens_seen": 69074944 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944734202607824, + "loss": 2.1789, + "theoretical_loss": 5.057955083679321, + "tokens_seen": 69140480 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944633901705116, + "loss": 2.3565, + "theoretical_loss": 5.0572599923728, + "tokens_seen": 69206016 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944533600802407, + "loss": 2.1727, + "theoretical_loss": 5.056565743090982, + "tokens_seen": 69271552 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944433299899699, + "loss": 2.3296, + "theoretical_loss": 5.055872334018677, + "tokens_seen": 69337088 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944332998996991, + "loss": 2.1646, + "theoretical_loss": 5.0551797633463185, + "tokens_seen": 69402624 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944232698094283, + "loss": 2.2147, + "theoretical_loss": 5.054488029269946, + "tokens_seen": 69468160 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944132397191575, + "loss": 2.3694, + "theoretical_loss": 5.053797129991171, + "tokens_seen": 69533696 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944032096288867, + "loss": 2.298, + "theoretical_loss": 5.053107063717167, + "tokens_seen": 69599232 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943931795386158, + "loss": 2.3268, + "theoretical_loss": 5.052417828660639, + "tokens_seen": 69664768 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494383149448345, + "loss": 2.2624, + "theoretical_loss": 5.051729423039802, + "tokens_seen": 69730304 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943731193580742, + "loss": 2.3726, + "theoretical_loss": 5.0510418450783625, + "tokens_seen": 69795840 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943630892678035, + "loss": 2.222, + "theoretical_loss": 5.050355093005495, + "tokens_seen": 69861376 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943530591775326, + "loss": 2.2936, + "theoretical_loss": 5.049669165055816, + "tokens_seen": 69926912 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943430290872618, + "loss": 2.4302, + "theoretical_loss": 5.048984059469369, + "tokens_seen": 69992448 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943329989969909, + "loss": 2.2562, + "theoretical_loss": 5.048299774491598, + "tokens_seen": 70057984 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943229689067202, + "loss": 2.4652, + "theoretical_loss": 5.047616308373328, + "tokens_seen": 70123520 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943129388164494, + "loss": 2.3046, + "theoretical_loss": 5.046933659370742, + "tokens_seen": 70189056 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943029087261786, + "loss": 2.386, + "theoretical_loss": 5.046251825745361, + "tokens_seen": 70254592 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942928786359078, + "loss": 2.3133, + "theoretical_loss": 5.0455708057640205, + "tokens_seen": 70320128 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942828485456369, + "loss": 2.3433, + "theoretical_loss": 5.044890597698853, + "tokens_seen": 70385664 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942728184553661, + "loss": 2.2241, + "theoretical_loss": 5.044211199827263, + "tokens_seen": 70451200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942627883650953, + "loss": 2.3377, + "theoretical_loss": 5.04353261043191, + "tokens_seen": 70516736 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942527582748245, + "loss": 2.239, + "theoretical_loss": 5.0428548278006815, + "tokens_seen": 70582272 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942427281845537, + "loss": 2.3106, + "theoretical_loss": 5.042177850226682, + "tokens_seen": 70647808 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942326980942828, + "loss": 2.2782, + "theoretical_loss": 5.0415016760082, + "tokens_seen": 70713344 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494222668004012, + "loss": 2.3348, + "theoretical_loss": 5.040826303448698, + "tokens_seen": 70778880 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942126379137412, + "loss": 2.2901, + "theoretical_loss": 5.040151730856788, + "tokens_seen": 70844416 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942026078234704, + "loss": 2.2724, + "theoretical_loss": 5.039477956546207, + "tokens_seen": 70909952 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941925777331996, + "loss": 2.2421, + "theoretical_loss": 5.038804978835808, + "tokens_seen": 70975488 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941825476429289, + "loss": 2.2497, + "theoretical_loss": 5.038132796049523, + "tokens_seen": 71041024 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941725175526579, + "loss": 2.3091, + "theoretical_loss": 5.037461406516362, + "tokens_seen": 71106560 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941624874623872, + "loss": 2.2528, + "theoretical_loss": 5.036790808570377, + "tokens_seen": 71172096 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941524573721163, + "loss": 2.2578, + "theoretical_loss": 5.036121000550651, + "tokens_seen": 71237632 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941424272818456, + "loss": 2.2925, + "theoretical_loss": 5.035451980801276, + "tokens_seen": 71303168 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941323971915748, + "loss": 2.2634, + "theoretical_loss": 5.034783747671332, + "tokens_seen": 71368704 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941223671013039, + "loss": 2.3204, + "theoretical_loss": 5.034116299514871, + "tokens_seen": 71434240 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941123370110331, + "loss": 2.2732, + "theoretical_loss": 5.033449634690893, + "tokens_seen": 71499776 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941023069207623, + "loss": 2.253, + "theoretical_loss": 5.032783751563327, + "tokens_seen": 71565312 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940922768304915, + "loss": 2.3244, + "theoretical_loss": 5.032118648501017, + "tokens_seen": 71630848 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940822467402207, + "loss": 2.2819, + "theoretical_loss": 5.031454323877698, + "tokens_seen": 71696384 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940722166499498, + "loss": 2.1835, + "theoretical_loss": 5.030790776071976, + "tokens_seen": 71761920 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494062186559679, + "loss": 2.2153, + "theoretical_loss": 5.030128003467312, + "tokens_seen": 71827456 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940521564694082, + "loss": 2.2657, + "theoretical_loss": 5.029466004452004, + "tokens_seen": 71892992 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940421263791374, + "loss": 2.2691, + "theoretical_loss": 5.028804777419165, + "tokens_seen": 71958528 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940320962888666, + "loss": 2.251, + "theoretical_loss": 5.028144320766701, + "tokens_seen": 72024064 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 30805, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 2.252187728881836, + "objective/train/theoretical_loss": 5.0274846328973055, + "objective/train/tokens_used": 72089600, + "theoretical_loss": 5.0274846328973055, + "tokens_seen": 72089600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940220661985958, + "loss": 2.1452, + "theoretical_loss": 5.0274846328973055, + "tokens_seen": 72089600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940120361083249, + "loss": 2.2103, + "theoretical_loss": 5.0268257122184234, + "tokens_seen": 72155136 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940020060180542, + "loss": 2.245, + "theoretical_loss": 5.026167557142247, + "tokens_seen": 72220672 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004939919759277834, + "loss": 2.1404, + "theoretical_loss": 5.02551016608569, + "tokens_seen": 72286208 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004939819458375126, + "loss": 2.151, + "theoretical_loss": 5.024853537470369, + "tokens_seen": 72351744 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004939719157472418, + "loss": 2.3185, + "theoretical_loss": 5.02419766972259, + "tokens_seen": 72417280 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493961885656971, + "loss": 2.1563, + "theoretical_loss": 5.023542561273327, + "tokens_seen": 72482816 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004939518555667001, + "loss": 2.2572, + "theoretical_loss": 5.022888210558206, + "tokens_seen": 72548352 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004939418254764293, + "loss": 2.1612, + "theoretical_loss": 5.02223461601748, + "tokens_seen": 72613888 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004939317953861585, + "loss": 2.2371, + "theoretical_loss": 5.021581776096022, + "tokens_seen": 72679424 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004939217652958877, + "loss": 2.3091, + "theoretical_loss": 5.0209296892433, + "tokens_seen": 72744960 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004939117352056169, + "loss": 2.2952, + "theoretical_loss": 5.020278353913362, + "tokens_seen": 72810496 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493901705115346, + "loss": 2.2414, + "theoretical_loss": 5.019627768564816, + "tokens_seen": 72876032 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938916750250752, + "loss": 2.1545, + "theoretical_loss": 5.018977931660814, + "tokens_seen": 72941568 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938816449348044, + "loss": 2.2811, + "theoretical_loss": 5.018328841669037, + "tokens_seen": 73007104 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938716148445337, + "loss": 2.3041, + "theoretical_loss": 5.017680497061671, + "tokens_seen": 73072640 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938615847542628, + "loss": 2.2379, + "theoretical_loss": 5.017032896315397, + "tokens_seen": 73138176 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493851554663992, + "loss": 2.1699, + "theoretical_loss": 5.016386037911371, + "tokens_seen": 73203712 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938415245737211, + "loss": 2.3359, + "theoretical_loss": 5.015739920335203, + "tokens_seen": 73269248 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938314944834504, + "loss": 2.2157, + "theoretical_loss": 5.015094542076945, + "tokens_seen": 73334784 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938214643931796, + "loss": 2.1677, + "theoretical_loss": 5.014449901631075, + "tokens_seen": 73400320 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938114343029088, + "loss": 2.1159, + "theoretical_loss": 5.013805997496473, + "tokens_seen": 73465856 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493801404212638, + "loss": 2.2672, + "theoretical_loss": 5.013162828176412, + "tokens_seen": 73531392 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937913741223671, + "loss": 2.2691, + "theoretical_loss": 5.012520392178539, + "tokens_seen": 73596928 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937813440320963, + "loss": 2.2653, + "theoretical_loss": 5.011878688014852, + "tokens_seen": 73662464 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937713139418255, + "loss": 2.1252, + "theoretical_loss": 5.011237714201694, + "tokens_seen": 73728000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937612838515547, + "loss": 2.2138, + "theoretical_loss": 5.010597469259729, + "tokens_seen": 73793536 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937512537612839, + "loss": 2.3196, + "theoretical_loss": 5.009957951713927, + "tokens_seen": 73859072 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493741223671013, + "loss": 2.2923, + "theoretical_loss": 5.009319160093551, + "tokens_seen": 73924608 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937311935807422, + "loss": 2.271, + "theoretical_loss": 5.008681092932137, + "tokens_seen": 73990144 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937211634904714, + "loss": 2.2828, + "theoretical_loss": 5.008043748767477, + "tokens_seen": 74055680 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937111334002006, + "loss": 2.2268, + "theoretical_loss": 5.007407126141608, + "tokens_seen": 74121216 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937011033099298, + "loss": 2.221, + "theoretical_loss": 5.00677122360079, + "tokens_seen": 74186752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936910732196591, + "loss": 2.1384, + "theoretical_loss": 5.006136039695496, + "tokens_seen": 74252288 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936810431293881, + "loss": 2.1497, + "theoretical_loss": 5.005501572980391, + "tokens_seen": 74317824 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936710130391174, + "loss": 2.1737, + "theoretical_loss": 5.004867822014317, + "tokens_seen": 74383360 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936609829488465, + "loss": 2.1983, + "theoretical_loss": 5.004234785360282, + "tokens_seen": 74448896 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936509528585758, + "loss": 2.2222, + "theoretical_loss": 5.0036024615854355, + "tokens_seen": 74514432 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493640922768305, + "loss": 2.1933, + "theoretical_loss": 5.002970849261065, + "tokens_seen": 74579968 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936308926780341, + "loss": 2.2592, + "theoretical_loss": 5.002339946962568, + "tokens_seen": 74645504 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936208625877633, + "loss": 2.0828, + "theoretical_loss": 5.001709753269443, + "tokens_seen": 74711040 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936108324974925, + "loss": 2.1325, + "theoretical_loss": 5.001080266765275, + "tokens_seen": 74776576 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936008024072217, + "loss": 2.2382, + "theoretical_loss": 5.0004514860377185, + "tokens_seen": 74842112 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935907723169509, + "loss": 2.0955, + "theoretical_loss": 4.999823409678481, + "tokens_seen": 74907648 + }, + { + "epoch": 0.02, + "learning_rate": 0.00049358074222668, + "loss": 2.2328, + "theoretical_loss": 4.999196036283308, + "tokens_seen": 74973184 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935707121364092, + "loss": 2.2441, + "theoretical_loss": 4.9985693644519715, + "tokens_seen": 75038720 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935606820461384, + "loss": 2.1417, + "theoretical_loss": 4.997943392788251, + "tokens_seen": 75104256 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935506519558676, + "loss": 2.1195, + "theoretical_loss": 4.997318119899922, + "tokens_seen": 75169792 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935406218655968, + "loss": 2.1926, + "theoretical_loss": 4.996693544398734, + "tokens_seen": 75235328 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493530591775326, + "loss": 2.2859, + "theoretical_loss": 4.996069664900406, + "tokens_seen": 75300864 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 32000, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 2.27396821975708, + "objective/train/theoretical_loss": 4.9954464800246035, + "objective/train/tokens_used": 75366400, + "theoretical_loss": 4.9954464800246035, + "tokens_seen": 75366400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935205616850551, + "loss": 2.2866, + "theoretical_loss": 4.9954464800246035, + "tokens_seen": 75366400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935105315947844, + "loss": 2.2071, + "theoretical_loss": 4.994823988394928, + "tokens_seen": 75431936 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935005015045135, + "loss": 2.2749, + "theoretical_loss": 4.994202188638901, + "tokens_seen": 75497472 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934904714142428, + "loss": 2.2353, + "theoretical_loss": 4.9935810793879485, + "tokens_seen": 75563008 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934804413239719, + "loss": 2.1394, + "theoretical_loss": 4.992960659277389, + "tokens_seen": 75628544 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934704112337011, + "loss": 2.3548, + "theoretical_loss": 4.992340926946417, + "tokens_seen": 75694080 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934603811434303, + "loss": 2.1784, + "theoretical_loss": 4.991721881038091, + "tokens_seen": 75759616 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934503510531595, + "loss": 2.2289, + "theoretical_loss": 4.991103520199314, + "tokens_seen": 75825152 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934403209628887, + "loss": 2.2852, + "theoretical_loss": 4.990485843080824, + "tokens_seen": 75890688 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934302908726179, + "loss": 2.1309, + "theoretical_loss": 4.989868848337181, + "tokens_seen": 75956224 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493420260782347, + "loss": 2.1591, + "theoretical_loss": 4.989252534626749, + "tokens_seen": 76021760 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934102306920762, + "loss": 2.1761, + "theoretical_loss": 4.988636900611683, + "tokens_seen": 76087296 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934002006018054, + "loss": 2.1528, + "theoretical_loss": 4.988021944957915, + "tokens_seen": 76152832 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933901705115346, + "loss": 2.2183, + "theoretical_loss": 4.987407666335143, + "tokens_seen": 76218368 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933801404212638, + "loss": 2.1554, + "theoretical_loss": 4.986794063416813, + "tokens_seen": 76283904 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493370110330993, + "loss": 2.108, + "theoretical_loss": 4.986181134880107, + "tokens_seen": 76349440 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933600802407221, + "loss": 2.2135, + "theoretical_loss": 4.98556887940593, + "tokens_seen": 76414976 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933500501504513, + "loss": 2.1118, + "theoretical_loss": 4.984957295678894, + "tokens_seen": 76480512 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933400200601805, + "loss": 2.1831, + "theoretical_loss": 4.98434638238731, + "tokens_seen": 76546048 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933299899699098, + "loss": 2.2219, + "theoretical_loss": 4.983736138223165, + "tokens_seen": 76611584 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933199598796389, + "loss": 2.2164, + "theoretical_loss": 4.983126561882118, + "tokens_seen": 76677120 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933099297893682, + "loss": 2.1204, + "theoretical_loss": 4.982517652063482, + "tokens_seen": 76742656 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932998996990972, + "loss": 2.0917, + "theoretical_loss": 4.981909407470209, + "tokens_seen": 76808192 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932898696088265, + "loss": 2.2749, + "theoretical_loss": 4.981301826808882, + "tokens_seen": 76873728 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932798395185557, + "loss": 2.1821, + "theoretical_loss": 4.9806949087896975, + "tokens_seen": 76939264 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932698094282849, + "loss": 2.2013, + "theoretical_loss": 4.980088652126453, + "tokens_seen": 77004800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932597793380141, + "loss": 2.2261, + "theoretical_loss": 4.9794830555365355, + "tokens_seen": 77070336 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932497492477432, + "loss": 2.1446, + "theoretical_loss": 4.978878117740907, + "tokens_seen": 77135872 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932397191574724, + "loss": 2.2321, + "theoretical_loss": 4.978273837464091, + "tokens_seen": 77201408 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932296890672016, + "loss": 2.0061, + "theoretical_loss": 4.977670213434161, + "tokens_seen": 77266944 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932196589769308, + "loss": 2.1495, + "theoretical_loss": 4.977067244382729, + "tokens_seen": 77332480 + }, + { + "epoch": 0.02, + "learning_rate": 0.00049320962888666, + "loss": 2.2102, + "theoretical_loss": 4.976464929044928, + "tokens_seen": 77398016 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931995987963893, + "loss": 2.2196, + "theoretical_loss": 4.975863266159403, + "tokens_seen": 77463552 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931895687061183, + "loss": 2.2876, + "theoretical_loss": 4.975262254468299, + "tokens_seen": 77529088 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931795386158476, + "loss": 2.2847, + "theoretical_loss": 4.9746618927172435, + "tokens_seen": 77594624 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931695085255767, + "loss": 2.0784, + "theoretical_loss": 4.9740621796553395, + "tokens_seen": 77660160 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493159478435306, + "loss": 2.0894, + "theoretical_loss": 4.973463114035149, + "tokens_seen": 77725696 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931494483450352, + "loss": 2.2039, + "theoretical_loss": 4.972864694612684, + "tokens_seen": 77791232 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931394182547643, + "loss": 2.2074, + "theoretical_loss": 4.972266920147391, + "tokens_seen": 77856768 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931293881644935, + "loss": 2.2441, + "theoretical_loss": 4.971669789402139, + "tokens_seen": 77922304 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931193580742227, + "loss": 2.1439, + "theoretical_loss": 4.971073301143207, + "tokens_seen": 77987840 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931093279839519, + "loss": 2.2781, + "theoretical_loss": 4.970477454140278, + "tokens_seen": 78053376 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930992978936811, + "loss": 2.2112, + "theoretical_loss": 4.9698822471664155, + "tokens_seen": 78118912 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930892678034102, + "loss": 2.0608, + "theoretical_loss": 4.96928767899806, + "tokens_seen": 78184448 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930792377131394, + "loss": 2.1707, + "theoretical_loss": 4.9686937484150135, + "tokens_seen": 78249984 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930692076228686, + "loss": 2.1927, + "theoretical_loss": 4.96810045420043, + "tokens_seen": 78315520 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930591775325978, + "loss": 2.2224, + "theoretical_loss": 4.967507795140797, + "tokens_seen": 78381056 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493049147442327, + "loss": 2.3382, + "theoretical_loss": 4.966915770025935, + "tokens_seen": 78446592 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930391173520562, + "loss": 2.1709, + "theoretical_loss": 4.966324377648973, + "tokens_seen": 78512128 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930290872617853, + "loss": 2.2622, + "theoretical_loss": 4.965733616806345, + "tokens_seen": 78577664 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 32925, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 2.1840953826904297, + "objective/train/theoretical_loss": 4.965143486297777, + "objective/train/tokens_used": 78643200, + "theoretical_loss": 4.965143486297777, + "tokens_seen": 78643200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930190571715146, + "loss": 2.1822, + "theoretical_loss": 4.965143486297777, + "tokens_seen": 78643200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930090270812437, + "loss": 2.2833, + "theoretical_loss": 4.964553984926271, + "tokens_seen": 78708736 + }, + { + "epoch": 0.02, + "learning_rate": 0.000492998996990973, + "loss": 2.1754, + "theoretical_loss": 4.963965111498098, + "tokens_seen": 78774272 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929889669007021, + "loss": 2.1293, + "theoretical_loss": 4.963376864822784, + "tokens_seen": 78839808 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929789368104313, + "loss": 2.1438, + "theoretical_loss": 4.962789243713102, + "tokens_seen": 78905344 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929689067201605, + "loss": 2.1497, + "theoretical_loss": 4.962202246985054, + "tokens_seen": 78970880 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929588766298897, + "loss": 2.2664, + "theoretical_loss": 4.9616158734578635, + "tokens_seen": 79036416 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929488465396189, + "loss": 2.128, + "theoretical_loss": 4.961030121953965, + "tokens_seen": 79101952 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929388164493481, + "loss": 2.2491, + "theoretical_loss": 4.960444991298992, + "tokens_seen": 79167488 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929287863590773, + "loss": 2.0694, + "theoretical_loss": 4.959860480321762, + "tokens_seen": 79233024 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929187562688064, + "loss": 2.1747, + "theoretical_loss": 4.959276587854272, + "tokens_seen": 79298560 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929087261785356, + "loss": 2.1429, + "theoretical_loss": 4.958693312731681, + "tokens_seen": 79364096 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928986960882648, + "loss": 2.0871, + "theoretical_loss": 4.9581106537923, + "tokens_seen": 79429632 + }, + { + "epoch": 0.02, + "learning_rate": 0.000492888665997994, + "loss": 2.1955, + "theoretical_loss": 4.957528609877587, + "tokens_seen": 79495168 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928786359077232, + "loss": 2.1452, + "theoretical_loss": 4.9569471798321265, + "tokens_seen": 79560704 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928686058174523, + "loss": 2.2442, + "theoretical_loss": 4.9563663625036245, + "tokens_seen": 79626240 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928585757271815, + "loss": 1.9568, + "theoretical_loss": 4.955786156742898, + "tokens_seen": 79691776 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928485456369107, + "loss": 2.2331, + "theoretical_loss": 4.955206561403859, + "tokens_seen": 79757312 + }, + { + "epoch": 0.02, + "learning_rate": 0.00049283851554664, + "loss": 2.1303, + "theoretical_loss": 4.954627575343509, + "tokens_seen": 79822848 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928284854563691, + "loss": 2.1988, + "theoretical_loss": 4.954049197421925, + "tokens_seen": 79888384 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928184553660984, + "loss": 2.0649, + "theoretical_loss": 4.953471426502249, + "tokens_seen": 79953920 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928084252758274, + "loss": 2.1494, + "theoretical_loss": 4.952894261450679, + "tokens_seen": 80019456 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927983951855567, + "loss": 2.073, + "theoretical_loss": 4.952317701136457, + "tokens_seen": 80084992 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927883650952859, + "loss": 2.121, + "theoretical_loss": 4.9517417444318585, + "tokens_seen": 80150528 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927783350050151, + "loss": 2.055, + "theoretical_loss": 4.951166390212181, + "tokens_seen": 80216064 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927683049147443, + "loss": 2.2028, + "theoretical_loss": 4.950591637355737, + "tokens_seen": 80281600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927582748244734, + "loss": 2.096, + "theoretical_loss": 4.950017484743835, + "tokens_seen": 80347136 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927482447342026, + "loss": 2.1068, + "theoretical_loss": 4.949443931260783, + "tokens_seen": 80412672 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927382146439318, + "loss": 2.1564, + "theoretical_loss": 4.948870975793863, + "tokens_seen": 80478208 + }, + { + "epoch": 0.02, + "learning_rate": 0.000492728184553661, + "loss": 2.0264, + "theoretical_loss": 4.9482986172333305, + "tokens_seen": 80543744 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927181544633902, + "loss": 2.1329, + "theoretical_loss": 4.9477268544724, + "tokens_seen": 80609280 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927081243731193, + "loss": 2.1925, + "theoretical_loss": 4.947155686407239, + "tokens_seen": 80674816 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926980942828485, + "loss": 2.1403, + "theoretical_loss": 4.946585111936951, + "tokens_seen": 80740352 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926880641925777, + "loss": 2.16, + "theoretical_loss": 4.94601512996357, + "tokens_seen": 80805888 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926780341023069, + "loss": 1.9972, + "theoretical_loss": 4.945445739392049, + "tokens_seen": 80871424 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926680040120361, + "loss": 2.1644, + "theoretical_loss": 4.944876939130252, + "tokens_seen": 80936960 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926579739217654, + "loss": 2.0494, + "theoretical_loss": 4.94430872808894, + "tokens_seen": 81002496 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926479438314944, + "loss": 2.1274, + "theoretical_loss": 4.943741105181765, + "tokens_seen": 81068032 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926379137412237, + "loss": 2.079, + "theoretical_loss": 4.943174069325255, + "tokens_seen": 81133568 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926278836509528, + "loss": 2.1513, + "theoretical_loss": 4.94260761943881, + "tokens_seen": 81199104 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926178535606821, + "loss": 2.1822, + "theoretical_loss": 4.942041754444688, + "tokens_seen": 81264640 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926078234704113, + "loss": 2.2301, + "theoretical_loss": 4.941476473267996, + "tokens_seen": 81330176 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925977933801404, + "loss": 2.1067, + "theoretical_loss": 4.940911774836682, + "tokens_seen": 81395712 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925877632898696, + "loss": 2.1636, + "theoretical_loss": 4.940347658081521, + "tokens_seen": 81461248 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925777331995988, + "loss": 2.1347, + "theoretical_loss": 4.939784121936112, + "tokens_seen": 81526784 + }, + { + "epoch": 0.02, + "learning_rate": 0.000492567703109328, + "loss": 2.1765, + "theoretical_loss": 4.9392211653368605, + "tokens_seen": 81592320 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925576730190572, + "loss": 2.1521, + "theoretical_loss": 4.938658787222975, + "tokens_seen": 81657856 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925476429287864, + "loss": 2.158, + "theoretical_loss": 4.938096986536452, + "tokens_seen": 81723392 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925376128385155, + "loss": 2.0789, + "theoretical_loss": 4.937535762222075, + "tokens_seen": 81788928 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925275827482447, + "loss": 2.1719, + "theoretical_loss": 4.936975113227394, + "tokens_seen": 81854464 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 34255, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 2.054847478866577, + "objective/train/theoretical_loss": 4.936415038502723, + "objective/train/tokens_used": 81920000, + "theoretical_loss": 4.936415038502723, + "tokens_seen": 81920000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925175526579739, + "loss": 2.141, + "theoretical_loss": 4.936415038502723, + "tokens_seen": 81920000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925075225677031, + "loss": 1.9784, + "theoretical_loss": 4.93585553700113, + "tokens_seen": 81985536 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924974924774323, + "loss": 2.2181, + "theoretical_loss": 4.935296607678426, + "tokens_seen": 82051072 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924874623871615, + "loss": 2.0712, + "theoretical_loss": 4.934738249493156, + "tokens_seen": 82116608 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924774322968907, + "loss": 2.1668, + "theoretical_loss": 4.934180461406591, + "tokens_seen": 82182144 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924674022066199, + "loss": 2.2201, + "theoretical_loss": 4.933623242382714, + "tokens_seen": 82247680 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924573721163491, + "loss": 2.1607, + "theoretical_loss": 4.9330665913882195, + "tokens_seen": 82313216 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924473420260783, + "loss": 2.0915, + "theoretical_loss": 4.932510507392495, + "tokens_seen": 82378752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924373119358075, + "loss": 2.0363, + "theoretical_loss": 4.931954989367618, + "tokens_seen": 82444288 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004924272818455366, + "loss": 2.1755, + "theoretical_loss": 4.931400036288343, + "tokens_seen": 82509824 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004924172517552658, + "loss": 2.1144, + "theoretical_loss": 4.930845647132097, + "tokens_seen": 82575360 + }, + { + "epoch": 0.03, + "learning_rate": 0.000492407221664995, + "loss": 2.1281, + "theoretical_loss": 4.9302918208789634, + "tokens_seen": 82640896 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923971915747242, + "loss": 2.24, + "theoretical_loss": 4.929738556511681, + "tokens_seen": 82706432 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923871614844534, + "loss": 2.1999, + "theoretical_loss": 4.929185853015629, + "tokens_seen": 82771968 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923771313941825, + "loss": 2.2553, + "theoretical_loss": 4.928633709378822, + "tokens_seen": 82837504 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923671013039117, + "loss": 2.1714, + "theoretical_loss": 4.9280821245918975, + "tokens_seen": 82903040 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923570712136409, + "loss": 2.1311, + "theoretical_loss": 4.92753109764811, + "tokens_seen": 82968576 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923470411233702, + "loss": 2.1532, + "theoretical_loss": 4.92698062754332, + "tokens_seen": 83034112 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923370110330993, + "loss": 2.1617, + "theoretical_loss": 4.9264307132759875, + "tokens_seen": 83099648 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923269809428286, + "loss": 2.1446, + "theoretical_loss": 4.92588135384716, + "tokens_seen": 83165184 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923169508525576, + "loss": 2.1197, + "theoretical_loss": 4.925332548260469, + "tokens_seen": 83230720 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923069207622869, + "loss": 2.1109, + "theoretical_loss": 4.924784295522113, + "tokens_seen": 83296256 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922968906720161, + "loss": 2.2319, + "theoretical_loss": 4.92423659464086, + "tokens_seen": 83361792 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922868605817453, + "loss": 2.092, + "theoretical_loss": 4.923689444628027, + "tokens_seen": 83427328 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922768304914745, + "loss": 2.1336, + "theoretical_loss": 4.923142844497482, + "tokens_seen": 83492864 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922668004012036, + "loss": 2.1012, + "theoretical_loss": 4.922596793265625, + "tokens_seen": 83558400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922567703109328, + "loss": 2.0814, + "theoretical_loss": 4.922051289951391, + "tokens_seen": 83623936 + }, + { + "epoch": 0.03, + "learning_rate": 0.000492246740220662, + "loss": 1.9917, + "theoretical_loss": 4.921506333576232, + "tokens_seen": 83689472 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922367101303912, + "loss": 2.1219, + "theoretical_loss": 4.920961923164114, + "tokens_seen": 83755008 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922266800401204, + "loss": 2.0749, + "theoretical_loss": 4.920418057741504, + "tokens_seen": 83820544 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922166499498495, + "loss": 2.1441, + "theoretical_loss": 4.919874736337368, + "tokens_seen": 83886080 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922066198595787, + "loss": 2.057, + "theoretical_loss": 4.9193319579831565, + "tokens_seen": 83951616 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921965897693079, + "loss": 2.1784, + "theoretical_loss": 4.918789721712799, + "tokens_seen": 84017152 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921865596790371, + "loss": 2.1791, + "theoretical_loss": 4.918248026562697, + "tokens_seen": 84082688 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921765295887663, + "loss": 2.1269, + "theoretical_loss": 4.917706871571712, + "tokens_seen": 84148224 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921664994984956, + "loss": 2.1503, + "theoretical_loss": 4.9171662557811615, + "tokens_seen": 84213760 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921564694082246, + "loss": 2.0575, + "theoretical_loss": 4.916626178234807, + "tokens_seen": 84279296 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921464393179539, + "loss": 2.1918, + "theoretical_loss": 4.916086637978851, + "tokens_seen": 84344832 + }, + { + "epoch": 0.03, + "learning_rate": 0.000492136409227683, + "loss": 2.1375, + "theoretical_loss": 4.915547634061921, + "tokens_seen": 84410368 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921263791374123, + "loss": 2.1462, + "theoretical_loss": 4.91500916553507, + "tokens_seen": 84475904 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921163490471415, + "loss": 2.1226, + "theoretical_loss": 4.914471231451762, + "tokens_seen": 84541440 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921063189568706, + "loss": 1.9283, + "theoretical_loss": 4.913933830867868, + "tokens_seen": 84606976 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920962888665998, + "loss": 2.0468, + "theoretical_loss": 4.913396962841655, + "tokens_seen": 84672512 + }, + { + "epoch": 0.03, + "learning_rate": 0.000492086258776329, + "loss": 2.2936, + "theoretical_loss": 4.9128606264337815, + "tokens_seen": 84738048 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920762286860582, + "loss": 2.143, + "theoretical_loss": 4.912324820707285, + "tokens_seen": 84803584 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920661985957874, + "loss": 2.135, + "theoretical_loss": 4.911789544727581, + "tokens_seen": 84869120 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920561685055166, + "loss": 2.0551, + "theoretical_loss": 4.911254797562448, + "tokens_seen": 84934656 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920461384152457, + "loss": 2.186, + "theoretical_loss": 4.910720578282021, + "tokens_seen": 85000192 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920361083249749, + "loss": 2.1556, + "theoretical_loss": 4.91018688595879, + "tokens_seen": 85065728 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920260782347041, + "loss": 2.1728, + "theoretical_loss": 4.909653719667584, + "tokens_seen": 85131264 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 35040, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 2.1184592247009277, + "objective/train/theoretical_loss": 4.909121078485567, + "objective/train/tokens_used": 85196800, + "theoretical_loss": 4.909121078485567, + "tokens_seen": 85196800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920160481444333, + "loss": 2.1303, + "theoretical_loss": 4.909121078485567, + "tokens_seen": 85196800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920060180541625, + "loss": 2.1422, + "theoretical_loss": 4.908588961492235, + "tokens_seen": 85262336 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004919959879638916, + "loss": 2.166, + "theoretical_loss": 4.908057367769396, + "tokens_seen": 85327872 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004919859578736209, + "loss": 2.0999, + "theoretical_loss": 4.907526296401175, + "tokens_seen": 85393408 + }, + { + "epoch": 0.03, + "learning_rate": 0.00049197592778335, + "loss": 2.0262, + "theoretical_loss": 4.906995746474001, + "tokens_seen": 85458944 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004919658976930793, + "loss": 2.0295, + "theoretical_loss": 4.9064657170766, + "tokens_seen": 85524480 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004919558676028084, + "loss": 2.0961, + "theoretical_loss": 4.905936207299984, + "tokens_seen": 85590016 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004919458375125377, + "loss": 2.1232, + "theoretical_loss": 4.90540721623745, + "tokens_seen": 85655552 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004919358074222668, + "loss": 2.1439, + "theoretical_loss": 4.904878742984569, + "tokens_seen": 85721088 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491925777331996, + "loss": 1.9956, + "theoretical_loss": 4.904350786639178, + "tokens_seen": 85786624 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004919157472417252, + "loss": 2.1989, + "theoretical_loss": 4.903823346301374, + "tokens_seen": 85852160 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004919057171514544, + "loss": 2.0608, + "theoretical_loss": 4.903296421073506, + "tokens_seen": 85917696 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918956870611836, + "loss": 2.0034, + "theoretical_loss": 4.902770010060166, + "tokens_seen": 85983232 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918856569709127, + "loss": 1.9477, + "theoretical_loss": 4.902244112368188, + "tokens_seen": 86048768 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918756268806419, + "loss": 2.2537, + "theoretical_loss": 4.901718727106631, + "tokens_seen": 86114304 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918655967903711, + "loss": 2.0522, + "theoretical_loss": 4.90119385338678, + "tokens_seen": 86179840 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918555667001003, + "loss": 2.0268, + "theoretical_loss": 4.900669490322134, + "tokens_seen": 86245376 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918455366098295, + "loss": 2.0836, + "theoretical_loss": 4.900145637028402, + "tokens_seen": 86310912 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918355065195586, + "loss": 2.1121, + "theoretical_loss": 4.899622292623493, + "tokens_seen": 86376448 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918254764292878, + "loss": 2.0849, + "theoretical_loss": 4.8990994562275105, + "tokens_seen": 86441984 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491815446339017, + "loss": 2.1797, + "theoretical_loss": 4.898577126962746, + "tokens_seen": 86507520 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918054162487463, + "loss": 2.057, + "theoretical_loss": 4.898055303953669, + "tokens_seen": 86573056 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917953861584754, + "loss": 2.1378, + "theoretical_loss": 4.897533986326922, + "tokens_seen": 86638592 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917853560682047, + "loss": 1.9764, + "theoretical_loss": 4.897013173211316, + "tokens_seen": 86704128 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917753259779337, + "loss": 2.1249, + "theoretical_loss": 4.896492863737818, + "tokens_seen": 86769664 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491765295887663, + "loss": 2.148, + "theoretical_loss": 4.895973057039548, + "tokens_seen": 86835200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917552657973922, + "loss": 2.1167, + "theoretical_loss": 4.89545375225177, + "tokens_seen": 86900736 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917452357071214, + "loss": 2.0332, + "theoretical_loss": 4.894934948511889, + "tokens_seen": 86966272 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917352056168506, + "loss": 2.1212, + "theoretical_loss": 4.894416644959437, + "tokens_seen": 87031808 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917251755265797, + "loss": 2.1001, + "theoretical_loss": 4.893898840736071, + "tokens_seen": 87097344 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917151454363089, + "loss": 2.0, + "theoretical_loss": 4.8933815349855685, + "tokens_seen": 87162880 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917051153460381, + "loss": 2.1466, + "theoretical_loss": 4.892864726853814, + "tokens_seen": 87228416 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916950852557673, + "loss": 2.0072, + "theoretical_loss": 4.892348415488799, + "tokens_seen": 87293952 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916850551654965, + "loss": 1.9493, + "theoretical_loss": 4.891832600040609, + "tokens_seen": 87359488 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916750250752258, + "loss": 1.9947, + "theoretical_loss": 4.891317279661422, + "tokens_seen": 87425024 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916649949849548, + "loss": 2.0316, + "theoretical_loss": 4.890802453505498, + "tokens_seen": 87490560 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916549648946841, + "loss": 2.1438, + "theoretical_loss": 4.890288120729174, + "tokens_seen": 87556096 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916449348044132, + "loss": 2.1975, + "theoretical_loss": 4.88977428049086, + "tokens_seen": 87621632 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916349047141425, + "loss": 2.0383, + "theoretical_loss": 4.889260931951026, + "tokens_seen": 87687168 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916248746238717, + "loss": 1.9804, + "theoretical_loss": 4.888748074272201, + "tokens_seen": 87752704 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916148445336008, + "loss": 2.0048, + "theoretical_loss": 4.888235706618964, + "tokens_seen": 87818240 + }, + { + "epoch": 0.03, + "learning_rate": 0.00049160481444333, + "loss": 2.1633, + "theoretical_loss": 4.887723828157939, + "tokens_seen": 87883776 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915947843530592, + "loss": 2.0701, + "theoretical_loss": 4.8872124380577855, + "tokens_seen": 87949312 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915847542627884, + "loss": 2.0728, + "theoretical_loss": 4.886701535489195, + "tokens_seen": 88014848 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915747241725176, + "loss": 2.0088, + "theoretical_loss": 4.886191119624883, + "tokens_seen": 88080384 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915646940822468, + "loss": 2.1099, + "theoretical_loss": 4.885681189639584, + "tokens_seen": 88145920 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915546639919759, + "loss": 2.0012, + "theoretical_loss": 4.885171744710044, + "tokens_seen": 88211456 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915446339017051, + "loss": 2.0901, + "theoretical_loss": 4.884662784015012, + "tokens_seen": 88276992 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915346038114343, + "loss": 2.0321, + "theoretical_loss": 4.884154306735239, + "tokens_seen": 88342528 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915245737211635, + "loss": 2.042, + "theoretical_loss": 4.8836463120534646, + "tokens_seen": 88408064 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 36339, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 2.0729150772094727, + "objective/train/theoretical_loss": 4.88313879915442, + "objective/train/tokens_used": 88473600, + "theoretical_loss": 4.88313879915442, + "tokens_seen": 88473600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915145436308927, + "loss": 2.1013, + "theoretical_loss": 4.88313879915442, + "tokens_seen": 88473600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915045135406218, + "loss": 2.1162, + "theoretical_loss": 4.882631767224812, + "tokens_seen": 88539136 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914944834503511, + "loss": 2.053, + "theoretical_loss": 4.882125215453321, + "tokens_seen": 88604672 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914844533600802, + "loss": 2.1618, + "theoretical_loss": 4.881619143030598, + "tokens_seen": 88670208 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914744232698095, + "loss": 2.0495, + "theoretical_loss": 4.8811135491492506, + "tokens_seen": 88735744 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914643931795386, + "loss": 1.998, + "theoretical_loss": 4.880608433003846, + "tokens_seen": 88801280 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914543630892679, + "loss": 2.1016, + "theoretical_loss": 4.880103793790896, + "tokens_seen": 88866816 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491444332998997, + "loss": 2.082, + "theoretical_loss": 4.87959963070886, + "tokens_seen": 88932352 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914343029087262, + "loss": 2.1702, + "theoretical_loss": 4.879095942958127, + "tokens_seen": 88997888 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914242728184554, + "loss": 1.9792, + "theoretical_loss": 4.878592729741024, + "tokens_seen": 89063424 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914142427281846, + "loss": 2.1423, + "theoretical_loss": 4.878089990261797, + "tokens_seen": 89128960 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914042126379138, + "loss": 2.0721, + "theoretical_loss": 4.87758772372661, + "tokens_seen": 89194496 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913941825476429, + "loss": 2.1672, + "theoretical_loss": 4.877085929343544, + "tokens_seen": 89260032 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913841524573721, + "loss": 2.0275, + "theoretical_loss": 4.876584606322583, + "tokens_seen": 89325568 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913741223671013, + "loss": 2.0965, + "theoretical_loss": 4.8760837538756086, + "tokens_seen": 89391104 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913640922768305, + "loss": 2.1259, + "theoretical_loss": 4.875583371216404, + "tokens_seen": 89456640 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913540621865597, + "loss": 2.0145, + "theoretical_loss": 4.875083457560633, + "tokens_seen": 89522176 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913440320962888, + "loss": 2.1947, + "theoretical_loss": 4.874584012125847, + "tokens_seen": 89587712 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491334002006018, + "loss": 2.096, + "theoretical_loss": 4.874085034131472, + "tokens_seen": 89653248 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913239719157472, + "loss": 2.0418, + "theoretical_loss": 4.873586522798803, + "tokens_seen": 89718784 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913139418254765, + "loss": 2.0009, + "theoretical_loss": 4.873088477351005, + "tokens_seen": 89784320 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913039117352056, + "loss": 2.0948, + "theoretical_loss": 4.8725908970130964, + "tokens_seen": 89849856 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912938816449349, + "loss": 1.9871, + "theoretical_loss": 4.872093781011952, + "tokens_seen": 89915392 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912838515546639, + "loss": 2.1193, + "theoretical_loss": 4.871597128576292, + "tokens_seen": 89980928 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912738214643932, + "loss": 2.094, + "theoretical_loss": 4.8711009389366815, + "tokens_seen": 90046464 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912637913741224, + "loss": 2.0373, + "theoretical_loss": 4.870605211325517, + "tokens_seen": 90112000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912537612838516, + "loss": 2.1345, + "theoretical_loss": 4.870109944977029, + "tokens_seen": 90177536 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912437311935808, + "loss": 1.9899, + "theoretical_loss": 4.869615139127273, + "tokens_seen": 90243072 + }, + { + "epoch": 0.03, + "learning_rate": 0.00049123370110331, + "loss": 2.0608, + "theoretical_loss": 4.869120793014117, + "tokens_seen": 90308608 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912236710130391, + "loss": 2.1495, + "theoretical_loss": 4.868626905877251, + "tokens_seen": 90374144 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912136409227683, + "loss": 2.0951, + "theoretical_loss": 4.8681334769581674, + "tokens_seen": 90439680 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912036108324975, + "loss": 2.2093, + "theoretical_loss": 4.867640505500161, + "tokens_seen": 90505216 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911935807422267, + "loss": 2.1228, + "theoretical_loss": 4.8671479907483235, + "tokens_seen": 90570752 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911835506519559, + "loss": 2.2754, + "theoretical_loss": 4.866655931949537, + "tokens_seen": 90636288 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491173520561685, + "loss": 2.0486, + "theoretical_loss": 4.86616432835247, + "tokens_seen": 90701824 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911634904714142, + "loss": 2.1122, + "theoretical_loss": 4.865673179207572, + "tokens_seen": 90767360 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911534603811434, + "loss": 2.1136, + "theoretical_loss": 4.865182483767063, + "tokens_seen": 90832896 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911434302908726, + "loss": 2.1166, + "theoretical_loss": 4.864692241284933, + "tokens_seen": 90898432 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911334002006019, + "loss": 1.9974, + "theoretical_loss": 4.864202451016939, + "tokens_seen": 90963968 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911233701103309, + "loss": 2.06, + "theoretical_loss": 4.863713112220592, + "tokens_seen": 91029504 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911133400200602, + "loss": 2.1148, + "theoretical_loss": 4.8632242241551555, + "tokens_seen": 91095040 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911033099297893, + "loss": 2.0001, + "theoretical_loss": 4.862735786081642, + "tokens_seen": 91160576 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004910932798395186, + "loss": 2.0647, + "theoretical_loss": 4.862247797262806, + "tokens_seen": 91226112 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004910832497492478, + "loss": 2.0334, + "theoretical_loss": 4.861760256963136, + "tokens_seen": 91291648 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491073219658977, + "loss": 1.9698, + "theoretical_loss": 4.861273164448854, + "tokens_seen": 91357184 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004910631895687061, + "loss": 2.0632, + "theoretical_loss": 4.860786518987906, + "tokens_seen": 91422720 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004910531594784353, + "loss": 2.1532, + "theoretical_loss": 4.86030031984996, + "tokens_seen": 91488256 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004910431293881645, + "loss": 2.1436, + "theoretical_loss": 4.859814566306397, + "tokens_seen": 91553792 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004910330992978937, + "loss": 2.1478, + "theoretical_loss": 4.859329257630311, + "tokens_seen": 91619328 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004910230692076229, + "loss": 2.0571, + "theoretical_loss": 4.858844393096497, + "tokens_seen": 91684864 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 37311, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.9555306434631348, + "objective/train/theoretical_loss": 4.858359971981454, + "objective/train/tokens_used": 91750400, + "theoretical_loss": 4.858359971981454, + "tokens_seen": 91750400 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491013039117352, + "loss": 2.0158, + "theoretical_loss": 4.858359971981454, + "tokens_seen": 91750400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004910030090270812, + "loss": 2.0495, + "theoretical_loss": 4.85787599356337, + "tokens_seen": 91815936 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909929789368104, + "loss": 2.1527, + "theoretical_loss": 4.857392457122128, + "tokens_seen": 91881472 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909829488465397, + "loss": 2.02, + "theoretical_loss": 4.856909361939288, + "tokens_seen": 91947008 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909729187562688, + "loss": 2.1547, + "theoretical_loss": 4.856426707298096, + "tokens_seen": 92012544 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909628886659981, + "loss": 2.1213, + "theoretical_loss": 4.8559444924834665, + "tokens_seen": 92078080 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909528585757272, + "loss": 2.0169, + "theoretical_loss": 4.855462716781985, + "tokens_seen": 92143616 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909428284854564, + "loss": 2.0907, + "theoretical_loss": 4.854981379481901, + "tokens_seen": 92209152 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909327983951856, + "loss": 2.0224, + "theoretical_loss": 4.854500479873119, + "tokens_seen": 92274688 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909227683049148, + "loss": 2.1277, + "theoretical_loss": 4.854020017247203, + "tokens_seen": 92340224 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490912738214644, + "loss": 2.0118, + "theoretical_loss": 4.853539990897358, + "tokens_seen": 92405760 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909027081243731, + "loss": 2.0112, + "theoretical_loss": 4.85306040011844, + "tokens_seen": 92471296 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908926780341023, + "loss": 2.1496, + "theoretical_loss": 4.852581244206938, + "tokens_seen": 92536832 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908826479438315, + "loss": 1.9855, + "theoretical_loss": 4.852102522460975, + "tokens_seen": 92602368 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908726178535607, + "loss": 2.1497, + "theoretical_loss": 4.851624234180306, + "tokens_seen": 92667904 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908625877632899, + "loss": 2.0846, + "theoretical_loss": 4.851146378666305, + "tokens_seen": 92733440 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490852557673019, + "loss": 2.1442, + "theoretical_loss": 4.85066895522197, + "tokens_seen": 92798976 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908425275827482, + "loss": 2.1319, + "theoretical_loss": 4.850191963151908, + "tokens_seen": 92864512 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908324974924774, + "loss": 2.0889, + "theoretical_loss": 4.849715401762337, + "tokens_seen": 92930048 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908224674022067, + "loss": 2.1456, + "theoretical_loss": 4.849239270361078, + "tokens_seen": 92995584 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908124373119358, + "loss": 2.0071, + "theoretical_loss": 4.848763568257554, + "tokens_seen": 93061120 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908024072216651, + "loss": 2.2721, + "theoretical_loss": 4.848288294762779, + "tokens_seen": 93126656 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907923771313941, + "loss": 1.9729, + "theoretical_loss": 4.847813449189358, + "tokens_seen": 93192192 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907823470411234, + "loss": 1.9648, + "theoretical_loss": 4.847339030851482, + "tokens_seen": 93257728 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907723169508526, + "loss": 2.0563, + "theoretical_loss": 4.84686503906492, + "tokens_seen": 93323264 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907622868605818, + "loss": 2.0532, + "theoretical_loss": 4.846391473147019, + "tokens_seen": 93388800 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490752256770311, + "loss": 2.0458, + "theoretical_loss": 4.8459183324166935, + "tokens_seen": 93454336 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907422266800401, + "loss": 2.0448, + "theoretical_loss": 4.845445616194426, + "tokens_seen": 93519872 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907321965897693, + "loss": 2.0978, + "theoretical_loss": 4.844973323802259, + "tokens_seen": 93585408 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907221664994985, + "loss": 2.0167, + "theoretical_loss": 4.844501454563792, + "tokens_seen": 93650944 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907121364092277, + "loss": 1.9392, + "theoretical_loss": 4.844030007804177, + "tokens_seen": 93716480 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907021063189569, + "loss": 1.9678, + "theoretical_loss": 4.843558982850113, + "tokens_seen": 93782016 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490692076228686, + "loss": 1.9663, + "theoretical_loss": 4.84308837902984, + "tokens_seen": 93847552 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906820461384152, + "loss": 2.0299, + "theoretical_loss": 4.842618195673138, + "tokens_seen": 93913088 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906720160481444, + "loss": 2.0662, + "theoretical_loss": 4.84214843211132, + "tokens_seen": 93978624 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906619859578736, + "loss": 1.9356, + "theoretical_loss": 4.841679087677226, + "tokens_seen": 94044160 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906519558676028, + "loss": 1.9558, + "theoretical_loss": 4.841210161705223, + "tokens_seen": 94109696 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906419257773321, + "loss": 2.015, + "theoretical_loss": 4.840741653531195, + "tokens_seen": 94175232 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906318956870611, + "loss": 2.0665, + "theoretical_loss": 4.840273562492545, + "tokens_seen": 94240768 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906218655967904, + "loss": 2.0958, + "theoretical_loss": 4.839805887928181, + "tokens_seen": 94306304 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906118355065195, + "loss": 1.9677, + "theoretical_loss": 4.839338629178522, + "tokens_seen": 94371840 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906018054162488, + "loss": 2.1221, + "theoretical_loss": 4.8388717855854875, + "tokens_seen": 94437376 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490591775325978, + "loss": 2.0324, + "theoretical_loss": 4.838405356492494, + "tokens_seen": 94502912 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905817452357072, + "loss": 2.0121, + "theoretical_loss": 4.83793934124445, + "tokens_seen": 94568448 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905717151454363, + "loss": 1.9922, + "theoretical_loss": 4.837473739187754, + "tokens_seen": 94633984 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905616850551655, + "loss": 1.9959, + "theoretical_loss": 4.837008549670285, + "tokens_seen": 94699520 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905516549648947, + "loss": 2.1954, + "theoretical_loss": 4.8365437720414075, + "tokens_seen": 94765056 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905416248746239, + "loss": 2.0049, + "theoretical_loss": 4.836079405651956, + "tokens_seen": 94830592 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905315947843531, + "loss": 2.0439, + "theoretical_loss": 4.835615449854238, + "tokens_seen": 94896128 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905215646940822, + "loss": 1.9486, + "theoretical_loss": 4.8351519040020285, + "tokens_seen": 94961664 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 38462, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 2.1275134086608887, + "objective/train/theoretical_loss": 4.834688767450562, + "objective/train/tokens_used": 95027200, + "theoretical_loss": 4.834688767450562, + "tokens_seen": 95027200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905115346038114, + "loss": 2.0225, + "theoretical_loss": 4.834688767450562, + "tokens_seen": 95027200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905015045135406, + "loss": 2.1486, + "theoretical_loss": 4.8342260395565315, + "tokens_seen": 95092736 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904914744232698, + "loss": 1.9855, + "theoretical_loss": 4.833763719678085, + "tokens_seen": 95158272 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490481444332999, + "loss": 1.9887, + "theoretical_loss": 4.833301807174822, + "tokens_seen": 95223808 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904714142427281, + "loss": 1.9819, + "theoretical_loss": 4.83284030140778, + "tokens_seen": 95289344 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904613841524574, + "loss": 1.9368, + "theoretical_loss": 4.832379201739442, + "tokens_seen": 95354880 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904513540621865, + "loss": 2.0481, + "theoretical_loss": 4.831918507533728, + "tokens_seen": 95420416 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904413239719158, + "loss": 1.9981, + "theoretical_loss": 4.831458218155989, + "tokens_seen": 95485952 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904312938816449, + "loss": 2.072, + "theoretical_loss": 4.830998332973005, + "tokens_seen": 95551488 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904212637913742, + "loss": 2.0901, + "theoretical_loss": 4.830538851352976, + "tokens_seen": 95617024 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904112337011033, + "loss": 2.0386, + "theoretical_loss": 4.830079772665529, + "tokens_seen": 95682560 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904012036108325, + "loss": 2.0953, + "theoretical_loss": 4.829621096281702, + "tokens_seen": 95748096 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903911735205617, + "loss": 1.987, + "theoretical_loss": 4.8291628215739415, + "tokens_seen": 95813632 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903811434302909, + "loss": 2.0044, + "theoretical_loss": 4.828704947916108, + "tokens_seen": 95879168 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903711133400201, + "loss": 1.9612, + "theoretical_loss": 4.82824747468346, + "tokens_seen": 95944704 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903610832497492, + "loss": 2.0226, + "theoretical_loss": 4.827790401252658, + "tokens_seen": 96010240 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903510531594784, + "loss": 2.0155, + "theoretical_loss": 4.827333727001756, + "tokens_seen": 96075776 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903410230692076, + "loss": 1.9466, + "theoretical_loss": 4.8268774513101995, + "tokens_seen": 96141312 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903309929789368, + "loss": 2.1026, + "theoretical_loss": 4.82642157355882, + "tokens_seen": 96206848 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490320962888666, + "loss": 2.2056, + "theoretical_loss": 4.825966093129834, + "tokens_seen": 96272384 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903109327983952, + "loss": 1.9112, + "theoretical_loss": 4.825511009406833, + "tokens_seen": 96337920 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903009027081243, + "loss": 2.0372, + "theoretical_loss": 4.825056321774788, + "tokens_seen": 96403456 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902908726178535, + "loss": 2.1271, + "theoretical_loss": 4.8246020296200385, + "tokens_seen": 96468992 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902808425275828, + "loss": 1.823, + "theoretical_loss": 4.82414813233029, + "tokens_seen": 96534528 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902708124373119, + "loss": 1.9885, + "theoretical_loss": 4.823694629294609, + "tokens_seen": 96600064 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902607823470412, + "loss": 2.0157, + "theoretical_loss": 4.823241519903428, + "tokens_seen": 96665600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902507522567703, + "loss": 1.9751, + "theoretical_loss": 4.8227888035485265, + "tokens_seen": 96731136 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902407221664995, + "loss": 2.0367, + "theoretical_loss": 4.82233647962304, + "tokens_seen": 96796672 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902306920762287, + "loss": 2.0172, + "theoretical_loss": 4.821884547521449, + "tokens_seen": 96862208 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902206619859579, + "loss": 2.1037, + "theoretical_loss": 4.821433006639578, + "tokens_seen": 96927744 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902106318956871, + "loss": 1.9378, + "theoretical_loss": 4.820981856374589, + "tokens_seen": 96993280 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902006018054163, + "loss": 1.964, + "theoretical_loss": 4.8205310961249825, + "tokens_seen": 97058816 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901905717151454, + "loss": 2.0315, + "theoretical_loss": 4.820080725290589, + "tokens_seen": 97124352 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901805416248746, + "loss": 1.9901, + "theoretical_loss": 4.819630743272566, + "tokens_seen": 97189888 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901705115346038, + "loss": 1.9195, + "theoretical_loss": 4.819181149473396, + "tokens_seen": 97255424 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490160481444333, + "loss": 2.0097, + "theoretical_loss": 4.81873194329688, + "tokens_seen": 97320960 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901504513540623, + "loss": 2.0956, + "theoretical_loss": 4.81828312414814, + "tokens_seen": 97386496 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901404212637913, + "loss": 1.9222, + "theoretical_loss": 4.817834691433602, + "tokens_seen": 97452032 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901303911735206, + "loss": 2.041, + "theoretical_loss": 4.81738664456101, + "tokens_seen": 97517568 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901203610832497, + "loss": 2.0114, + "theoretical_loss": 4.816938982939407, + "tokens_seen": 97583104 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490110330992979, + "loss": 1.9939, + "theoretical_loss": 4.816491705979138, + "tokens_seen": 97648640 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901003009027082, + "loss": 1.9159, + "theoretical_loss": 4.816044813091848, + "tokens_seen": 97714176 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900902708124374, + "loss": 2.0422, + "theoretical_loss": 4.815598303690473, + "tokens_seen": 97779712 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900802407221665, + "loss": 2.0264, + "theoretical_loss": 4.8151521771892405, + "tokens_seen": 97845248 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900702106318957, + "loss": 2.0854, + "theoretical_loss": 4.814706433003665, + "tokens_seen": 97910784 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900601805416249, + "loss": 1.963, + "theoretical_loss": 4.8142610705505415, + "tokens_seen": 97976320 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900501504513541, + "loss": 2.0787, + "theoretical_loss": 4.813816089247945, + "tokens_seen": 98041856 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900401203610833, + "loss": 1.9771, + "theoretical_loss": 4.813371488515227, + "tokens_seen": 98107392 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900300902708124, + "loss": 1.997, + "theoretical_loss": 4.812927267773008, + "tokens_seen": 98172928 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900200601805416, + "loss": 1.9904, + "theoretical_loss": 4.812483426443181, + "tokens_seen": 98238464 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 39367, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.932283878326416, + "objective/train/theoretical_loss": 4.812039963948898, + "objective/train/tokens_used": 98304000, + "theoretical_loss": 4.812039963948898, + "tokens_seen": 98304000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900100300902708, + "loss": 1.948, + "theoretical_loss": 4.812039963948898, + "tokens_seen": 98304000 + }, + { + "epoch": 0.03, + "learning_rate": 0.00049, + "loss": 1.888, + "theoretical_loss": 4.811596879714575, + "tokens_seen": 98369536 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899899699097292, + "loss": 2.0716, + "theoretical_loss": 4.811154173165886, + "tokens_seen": 98435072 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899799398194583, + "loss": 2.0828, + "theoretical_loss": 4.810711843729758, + "tokens_seen": 98500608 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899699097291876, + "loss": 1.892, + "theoretical_loss": 4.810269890834364, + "tokens_seen": 98566144 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899598796389167, + "loss": 2.041, + "theoretical_loss": 4.809828313909129, + "tokens_seen": 98631680 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489949849548646, + "loss": 1.9778, + "theoretical_loss": 4.809387112384721, + "tokens_seen": 98697216 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899398194583751, + "loss": 2.0342, + "theoretical_loss": 4.808946285693043, + "tokens_seen": 98762752 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899297893681044, + "loss": 2.0035, + "theoretical_loss": 4.808505833267237, + "tokens_seen": 98828288 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899197592778335, + "loss": 2.0771, + "theoretical_loss": 4.808065754541676, + "tokens_seen": 98893824 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899097291875627, + "loss": 2.013, + "theoretical_loss": 4.807626048951965, + "tokens_seen": 98959360 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898996990972919, + "loss": 2.0409, + "theoretical_loss": 4.807186715934931, + "tokens_seen": 99024896 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898896690070211, + "loss": 1.9791, + "theoretical_loss": 4.806747754928622, + "tokens_seen": 99090432 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898796389167503, + "loss": 1.887, + "theoretical_loss": 4.806309165372311, + "tokens_seen": 99155968 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898696088264794, + "loss": 1.9089, + "theoretical_loss": 4.805870946706479, + "tokens_seen": 99221504 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898595787362086, + "loss": 2.1005, + "theoretical_loss": 4.805433098372822, + "tokens_seen": 99287040 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898495486459378, + "loss": 1.9704, + "theoretical_loss": 4.8049956198142425, + "tokens_seen": 99352576 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489839518555667, + "loss": 2.0218, + "theoretical_loss": 4.804558510474852, + "tokens_seen": 99418112 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898294884653962, + "loss": 2.0656, + "theoretical_loss": 4.804121769799959, + "tokens_seen": 99483648 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898194583751254, + "loss": 2.1153, + "theoretical_loss": 4.8036853972360705, + "tokens_seen": 99549184 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898094282848545, + "loss": 2.0669, + "theoretical_loss": 4.80324939223089, + "tokens_seen": 99614720 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897993981945837, + "loss": 2.09, + "theoretical_loss": 4.802813754233311, + "tokens_seen": 99680256 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489789368104313, + "loss": 1.9777, + "theoretical_loss": 4.802378482693417, + "tokens_seen": 99745792 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897793380140421, + "loss": 2.1037, + "theoretical_loss": 4.801943577062472, + "tokens_seen": 99811328 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897693079237714, + "loss": 2.0342, + "theoretical_loss": 4.801509036792925, + "tokens_seen": 99876864 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897592778335005, + "loss": 2.075, + "theoretical_loss": 4.801074861338401, + "tokens_seen": 99942400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897492477432297, + "loss": 2.0537, + "theoretical_loss": 4.800641050153703, + "tokens_seen": 100007936 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897392176529589, + "loss": 1.9495, + "theoretical_loss": 4.800207602694801, + "tokens_seen": 100073472 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897291875626881, + "loss": 1.9572, + "theoretical_loss": 4.7997745184188325, + "tokens_seen": 100139008 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897191574724173, + "loss": 2.067, + "theoretical_loss": 4.799341796784106, + "tokens_seen": 100204544 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897091273821465, + "loss": 1.9989, + "theoretical_loss": 4.798909437250086, + "tokens_seen": 100270080 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896990972918756, + "loss": 1.9641, + "theoretical_loss": 4.798477439277397, + "tokens_seen": 100335616 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896890672016048, + "loss": 2.1172, + "theoretical_loss": 4.7980458023278185, + "tokens_seen": 100401152 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489679037111334, + "loss": 2.0706, + "theoretical_loss": 4.797614525864282, + "tokens_seen": 100466688 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896690070210632, + "loss": 2.0677, + "theoretical_loss": 4.797183609350869, + "tokens_seen": 100532224 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896589769307924, + "loss": 1.8514, + "theoretical_loss": 4.796753052252802, + "tokens_seen": 100597760 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896489468405215, + "loss": 2.0691, + "theoretical_loss": 4.79632285403645, + "tokens_seen": 100663296 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896389167502507, + "loss": 1.9544, + "theoretical_loss": 4.795893014169323, + "tokens_seen": 100728832 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896288866599799, + "loss": 2.0451, + "theoretical_loss": 4.795463532120058, + "tokens_seen": 100794368 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896188565697091, + "loss": 1.9919, + "theoretical_loss": 4.795034407358435, + "tokens_seen": 100859904 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896088264794384, + "loss": 1.9879, + "theoretical_loss": 4.794605639355357, + "tokens_seen": 100925440 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895987963891674, + "loss": 2.1316, + "theoretical_loss": 4.794177227582855, + "tokens_seen": 100990976 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895887662988967, + "loss": 2.0067, + "theoretical_loss": 4.793749171514085, + "tokens_seen": 101056512 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895787362086259, + "loss": 2.0172, + "theoretical_loss": 4.793321470623321, + "tokens_seen": 101122048 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895687061183551, + "loss": 1.8233, + "theoretical_loss": 4.792894124385955, + "tokens_seen": 101187584 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895586760280843, + "loss": 2.0046, + "theoretical_loss": 4.792467132278494, + "tokens_seen": 101253120 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895486459378135, + "loss": 1.8794, + "theoretical_loss": 4.792040493778553, + "tokens_seen": 101318656 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895386158475426, + "loss": 1.9705, + "theoretical_loss": 4.791614208364859, + "tokens_seen": 101384192 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895285857572718, + "loss": 2.0554, + "theoretical_loss": 4.791188275517239, + "tokens_seen": 101449728 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489518555667001, + "loss": 1.9575, + "theoretical_loss": 4.7907626947166255, + "tokens_seen": 101515264 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 40629, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.930105209350586, + "objective/train/theoretical_loss": 4.790337465445049, + "objective/train/tokens_used": 101580800, + "theoretical_loss": 4.790337465445049, + "tokens_seen": 101580800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895085255767302, + "loss": 2.0434, + "theoretical_loss": 4.790337465445049, + "tokens_seen": 101580800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894984954864594, + "loss": 2.0776, + "theoretical_loss": 4.7899125871856345, + "tokens_seen": 101646336 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894884653961885, + "loss": 2.0423, + "theoretical_loss": 4.789488059422599, + "tokens_seen": 101711872 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894784353059178, + "loss": 2.013, + "theoretical_loss": 4.789063881641251, + "tokens_seen": 101777408 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894684052156469, + "loss": 2.0751, + "theoretical_loss": 4.7886400533279865, + "tokens_seen": 101842944 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894583751253762, + "loss": 2.0156, + "theoretical_loss": 4.788216573970281, + "tokens_seen": 101908480 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894483450351053, + "loss": 2.026, + "theoretical_loss": 4.787793443056694, + "tokens_seen": 101974016 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894383149448346, + "loss": 1.9055, + "theoretical_loss": 4.787370660076862, + "tokens_seen": 102039552 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894282848545637, + "loss": 2.0088, + "theoretical_loss": 4.786948224521495, + "tokens_seen": 102105088 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894182547642929, + "loss": 1.9269, + "theoretical_loss": 4.786526135882378, + "tokens_seen": 102170624 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894082246740221, + "loss": 1.9004, + "theoretical_loss": 4.786104393652359, + "tokens_seen": 102236160 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893981945837513, + "loss": 1.944, + "theoretical_loss": 4.785682997325358, + "tokens_seen": 102301696 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893881644934805, + "loss": 1.8461, + "theoretical_loss": 4.7852619463963535, + "tokens_seen": 102367232 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893781344032096, + "loss": 1.9523, + "theoretical_loss": 4.784841240361389, + "tokens_seen": 102432768 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893681043129388, + "loss": 1.89, + "theoretical_loss": 4.784420878717558, + "tokens_seen": 102498304 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489358074222668, + "loss": 1.9798, + "theoretical_loss": 4.7840008609630145, + "tokens_seen": 102563840 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893480441323972, + "loss": 1.9498, + "theoretical_loss": 4.783581186596962, + "tokens_seen": 102629376 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893380140421264, + "loss": 2.0365, + "theoretical_loss": 4.78316185511965, + "tokens_seen": 102694912 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893279839518556, + "loss": 2.068, + "theoretical_loss": 4.782742866032379, + "tokens_seen": 102760448 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893179538615847, + "loss": 2.0142, + "theoretical_loss": 4.782324218837486, + "tokens_seen": 102825984 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893079237713139, + "loss": 1.9616, + "theoretical_loss": 4.781905913038351, + "tokens_seen": 102891520 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892978936810432, + "loss": 1.9779, + "theoretical_loss": 4.781487948139393, + "tokens_seen": 102957056 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892878635907723, + "loss": 1.8424, + "theoretical_loss": 4.7810703236460625, + "tokens_seen": 103022592 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892778335005016, + "loss": 1.9713, + "theoretical_loss": 4.780653039064843, + "tokens_seen": 103088128 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892678034102307, + "loss": 1.9692, + "theoretical_loss": 4.780236093903245, + "tokens_seen": 103153664 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892577733199599, + "loss": 1.9727, + "theoretical_loss": 4.779819487669805, + "tokens_seen": 103219200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892477432296891, + "loss": 1.9565, + "theoretical_loss": 4.779403219874085, + "tokens_seen": 103284736 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892377131394183, + "loss": 1.9347, + "theoretical_loss": 4.778987290026665, + "tokens_seen": 103350272 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892276830491475, + "loss": 2.0555, + "theoretical_loss": 4.778571697639142, + "tokens_seen": 103415808 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892176529588767, + "loss": 1.9748, + "theoretical_loss": 4.778156442224132, + "tokens_seen": 103481344 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892076228686058, + "loss": 1.9287, + "theoretical_loss": 4.7777415232952585, + "tokens_seen": 103546880 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489197592778335, + "loss": 2.0705, + "theoretical_loss": 4.777326940367155, + "tokens_seen": 103612416 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891875626880642, + "loss": 1.9995, + "theoretical_loss": 4.776912692955463, + "tokens_seen": 103677952 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891775325977934, + "loss": 1.9277, + "theoretical_loss": 4.776498780576826, + "tokens_seen": 103743488 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891675025075226, + "loss": 1.9616, + "theoretical_loss": 4.776085202748893, + "tokens_seen": 103809024 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891574724172517, + "loss": 1.9205, + "theoretical_loss": 4.775671958990305, + "tokens_seen": 103874560 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891474423269809, + "loss": 1.9568, + "theoretical_loss": 4.775259048820702, + "tokens_seen": 103940096 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891374122367101, + "loss": 2.0268, + "theoretical_loss": 4.7748464717607195, + "tokens_seen": 104005632 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891273821464393, + "loss": 1.9105, + "theoretical_loss": 4.774434227331979, + "tokens_seen": 104071168 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891173520561686, + "loss": 1.9136, + "theoretical_loss": 4.774022315057092, + "tokens_seen": 104136704 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891073219658976, + "loss": 1.9827, + "theoretical_loss": 4.773610734459654, + "tokens_seen": 104202240 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890972918756269, + "loss": 1.9175, + "theoretical_loss": 4.773199485064243, + "tokens_seen": 104267776 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890872617853561, + "loss": 2.0399, + "theoretical_loss": 4.772788566396418, + "tokens_seen": 104333312 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890772316950853, + "loss": 1.9006, + "theoretical_loss": 4.772377977982714, + "tokens_seen": 104398848 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890672016048145, + "loss": 1.9958, + "theoretical_loss": 4.771967719350641, + "tokens_seen": 104464384 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890571715145437, + "loss": 1.9204, + "theoretical_loss": 4.77155779002868, + "tokens_seen": 104529920 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890471414242728, + "loss": 1.9731, + "theoretical_loss": 4.771148189546282, + "tokens_seen": 104595456 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489037111334002, + "loss": 1.9519, + "theoretical_loss": 4.770738917433864, + "tokens_seen": 104660992 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890270812437312, + "loss": 1.9821, + "theoretical_loss": 4.770329973222809, + "tokens_seen": 104726528 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890170511534604, + "loss": 1.9938, + "theoretical_loss": 4.769921356445458, + "tokens_seen": 104792064 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 41950, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.8759589195251465, + "objective/train/theoretical_loss": 4.769513066635114, + "objective/train/tokens_used": 104857600, + "theoretical_loss": 4.769513066635114, + "tokens_seen": 104857600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890070210631896, + "loss": 1.9428, + "theoretical_loss": 4.769513066635114, + "tokens_seen": 104857600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889969909729187, + "loss": 2.0106, + "theoretical_loss": 4.769105103326036, + "tokens_seen": 104923136 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889869608826479, + "loss": 1.965, + "theoretical_loss": 4.768697466053435, + "tokens_seen": 104988672 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889769307923771, + "loss": 1.9918, + "theoretical_loss": 4.768290154353474, + "tokens_seen": 105054208 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889669007021063, + "loss": 1.9889, + "theoretical_loss": 4.767883167763267, + "tokens_seen": 105119744 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889568706118355, + "loss": 1.9944, + "theoretical_loss": 4.76747650582087, + "tokens_seen": 105185280 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889468405215647, + "loss": 1.9452, + "theoretical_loss": 4.767070168065285, + "tokens_seen": 105250816 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889368104312939, + "loss": 2.1398, + "theoretical_loss": 4.766664154036456, + "tokens_seen": 105316352 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488926780341023, + "loss": 1.9146, + "theoretical_loss": 4.766258463275265, + "tokens_seen": 105381888 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889167502507523, + "loss": 1.918, + "theoretical_loss": 4.7658530953235285, + "tokens_seen": 105447424 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889067201604815, + "loss": 1.9228, + "theoretical_loss": 4.765448049723998, + "tokens_seen": 105512960 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888966900702107, + "loss": 1.9958, + "theoretical_loss": 4.765043326020358, + "tokens_seen": 105578496 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888866599799398, + "loss": 2.0097, + "theoretical_loss": 4.764638923757217, + "tokens_seen": 105644032 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488876629889669, + "loss": 1.8719, + "theoretical_loss": 4.764234842480114, + "tokens_seen": 105709568 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888665997993982, + "loss": 1.9613, + "theoretical_loss": 4.7638310817355105, + "tokens_seen": 105775104 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888565697091274, + "loss": 1.9562, + "theoretical_loss": 4.763427641070789, + "tokens_seen": 105840640 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888465396188566, + "loss": 1.9121, + "theoretical_loss": 4.76302452003425, + "tokens_seen": 105906176 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888365095285858, + "loss": 1.9795, + "theoretical_loss": 4.7626217181751125, + "tokens_seen": 105971712 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888264794383149, + "loss": 2.0448, + "theoretical_loss": 4.762219235043508, + "tokens_seen": 106037248 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888164493480441, + "loss": 1.9872, + "theoretical_loss": 4.761817070190481, + "tokens_seen": 106102784 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888064192577733, + "loss": 1.9033, + "theoretical_loss": 4.761415223167982, + "tokens_seen": 106168320 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887963891675025, + "loss": 2.0154, + "theoretical_loss": 4.761013693528874, + "tokens_seen": 106233856 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887863590772317, + "loss": 2.0962, + "theoretical_loss": 4.760612480826917, + "tokens_seen": 106299392 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488776328986961, + "loss": 1.8693, + "theoretical_loss": 4.76021158461678, + "tokens_seen": 106364928 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048876629889669, + "loss": 1.9897, + "theoretical_loss": 4.759811004454027, + "tokens_seen": 106430464 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887562688064193, + "loss": 2.1109, + "theoretical_loss": 4.759410739895122, + "tokens_seen": 106496000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887462387161484, + "loss": 1.8812, + "theoretical_loss": 4.759010790497422, + "tokens_seen": 106561536 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887362086258777, + "loss": 2.1524, + "theoretical_loss": 4.758611155819178, + "tokens_seen": 106627072 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887261785356069, + "loss": 1.962, + "theoretical_loss": 4.758211835419531, + "tokens_seen": 106692608 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488716148445336, + "loss": 1.967, + "theoretical_loss": 4.757812828858508, + "tokens_seen": 106758144 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887061183550652, + "loss": 1.8189, + "theoretical_loss": 4.757414135697024, + "tokens_seen": 106823680 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886960882647944, + "loss": 1.9784, + "theoretical_loss": 4.757015755496877, + "tokens_seen": 106889216 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886860581745236, + "loss": 1.968, + "theoretical_loss": 4.756617687820743, + "tokens_seen": 106954752 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886760280842528, + "loss": 1.9788, + "theoretical_loss": 4.756219932232181, + "tokens_seen": 107020288 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886659979939819, + "loss": 1.8555, + "theoretical_loss": 4.755822488295621, + "tokens_seen": 107085824 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886559679037111, + "loss": 1.9264, + "theoretical_loss": 4.755425355576373, + "tokens_seen": 107151360 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886459378134403, + "loss": 1.9926, + "theoretical_loss": 4.755028533640614, + "tokens_seen": 107216896 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886359077231695, + "loss": 2.0227, + "theoretical_loss": 4.754632022055392, + "tokens_seen": 107282432 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886258776328988, + "loss": 1.9045, + "theoretical_loss": 4.754235820388624, + "tokens_seen": 107347968 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886158475426278, + "loss": 2.0707, + "theoretical_loss": 4.7538399282090875, + "tokens_seen": 107413504 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886058174523571, + "loss": 1.9044, + "theoretical_loss": 4.753444345086428, + "tokens_seen": 107479040 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885957873620863, + "loss": 1.9009, + "theoretical_loss": 4.753049070591146, + "tokens_seen": 107544576 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885857572718155, + "loss": 1.8984, + "theoretical_loss": 4.752654104294605, + "tokens_seen": 107610112 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885757271815447, + "loss": 1.8771, + "theoretical_loss": 4.752259445769022, + "tokens_seen": 107675648 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885656970912739, + "loss": 1.9543, + "theoretical_loss": 4.751865094587465, + "tokens_seen": 107741184 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488555667001003, + "loss": 1.8868, + "theoretical_loss": 4.75147105032386, + "tokens_seen": 107806720 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885456369107322, + "loss": 1.858, + "theoretical_loss": 4.751077312552976, + "tokens_seen": 107872256 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885356068204614, + "loss": 1.9168, + "theoretical_loss": 4.750683880850433, + "tokens_seen": 107937792 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885255767301906, + "loss": 1.9647, + "theoretical_loss": 4.750290754792694, + "tokens_seen": 108003328 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885155466399198, + "loss": 2.0433, + "theoretical_loss": 4.749897933957065, + "tokens_seen": 108068864 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 42814, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 2.092928886413574, + "objective/train/theoretical_loss": 4.749505417921692, + "objective/train/tokens_used": 108134400, + "theoretical_loss": 4.749505417921692, + "tokens_seen": 108134400 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488505516549649, + "loss": 2.1041, + "theoretical_loss": 4.749505417921692, + "tokens_seen": 108134400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884954864593781, + "loss": 1.8769, + "theoretical_loss": 4.7491132062655605, + "tokens_seen": 108199936 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884854563691073, + "loss": 2.0013, + "theoretical_loss": 4.748721298568491, + "tokens_seen": 108265472 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884754262788365, + "loss": 1.8667, + "theoretical_loss": 4.748329694411137, + "tokens_seen": 108331008 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884653961885657, + "loss": 1.8907, + "theoretical_loss": 4.747938393374987, + "tokens_seen": 108396544 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884553660982949, + "loss": 1.9393, + "theoretical_loss": 4.747547395042355, + "tokens_seen": 108462080 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884453360080241, + "loss": 2.0152, + "theoretical_loss": 4.747156698996388, + "tokens_seen": 108527616 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884353059177532, + "loss": 1.9983, + "theoretical_loss": 4.746766304821053, + "tokens_seen": 108593152 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884252758274825, + "loss": 1.9259, + "theoretical_loss": 4.746376212101142, + "tokens_seen": 108658688 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884152457372117, + "loss": 1.9706, + "theoretical_loss": 4.745986420422267, + "tokens_seen": 108724224 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884052156469409, + "loss": 1.9142, + "theoretical_loss": 4.745596929370863, + "tokens_seen": 108789760 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048839518555667, + "loss": 1.8699, + "theoretical_loss": 4.745207738534177, + "tokens_seen": 108855296 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883851554663992, + "loss": 1.9701, + "theoretical_loss": 4.744818847500274, + "tokens_seen": 108920832 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883751253761284, + "loss": 1.8762, + "theoretical_loss": 4.744430255858029, + "tokens_seen": 108986368 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883650952858576, + "loss": 2.0451, + "theoretical_loss": 4.744041963197132, + "tokens_seen": 109051904 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883550651955868, + "loss": 1.9039, + "theoretical_loss": 4.743653969108076, + "tokens_seen": 109117440 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488345035105316, + "loss": 1.9741, + "theoretical_loss": 4.743266273182163, + "tokens_seen": 109182976 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883350050150451, + "loss": 1.8985, + "theoretical_loss": 4.742878875011501, + "tokens_seen": 109248512 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883249749247743, + "loss": 2.0356, + "theoretical_loss": 4.742491774188997, + "tokens_seen": 109314048 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883149448345035, + "loss": 1.9357, + "theoretical_loss": 4.742104970308359, + "tokens_seen": 109379584 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883049147442327, + "loss": 1.8623, + "theoretical_loss": 4.741718462964095, + "tokens_seen": 109445120 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004882948846539619, + "loss": 2.018, + "theoretical_loss": 4.741332251751509, + "tokens_seen": 109510656 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004882848545636911, + "loss": 1.8953, + "theoretical_loss": 4.740946336266697, + "tokens_seen": 109576192 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004882748244734203, + "loss": 2.01, + "theoretical_loss": 4.740560716106549, + "tokens_seen": 109641728 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048826479438314946, + "loss": 1.9464, + "theoretical_loss": 4.740175390868742, + "tokens_seen": 109707264 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048825476429287864, + "loss": 1.9104, + "theoretical_loss": 4.739790360151744, + "tokens_seen": 109772800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004882447342026078, + "loss": 1.9454, + "theoretical_loss": 4.73940562355481, + "tokens_seen": 109838336 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488234704112337, + "loss": 1.8671, + "theoretical_loss": 4.739021180677976, + "tokens_seen": 109903872 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048822467402206624, + "loss": 1.8892, + "theoretical_loss": 4.73863703112206, + "tokens_seen": 109969408 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048821464393179536, + "loss": 1.859, + "theoretical_loss": 4.738253174488662, + "tokens_seen": 110034944 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004882046138415246, + "loss": 1.9442, + "theoretical_loss": 4.73786961038016, + "tokens_seen": 110100480 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004881945837512537, + "loss": 1.8946, + "theoretical_loss": 4.737486338399704, + "tokens_seen": 110166016 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048818455366098296, + "loss": 1.8926, + "theoretical_loss": 4.737103358151225, + "tokens_seen": 110231552 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048817452357071214, + "loss": 1.8633, + "theoretical_loss": 4.7367206692394195, + "tokens_seen": 110297088 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004881644934804413, + "loss": 1.8816, + "theoretical_loss": 4.736338271269759, + "tokens_seen": 110362624 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004881544633901705, + "loss": 1.9981, + "theoretical_loss": 4.735956163848478, + "tokens_seen": 110428160 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048814443329989974, + "loss": 1.9533, + "theoretical_loss": 4.735574346582582, + "tokens_seen": 110493696 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048813440320962887, + "loss": 1.9083, + "theoretical_loss": 4.735192819079838, + "tokens_seen": 110559232 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004881243731193581, + "loss": 1.9374, + "theoretical_loss": 4.734811580948779, + "tokens_seen": 110624768 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048811434302908723, + "loss": 1.8778, + "theoretical_loss": 4.734430631798692, + "tokens_seen": 110690304 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048810431293881646, + "loss": 1.8312, + "theoretical_loss": 4.734049971239628, + "tokens_seen": 110755840 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048809428284854564, + "loss": 2.0708, + "theoretical_loss": 4.7336695988823925, + "tokens_seen": 110821376 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004880842527582748, + "loss": 2.0025, + "theoretical_loss": 4.733289514338546, + "tokens_seen": 110886912 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488074222668004, + "loss": 1.957, + "theoretical_loss": 4.7329097172204, + "tokens_seen": 110952448 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004880641925777332, + "loss": 2.0032, + "theoretical_loss": 4.73253020714102, + "tokens_seen": 111017984 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048805416248746237, + "loss": 1.936, + "theoretical_loss": 4.732150983714217, + "tokens_seen": 111083520 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004880441323971916, + "loss": 2.0026, + "theoretical_loss": 4.731772046554552, + "tokens_seen": 111149056 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048803410230692073, + "loss": 1.851, + "theoretical_loss": 4.731393395277331, + "tokens_seen": 111214592 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048802407221664997, + "loss": 1.7546, + "theoretical_loss": 4.731015029498598, + "tokens_seen": 111280128 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048801404212637915, + "loss": 1.8877, + "theoretical_loss": 4.730636948835148, + "tokens_seen": 111345664 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 43938, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.90706205368042, + "objective/train/theoretical_loss": 4.730259152904507, + "objective/train/tokens_used": 111411200, + "theoretical_loss": 4.730259152904507, + "tokens_seen": 111411200 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048800401203610833, + "loss": 1.8955, + "theoretical_loss": 4.730259152904507, + "tokens_seen": 111411200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004879939819458375, + "loss": 1.9819, + "theoretical_loss": 4.7298816413249405, + "tokens_seen": 111476736 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004879839518555667, + "loss": 1.9696, + "theoretical_loss": 4.729504413715454, + "tokens_seen": 111542272 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004879739217652959, + "loss": 1.9229, + "theoretical_loss": 4.729127469695783, + "tokens_seen": 111607808 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004879638916750251, + "loss": 1.9245, + "theoretical_loss": 4.728750808886394, + "tokens_seen": 111673344 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004879538615847543, + "loss": 1.8208, + "theoretical_loss": 4.72837443090849, + "tokens_seen": 111738880 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048794383149448347, + "loss": 2.0067, + "theoretical_loss": 4.727998335383996, + "tokens_seen": 111804416 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048793380140421265, + "loss": 1.9672, + "theoretical_loss": 4.727622521935563, + "tokens_seen": 111869952 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048792377131394183, + "loss": 1.9651, + "theoretical_loss": 4.7272469901865755, + "tokens_seen": 111935488 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048791374122367107, + "loss": 2.0028, + "theoretical_loss": 4.726871739761132, + "tokens_seen": 112001024 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004879037111334002, + "loss": 1.8936, + "theoretical_loss": 4.726496770284056, + "tokens_seen": 112066560 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048789368104312943, + "loss": 1.8821, + "theoretical_loss": 4.726122081380891, + "tokens_seen": 112132096 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048788365095285856, + "loss": 2.0407, + "theoretical_loss": 4.725747672677894, + "tokens_seen": 112197632 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004878736208625878, + "loss": 1.9551, + "theoretical_loss": 4.725373543802043, + "tokens_seen": 112263168 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048786359077231697, + "loss": 1.7829, + "theoretical_loss": 4.724999694381027, + "tokens_seen": 112328704 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048785356068204615, + "loss": 1.9288, + "theoretical_loss": 4.724626124043247, + "tokens_seen": 112394240 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048784353059177533, + "loss": 1.9643, + "theoretical_loss": 4.724252832417817, + "tokens_seen": 112459776 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048783350050150457, + "loss": 1.8601, + "theoretical_loss": 4.723879819134558, + "tokens_seen": 112525312 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004878234704112337, + "loss": 1.9831, + "theoretical_loss": 4.7235070838239945, + "tokens_seen": 112590848 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048781344032096293, + "loss": 2.0336, + "theoretical_loss": 4.723134626117363, + "tokens_seen": 112656384 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048780341023069206, + "loss": 1.9725, + "theoretical_loss": 4.722762445646598, + "tokens_seen": 112721920 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004877933801404213, + "loss": 1.8731, + "theoretical_loss": 4.722390542044339, + "tokens_seen": 112787456 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004877833500501505, + "loss": 1.9106, + "theoretical_loss": 4.722018914943924, + "tokens_seen": 112852992 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048777331995987966, + "loss": 1.7549, + "theoretical_loss": 4.721647563979385, + "tokens_seen": 112918528 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048776328986960884, + "loss": 1.787, + "theoretical_loss": 4.72127648878546, + "tokens_seen": 112984064 + }, + { + "epoch": 0.03, + "learning_rate": 0.000487753259779338, + "loss": 1.9026, + "theoretical_loss": 4.720905688997572, + "tokens_seen": 113049600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004877432296890672, + "loss": 1.8825, + "theoretical_loss": 4.720535164251842, + "tokens_seen": 113115136 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048773319959879644, + "loss": 1.8366, + "theoretical_loss": 4.720164914185084, + "tokens_seen": 113180672 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048772316950852556, + "loss": 1.9851, + "theoretical_loss": 4.719794938434794, + "tokens_seen": 113246208 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004877131394182548, + "loss": 1.7881, + "theoretical_loss": 4.7194252366391645, + "tokens_seen": 113311744 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004877031093279839, + "loss": 1.8844, + "theoretical_loss": 4.719055808437068, + "tokens_seen": 113377280 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048769307923771316, + "loss": 1.8634, + "theoretical_loss": 4.718686653468065, + "tokens_seen": 113442816 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048768304914744234, + "loss": 2.088, + "theoretical_loss": 4.718317771372398, + "tokens_seen": 113508352 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004876730190571715, + "loss": 1.9681, + "theoretical_loss": 4.7179491617909886, + "tokens_seen": 113573888 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004876629889669007, + "loss": 1.8803, + "theoretical_loss": 4.717580824365439, + "tokens_seen": 113639424 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048765295887662994, + "loss": 1.8657, + "theoretical_loss": 4.71721275873803, + "tokens_seen": 113704960 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048764292878635907, + "loss": 1.9549, + "theoretical_loss": 4.716844964551717, + "tokens_seen": 113770496 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004876328986960883, + "loss": 1.8277, + "theoretical_loss": 4.716477441450131, + "tokens_seen": 113836032 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048762286860581743, + "loss": 1.8125, + "theoretical_loss": 4.716110189077575, + "tokens_seen": 113901568 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048761283851554666, + "loss": 1.9411, + "theoretical_loss": 4.715743207079022, + "tokens_seen": 113967104 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048760280842527584, + "loss": 1.966, + "theoretical_loss": 4.715376495100118, + "tokens_seen": 114032640 + }, + { + "epoch": 0.03, + "learning_rate": 0.000487592778335005, + "loss": 1.8943, + "theoretical_loss": 4.7150100527871714, + "tokens_seen": 114098176 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004875827482447342, + "loss": 1.8503, + "theoretical_loss": 4.714643879787161, + "tokens_seen": 114163712 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004875727181544634, + "loss": 1.9897, + "theoretical_loss": 4.714277975747729, + "tokens_seen": 114229248 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048756268806419257, + "loss": 2.0346, + "theoretical_loss": 4.7139123403171785, + "tokens_seen": 114294784 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004875526579739218, + "loss": 1.8811, + "theoretical_loss": 4.713546973144476, + "tokens_seen": 114360320 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048754262788365093, + "loss": 1.8568, + "theoretical_loss": 4.713181873879248, + "tokens_seen": 114425856 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048753259779338017, + "loss": 1.9504, + "theoretical_loss": 4.712817042171776, + "tokens_seen": 114491392 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048752256770310935, + "loss": 1.9446, + "theoretical_loss": 4.712452477673001, + "tokens_seen": 114556928 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048751253761283853, + "loss": 1.8835, + "theoretical_loss": 4.712088180034517, + "tokens_seen": 114622464 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 44841, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.8804341554641724, + "objective/train/theoretical_loss": 4.711724148908571, + "objective/train/tokens_used": 114688000, + "theoretical_loss": 4.711724148908571, + "tokens_seen": 114688000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004875025075225677, + "loss": 1.9543, + "theoretical_loss": 4.711724148908571, + "tokens_seen": 114688000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004874924774322969, + "loss": 1.8946, + "theoretical_loss": 4.711360383948064, + "tokens_seen": 114753536 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048748244734202607, + "loss": 2.0906, + "theoretical_loss": 4.710996884806542, + "tokens_seen": 114819072 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004874724172517553, + "loss": 1.9532, + "theoretical_loss": 4.710633651138206, + "tokens_seen": 114884608 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048746238716148443, + "loss": 1.8555, + "theoretical_loss": 4.710270682597898, + "tokens_seen": 114950144 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048745235707121367, + "loss": 1.9252, + "theoretical_loss": 4.709907978841109, + "tokens_seen": 115015680 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004874423269809428, + "loss": 1.9621, + "theoretical_loss": 4.709545539523971, + "tokens_seen": 115081216 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048743229689067203, + "loss": 2.0467, + "theoretical_loss": 4.709183364303258, + "tokens_seen": 115146752 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004874222668004012, + "loss": 2.0153, + "theoretical_loss": 4.708821452836388, + "tokens_seen": 115212288 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004874122367101304, + "loss": 1.9269, + "theoretical_loss": 4.708459804781414, + "tokens_seen": 115277824 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004874022066198596, + "loss": 1.8008, + "theoretical_loss": 4.708098419797031, + "tokens_seen": 115343360 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048739217652958876, + "loss": 1.87, + "theoretical_loss": 4.707737297542563, + "tokens_seen": 115408896 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048738214643931794, + "loss": 1.9335, + "theoretical_loss": 4.707376437677974, + "tokens_seen": 115474432 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048737211634904717, + "loss": 1.8555, + "theoretical_loss": 4.707015839863859, + "tokens_seen": 115539968 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004873620862587763, + "loss": 1.7633, + "theoretical_loss": 4.706655503761443, + "tokens_seen": 115605504 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048735205616850553, + "loss": 1.8769, + "theoretical_loss": 4.706295429032584, + "tokens_seen": 115671040 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004873420260782347, + "loss": 1.8795, + "theoretical_loss": 4.705935615339763, + "tokens_seen": 115736576 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004873319959879639, + "loss": 1.9098, + "theoretical_loss": 4.705576062346094, + "tokens_seen": 115802112 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004873219658976931, + "loss": 1.8635, + "theoretical_loss": 4.70521676971531, + "tokens_seen": 115867648 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048731193580742226, + "loss": 2.046, + "theoretical_loss": 4.704857737111773, + "tokens_seen": 115933184 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048730190571715144, + "loss": 1.9971, + "theoretical_loss": 4.704498964200463, + "tokens_seen": 115998720 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004872918756268807, + "loss": 1.9439, + "theoretical_loss": 4.70414045064698, + "tokens_seen": 116064256 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004872818455366098, + "loss": 1.9388, + "theoretical_loss": 4.703782196117549, + "tokens_seen": 116129792 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048727181544633904, + "loss": 1.8935, + "theoretical_loss": 4.703424200279007, + "tokens_seen": 116195328 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048726178535606816, + "loss": 1.9276, + "theoretical_loss": 4.703066462798809, + "tokens_seen": 116260864 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004872517552657974, + "loss": 1.9142, + "theoretical_loss": 4.7027089833450235, + "tokens_seen": 116326400 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004872417251755266, + "loss": 1.8061, + "theoretical_loss": 4.7023517615863355, + "tokens_seen": 116391936 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048723169508525576, + "loss": 1.9268, + "theoretical_loss": 4.701994797192038, + "tokens_seen": 116457472 + }, + { + "epoch": 0.04, + "learning_rate": 0.000487221664994985, + "loss": 2.0122, + "theoretical_loss": 4.701638089832034, + "tokens_seen": 116523008 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004872116349047141, + "loss": 1.9872, + "theoretical_loss": 4.701281639176839, + "tokens_seen": 116588544 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048720160481444336, + "loss": 1.8214, + "theoretical_loss": 4.700925444897575, + "tokens_seen": 116654080 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048719157472417254, + "loss": 1.9365, + "theoretical_loss": 4.700569506665965, + "tokens_seen": 116719616 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004871815446339017, + "loss": 2.0266, + "theoretical_loss": 4.700213824154341, + "tokens_seen": 116785152 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004871715145436309, + "loss": 1.9013, + "theoretical_loss": 4.699858397035637, + "tokens_seen": 116850688 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048716148445336014, + "loss": 1.9698, + "theoretical_loss": 4.699503224983388, + "tokens_seen": 116916224 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048715145436308927, + "loss": 1.9625, + "theoretical_loss": 4.699148307671729, + "tokens_seen": 116981760 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004871414242728185, + "loss": 1.9075, + "theoretical_loss": 4.698793644775394, + "tokens_seen": 117047296 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048713139418254763, + "loss": 1.8326, + "theoretical_loss": 4.698439235969713, + "tokens_seen": 117112832 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048712136409227686, + "loss": 1.9807, + "theoretical_loss": 4.698085080930612, + "tokens_seen": 117178368 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048711133400200604, + "loss": 1.8354, + "theoretical_loss": 4.697731179334614, + "tokens_seen": 117243904 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004871013039117352, + "loss": 1.9865, + "theoretical_loss": 4.69737753085883, + "tokens_seen": 117309440 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004870912738214644, + "loss": 1.8534, + "theoretical_loss": 4.697024135180968, + "tokens_seen": 117374976 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004870812437311936, + "loss": 1.9062, + "theoretical_loss": 4.696670991979321, + "tokens_seen": 117440512 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048707121364092277, + "loss": 1.8709, + "theoretical_loss": 4.696318100932773, + "tokens_seen": 117506048 + }, + { + "epoch": 0.04, + "learning_rate": 0.000487061183550652, + "loss": 1.8007, + "theoretical_loss": 4.695965461720796, + "tokens_seen": 117571584 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048705115346038113, + "loss": 1.9068, + "theoretical_loss": 4.695613074023446, + "tokens_seen": 117637120 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048704112337011037, + "loss": 1.8793, + "theoretical_loss": 4.695260937521364, + "tokens_seen": 117702656 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048703109327983955, + "loss": 1.9024, + "theoretical_loss": 4.694909051895775, + "tokens_seen": 117768192 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048702106318956873, + "loss": 1.8775, + "theoretical_loss": 4.694557416828484, + "tokens_seen": 117833728 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004870110330992979, + "loss": 1.8504, + "theoretical_loss": 4.694206032001879, + "tokens_seen": 117899264 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 46098, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.7792043685913086, + "objective/train/theoretical_loss": 4.693854897098923, + "objective/train/tokens_used": 117964800, + "theoretical_loss": 4.693854897098923, + "tokens_seen": 117964800 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004870010030090271, + "loss": 1.7758, + "theoretical_loss": 4.693854897098923, + "tokens_seen": 117964800 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048699097291875627, + "loss": 1.8839, + "theoretical_loss": 4.693504011803159, + "tokens_seen": 118030336 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004869809428284855, + "loss": 1.9303, + "theoretical_loss": 4.6931533757987065, + "tokens_seen": 118095872 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048697091273821463, + "loss": 1.8405, + "theoretical_loss": 4.692802988770258, + "tokens_seen": 118161408 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048696088264794387, + "loss": 1.8896, + "theoretical_loss": 4.692452850403081, + "tokens_seen": 118226944 + }, + { + "epoch": 0.04, + "learning_rate": 0.000486950852557673, + "loss": 1.8411, + "theoretical_loss": 4.692102960383014, + "tokens_seen": 118292480 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048694082246740223, + "loss": 1.8538, + "theoretical_loss": 4.691753318396467, + "tokens_seen": 118358016 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004869307923771314, + "loss": 1.816, + "theoretical_loss": 4.691403924130418, + "tokens_seen": 118423552 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004869207622868606, + "loss": 1.8614, + "theoretical_loss": 4.691054777272415, + "tokens_seen": 118489088 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004869107321965898, + "loss": 1.9348, + "theoretical_loss": 4.69070587751057, + "tokens_seen": 118554624 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048690070210631896, + "loss": 1.9431, + "theoretical_loss": 4.690357224533564, + "tokens_seen": 118620160 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048689067201604814, + "loss": 1.9338, + "theoretical_loss": 4.690008818030638, + "tokens_seen": 118685696 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048688064192577737, + "loss": 1.9841, + "theoretical_loss": 4.689660657691598, + "tokens_seen": 118751232 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004868706118355065, + "loss": 1.8464, + "theoretical_loss": 4.689312743206811, + "tokens_seen": 118816768 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048686058174523573, + "loss": 1.792, + "theoretical_loss": 4.688965074267203, + "tokens_seen": 118882304 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004868505516549649, + "loss": 1.8858, + "theoretical_loss": 4.688617650564259, + "tokens_seen": 118947840 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004868405215646941, + "loss": 1.8135, + "theoretical_loss": 4.688270471790023, + "tokens_seen": 119013376 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004868304914744233, + "loss": 1.8696, + "theoretical_loss": 4.687923537637094, + "tokens_seen": 119078912 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048682046138415246, + "loss": 1.7356, + "theoretical_loss": 4.687576847798622, + "tokens_seen": 119144448 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048681043129388164, + "loss": 1.8803, + "theoretical_loss": 4.687230401968319, + "tokens_seen": 119209984 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004868004012036109, + "loss": 1.7796, + "theoretical_loss": 4.68688419984044, + "tokens_seen": 119275520 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048679037111334, + "loss": 1.8157, + "theoretical_loss": 4.686538241109796, + "tokens_seen": 119341056 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048678034102306924, + "loss": 2.0127, + "theoretical_loss": 4.686192525471746, + "tokens_seen": 119406592 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048677031093279836, + "loss": 1.8969, + "theoretical_loss": 4.685847052622199, + "tokens_seen": 119472128 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004867602808425276, + "loss": 1.9661, + "theoretical_loss": 4.685501822257608, + "tokens_seen": 119537664 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004867502507522568, + "loss": 1.8319, + "theoretical_loss": 4.685156834074972, + "tokens_seen": 119603200 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048674022066198596, + "loss": 1.8965, + "theoretical_loss": 4.684812087771839, + "tokens_seen": 119668736 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048673019057171514, + "loss": 1.897, + "theoretical_loss": 4.684467583046292, + "tokens_seen": 119734272 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004867201604814443, + "loss": 1.7594, + "theoretical_loss": 4.6841233195969645, + "tokens_seen": 119799808 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004867101303911735, + "loss": 1.9229, + "theoretical_loss": 4.683779297123024, + "tokens_seen": 119865344 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048670010030090274, + "loss": 1.9224, + "theoretical_loss": 4.683435515324181, + "tokens_seen": 119930880 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048669007021063187, + "loss": 1.9772, + "theoretical_loss": 4.683091973900682, + "tokens_seen": 119996416 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004866800401203611, + "loss": 1.8613, + "theoretical_loss": 4.682748672553313, + "tokens_seen": 120061952 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004866700100300903, + "loss": 1.9351, + "theoretical_loss": 4.68240561098339, + "tokens_seen": 120127488 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048665997993981947, + "loss": 1.9139, + "theoretical_loss": 4.68206278889277, + "tokens_seen": 120193024 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048664994984954865, + "loss": 1.7851, + "theoretical_loss": 4.681720205983839, + "tokens_seen": 120258560 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048663991975927783, + "loss": 1.902, + "theoretical_loss": 4.681377861959516, + "tokens_seen": 120324096 + }, + { + "epoch": 0.04, + "learning_rate": 0.000486629889669007, + "loss": 1.9159, + "theoretical_loss": 4.681035756523249, + "tokens_seen": 120389632 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048661985957873624, + "loss": 1.8434, + "theoretical_loss": 4.680693889379018, + "tokens_seen": 120455168 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048660982948846537, + "loss": 1.9042, + "theoretical_loss": 4.680352260231329, + "tokens_seen": 120520704 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004865997993981946, + "loss": 1.9569, + "theoretical_loss": 4.680010868785216, + "tokens_seen": 120586240 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048658976930792373, + "loss": 1.7838, + "theoretical_loss": 4.679669714746238, + "tokens_seen": 120651776 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048657973921765297, + "loss": 1.7172, + "theoretical_loss": 4.679328797820478, + "tokens_seen": 120717312 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048656970912738215, + "loss": 1.9147, + "theoretical_loss": 4.678988117714544, + "tokens_seen": 120782848 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048655967903711133, + "loss": 1.869, + "theoretical_loss": 4.678647674135563, + "tokens_seen": 120848384 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004865496489468405, + "loss": 1.8656, + "theoretical_loss": 4.6783074667911855, + "tokens_seen": 120913920 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048653961885656975, + "loss": 1.9468, + "theoretical_loss": 4.677967495389581, + "tokens_seen": 120979456 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004865295887662989, + "loss": 2.0255, + "theoretical_loss": 4.677627759639435, + "tokens_seen": 121044992 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004865195586760281, + "loss": 1.8835, + "theoretical_loss": 4.677288259249954, + "tokens_seen": 121110528 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048650952858575724, + "loss": 1.9723, + "theoretical_loss": 4.676948993930857, + "tokens_seen": 121176064 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 47011, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.9057981967926025, + "objective/train/theoretical_loss": 4.67660996339238, + "objective/train/tokens_used": 121241600, + "theoretical_loss": 4.67660996339238, + "tokens_seen": 121241600 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048649949849548647, + "loss": 1.9599, + "theoretical_loss": 4.67660996339238, + "tokens_seen": 121241600 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048648946840521565, + "loss": 1.774, + "theoretical_loss": 4.676271167345272, + "tokens_seen": 121307136 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048647943831494483, + "loss": 1.8502, + "theoretical_loss": 4.675932605500794, + "tokens_seen": 121372672 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048646940822467407, + "loss": 1.8388, + "theoretical_loss": 4.675594277570718, + "tokens_seen": 121438208 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004864593781344032, + "loss": 1.8774, + "theoretical_loss": 4.675256183267327, + "tokens_seen": 121503744 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048644934804413243, + "loss": 1.8869, + "theoretical_loss": 4.674918322303411, + "tokens_seen": 121569280 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004864393179538616, + "loss": 1.7329, + "theoretical_loss": 4.6745806943922705, + "tokens_seen": 121634816 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004864292878635908, + "loss": 1.8885, + "theoretical_loss": 4.67424329924771, + "tokens_seen": 121700352 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048641925777332, + "loss": 1.9476, + "theoretical_loss": 4.673906136584041, + "tokens_seen": 121765888 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048640922768304916, + "loss": 1.8454, + "theoretical_loss": 4.673569206116078, + "tokens_seen": 121831424 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048639919759277834, + "loss": 1.7973, + "theoretical_loss": 4.6732325075591366, + "tokens_seen": 121896960 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048638916750250757, + "loss": 1.9334, + "theoretical_loss": 4.67289604062904, + "tokens_seen": 121962496 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004863791374122367, + "loss": 1.8476, + "theoretical_loss": 4.672559805042107, + "tokens_seen": 122028032 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048636910732196593, + "loss": 1.9438, + "theoretical_loss": 4.672223800515159, + "tokens_seen": 122093568 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004863590772316951, + "loss": 1.8246, + "theoretical_loss": 4.671888026765512, + "tokens_seen": 122159104 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004863490471414243, + "loss": 1.8628, + "theoretical_loss": 4.6715524835109825, + "tokens_seen": 122224640 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004863390170511535, + "loss": 1.9056, + "theoretical_loss": 4.671217170469884, + "tokens_seen": 122290176 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048632898696088266, + "loss": 2.0235, + "theoretical_loss": 4.67088208736102, + "tokens_seen": 122355712 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048631895687061184, + "loss": 1.877, + "theoretical_loss": 4.670547233903694, + "tokens_seen": 122421248 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004863089267803411, + "loss": 1.9816, + "theoretical_loss": 4.670212609817698, + "tokens_seen": 122486784 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004862988966900702, + "loss": 1.7887, + "theoretical_loss": 4.669878214823316, + "tokens_seen": 122552320 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048628886659979944, + "loss": 1.9247, + "theoretical_loss": 4.669544048641325, + "tokens_seen": 122617856 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048627883650952857, + "loss": 1.9674, + "theoretical_loss": 4.669210110992989, + "tokens_seen": 122683392 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004862688064192578, + "loss": 1.9629, + "theoretical_loss": 4.668876401600059, + "tokens_seen": 122748928 + }, + { + "epoch": 0.04, + "learning_rate": 0.000486258776328987, + "loss": 1.8885, + "theoretical_loss": 4.668542920184779, + "tokens_seen": 122814464 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048624874623871616, + "loss": 1.951, + "theoretical_loss": 4.6682096664698705, + "tokens_seen": 122880000 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048623871614844534, + "loss": 1.8017, + "theoretical_loss": 4.667876640178546, + "tokens_seen": 122945536 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004862286860581745, + "loss": 1.8878, + "theoretical_loss": 4.667543841034502, + "tokens_seen": 123011072 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004862186559679037, + "loss": 1.8555, + "theoretical_loss": 4.667211268761914, + "tokens_seen": 123076608 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048620862587763294, + "loss": 1.9108, + "theoretical_loss": 4.66687892308544, + "tokens_seen": 123142144 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048619859578736207, + "loss": 1.9065, + "theoretical_loss": 4.6665468037302205, + "tokens_seen": 123207680 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004861885656970913, + "loss": 1.8606, + "theoretical_loss": 4.666214910421875, + "tokens_seen": 123273216 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004861785356068205, + "loss": 1.8728, + "theoretical_loss": 4.665883242886498, + "tokens_seen": 123338752 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048616850551654967, + "loss": 1.861, + "theoretical_loss": 4.6655518008506665, + "tokens_seen": 123404288 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048615847542627885, + "loss": 1.8837, + "theoretical_loss": 4.665220584041428, + "tokens_seen": 123469824 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048614844533600803, + "loss": 1.9957, + "theoretical_loss": 4.664889592186309, + "tokens_seen": 123535360 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004861384152457372, + "loss": 1.8113, + "theoretical_loss": 4.6645588250133105, + "tokens_seen": 123600896 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048612838515546644, + "loss": 1.8664, + "theoretical_loss": 4.664228282250902, + "tokens_seen": 123666432 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048611835506519557, + "loss": 2.0096, + "theoretical_loss": 4.663897963628029, + "tokens_seen": 123731968 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004861083249749248, + "loss": 1.8198, + "theoretical_loss": 4.6635678688741065, + "tokens_seen": 123797504 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048609829488465393, + "loss": 1.8698, + "theoretical_loss": 4.663237997719017, + "tokens_seen": 123863040 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048608826479438317, + "loss": 1.7714, + "theoretical_loss": 4.662908349893116, + "tokens_seen": 123928576 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048607823470411235, + "loss": 1.892, + "theoretical_loss": 4.662578925127223, + "tokens_seen": 123994112 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048606820461384153, + "loss": 1.8121, + "theoretical_loss": 4.662249723152624, + "tokens_seen": 124059648 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004860581745235707, + "loss": 1.8234, + "theoretical_loss": 4.661920743701073, + "tokens_seen": 124125184 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048604814443329995, + "loss": 1.8948, + "theoretical_loss": 4.661591986504786, + "tokens_seen": 124190720 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004860381143430291, + "loss": 1.907, + "theoretical_loss": 4.661263451296444, + "tokens_seen": 124256256 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004860280842527583, + "loss": 1.9109, + "theoretical_loss": 4.660935137809188, + "tokens_seen": 124321792 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048601805416248744, + "loss": 2.0244, + "theoretical_loss": 4.660607045776624, + "tokens_seen": 124387328 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048600802407221667, + "loss": 1.8107, + "theoretical_loss": 4.660279174932814, + "tokens_seen": 124452864 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 48145, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6503289937973022, + "objective/train/theoretical_loss": 4.659951525012284, + "objective/train/tokens_used": 124518400, + "theoretical_loss": 4.659951525012284, + "tokens_seen": 124518400 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048599799398194585, + "loss": 1.7469, + "theoretical_loss": 4.659951525012284, + "tokens_seen": 124518400 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048598796389167503, + "loss": 1.8561, + "theoretical_loss": 4.6596240957500115, + "tokens_seen": 124583936 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004859779338014042, + "loss": 1.7976, + "theoretical_loss": 4.659296886881438, + "tokens_seen": 124649472 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004859679037111334, + "loss": 1.8283, + "theoretical_loss": 4.658969898142456, + "tokens_seen": 124715008 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004859578736208626, + "loss": 1.7982, + "theoretical_loss": 4.658643129269416, + "tokens_seen": 124780544 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004859478435305918, + "loss": 1.8106, + "theoretical_loss": 4.658316579999122, + "tokens_seen": 124846080 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048593781344032094, + "loss": 1.915, + "theoretical_loss": 4.65799025006883, + "tokens_seen": 124911616 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004859277833500502, + "loss": 1.8415, + "theoretical_loss": 4.65766413921625, + "tokens_seen": 124977152 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004859177532597793, + "loss": 1.755, + "theoretical_loss": 4.657338247179538, + "tokens_seen": 125042688 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048590772316950854, + "loss": 1.878, + "theoretical_loss": 4.657012573697307, + "tokens_seen": 125108224 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858976930792377, + "loss": 1.8609, + "theoretical_loss": 4.656687118508616, + "tokens_seen": 125173760 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858876629889669, + "loss": 1.9354, + "theoretical_loss": 4.656361881352968, + "tokens_seen": 125239296 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858776328986961, + "loss": 1.8449, + "theoretical_loss": 4.656036861970319, + "tokens_seen": 125304832 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858676028084253, + "loss": 1.8259, + "theoretical_loss": 4.655712060101068, + "tokens_seen": 125370368 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048585757271815444, + "loss": 1.9039, + "theoretical_loss": 4.65538747548606, + "tokens_seen": 125435904 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858475426278837, + "loss": 1.8668, + "theoretical_loss": 4.655063107866582, + "tokens_seen": 125501440 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858375125376128, + "loss": 1.8305, + "theoretical_loss": 4.654738956984366, + "tokens_seen": 125566976 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048582748244734204, + "loss": 1.9072, + "theoretical_loss": 4.654415022581585, + "tokens_seen": 125632512 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858174523570712, + "loss": 1.9057, + "theoretical_loss": 4.654091304400853, + "tokens_seen": 125698048 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858074222668004, + "loss": 1.8446, + "theoretical_loss": 4.653767802185225, + "tokens_seen": 125763584 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004857973921765296, + "loss": 1.8911, + "theoretical_loss": 4.653444515678195, + "tokens_seen": 125829120 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048578736208625877, + "loss": 1.8582, + "theoretical_loss": 4.653121444623693, + "tokens_seen": 125894656 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048577733199598795, + "loss": 1.8565, + "theoretical_loss": 4.652798588766088, + "tokens_seen": 125960192 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004857673019057172, + "loss": 1.8189, + "theoretical_loss": 4.652475947850185, + "tokens_seen": 126025728 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004857572718154463, + "loss": 1.8391, + "theoretical_loss": 4.652153521621223, + "tokens_seen": 126091264 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048574724172517554, + "loss": 1.8113, + "theoretical_loss": 4.651831309824875, + "tokens_seen": 126156800 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048573721163490467, + "loss": 1.7873, + "theoretical_loss": 4.65150931220725, + "tokens_seen": 126222336 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004857271815446339, + "loss": 1.8426, + "theoretical_loss": 4.651187528514887, + "tokens_seen": 126287872 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048571715145436314, + "loss": 1.8425, + "theoretical_loss": 4.650865958494755, + "tokens_seen": 126353408 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048570712136409227, + "loss": 1.8188, + "theoretical_loss": 4.650544601894257, + "tokens_seen": 126418944 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004856970912738215, + "loss": 1.9125, + "theoretical_loss": 4.650223458461221, + "tokens_seen": 126484480 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004856870611835507, + "loss": 1.8314, + "theoretical_loss": 4.649902527943908, + "tokens_seen": 126550016 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048567703109327987, + "loss": 1.8064, + "theoretical_loss": 4.649581810091003, + "tokens_seen": 126615552 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048566700100300905, + "loss": 1.8984, + "theoretical_loss": 4.649261304651619, + "tokens_seen": 126681088 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048565697091273823, + "loss": 1.7619, + "theoretical_loss": 4.6489410113752925, + "tokens_seen": 126746624 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004856469408224674, + "loss": 1.8714, + "theoretical_loss": 4.6486209300119885, + "tokens_seen": 126812160 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048563691073219664, + "loss": 1.9425, + "theoretical_loss": 4.648301060312093, + "tokens_seen": 126877696 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048562688064192577, + "loss": 1.9184, + "theoretical_loss": 4.647981402026417, + "tokens_seen": 126943232 + }, + { + "epoch": 0.04, + "learning_rate": 0.000485616850551655, + "loss": 1.8774, + "theoretical_loss": 4.647661954906188, + "tokens_seen": 127008768 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048560682046138413, + "loss": 2.0485, + "theoretical_loss": 4.647342718703062, + "tokens_seen": 127074304 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048559679037111337, + "loss": 1.8451, + "theoretical_loss": 4.647023693169107, + "tokens_seen": 127139840 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048558676028084255, + "loss": 1.8617, + "theoretical_loss": 4.646704878056816, + "tokens_seen": 127205376 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048557673019057173, + "loss": 1.8733, + "theoretical_loss": 4.6463862731191, + "tokens_seen": 127270912 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004855667001003009, + "loss": 1.8804, + "theoretical_loss": 4.6460678781092835, + "tokens_seen": 127336448 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048555667001003015, + "loss": 1.9336, + "theoretical_loss": 4.6457496927811075, + "tokens_seen": 127401984 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004855466399197593, + "loss": 1.8442, + "theoretical_loss": 4.645431716888734, + "tokens_seen": 127467520 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004855366098294885, + "loss": 1.8104, + "theoretical_loss": 4.645113950186732, + "tokens_seen": 127533056 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048552657973921764, + "loss": 1.829, + "theoretical_loss": 4.64479639243009, + "tokens_seen": 127598592 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048551654964894687, + "loss": 1.8395, + "theoretical_loss": 4.644479043374204, + "tokens_seen": 127664128 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048550651955867605, + "loss": 1.9527, + "theoretical_loss": 4.644161902774886, + "tokens_seen": 127729664 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 49387, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.732029676437378, + "objective/train/theoretical_loss": 4.643844970388358, + "objective/train/tokens_used": 127795200, + "theoretical_loss": 4.643844970388358, + "tokens_seen": 127795200 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048549648946840523, + "loss": 1.8169, + "theoretical_loss": 4.643844970388358, + "tokens_seen": 127795200 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004854864593781344, + "loss": 1.893, + "theoretical_loss": 4.6435282459712495, + "tokens_seen": 127860736 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004854764292878636, + "loss": 1.7738, + "theoretical_loss": 4.643211729280601, + "tokens_seen": 127926272 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004854663991975928, + "loss": 1.9495, + "theoretical_loss": 4.64289542007386, + "tokens_seen": 127991808 + }, + { + "epoch": 0.04, + "learning_rate": 0.000485456369107322, + "loss": 1.8342, + "theoretical_loss": 4.642579318108883, + "tokens_seen": 128057344 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048544633901705114, + "loss": 1.8126, + "theoretical_loss": 4.642263423143932, + "tokens_seen": 128122880 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004854363089267804, + "loss": 1.9422, + "theoretical_loss": 4.641947734937673, + "tokens_seen": 128188416 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004854262788365095, + "loss": 1.9898, + "theoretical_loss": 4.641632253249179, + "tokens_seen": 128253952 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048541624874623874, + "loss": 1.782, + "theoretical_loss": 4.641316977837924, + "tokens_seen": 128319488 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004854062186559679, + "loss": 2.0229, + "theoretical_loss": 4.641001908463787, + "tokens_seen": 128385024 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004853961885656971, + "loss": 1.9357, + "theoretical_loss": 4.6406870448870485, + "tokens_seen": 128450560 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004853861584754263, + "loss": 1.7598, + "theoretical_loss": 4.64037238686839, + "tokens_seen": 128516096 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004853761283851555, + "loss": 1.9005, + "theoretical_loss": 4.640057934168891, + "tokens_seen": 128581632 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048536609829488464, + "loss": 1.8687, + "theoretical_loss": 4.639743686550034, + "tokens_seen": 128647168 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004853560682046139, + "loss": 1.8091, + "theoretical_loss": 4.639429643773697, + "tokens_seen": 128712704 + }, + { + "epoch": 0.04, + "learning_rate": 0.000485346038114343, + "loss": 1.9127, + "theoretical_loss": 4.639115805602156, + "tokens_seen": 128778240 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048533600802407224, + "loss": 1.9228, + "theoretical_loss": 4.638802171798086, + "tokens_seen": 128843776 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004853259779338014, + "loss": 1.9446, + "theoretical_loss": 4.638488742124554, + "tokens_seen": 128909312 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004853159478435306, + "loss": 1.9726, + "theoretical_loss": 4.638175516345027, + "tokens_seen": 128974848 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004853059177532598, + "loss": 1.87, + "theoretical_loss": 4.637862494223359, + "tokens_seen": 129040384 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048529588766298897, + "loss": 1.9156, + "theoretical_loss": 4.637549675523806, + "tokens_seen": 129105920 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048528585757271815, + "loss": 1.8291, + "theoretical_loss": 4.637237060011007, + "tokens_seen": 129171456 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004852758274824474, + "loss": 1.8545, + "theoretical_loss": 4.636924647450002, + "tokens_seen": 129236992 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004852657973921765, + "loss": 1.9444, + "theoretical_loss": 4.636612437606212, + "tokens_seen": 129302528 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048525576730190574, + "loss": 1.8955, + "theoretical_loss": 4.636300430245457, + "tokens_seen": 129368064 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048524573721163487, + "loss": 1.7892, + "theoretical_loss": 4.63598862513394, + "tokens_seen": 129433600 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004852357071213641, + "loss": 1.908, + "theoretical_loss": 4.635677022038254, + "tokens_seen": 129499136 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004852256770310933, + "loss": 1.7611, + "theoretical_loss": 4.63536562072538, + "tokens_seen": 129564672 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048521564694082247, + "loss": 1.9965, + "theoretical_loss": 4.635054420962684, + "tokens_seen": 129630208 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048520561685055165, + "loss": 1.8529, + "theoretical_loss": 4.634743422517918, + "tokens_seen": 129695744 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004851955867602809, + "loss": 1.9024, + "theoretical_loss": 4.634432625159221, + "tokens_seen": 129761280 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048518555667001, + "loss": 1.8454, + "theoretical_loss": 4.634122028655112, + "tokens_seen": 129826816 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048517552657973925, + "loss": 1.8879, + "theoretical_loss": 4.633811632774496, + "tokens_seen": 129892352 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004851654964894684, + "loss": 1.8318, + "theoretical_loss": 4.633501437286659, + "tokens_seen": 129957888 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004851554663991976, + "loss": 1.8949, + "theoretical_loss": 4.633191441961272, + "tokens_seen": 130023424 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004851454363089268, + "loss": 1.9055, + "theoretical_loss": 4.63288164656838, + "tokens_seen": 130088960 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048513540621865597, + "loss": 1.8505, + "theoretical_loss": 4.632572050878415, + "tokens_seen": 130154496 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048512537612838515, + "loss": 1.9176, + "theoretical_loss": 4.632262654662181, + "tokens_seen": 130220032 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048511534603811433, + "loss": 1.8919, + "theoretical_loss": 4.631953457690868, + "tokens_seen": 130285568 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004851053159478435, + "loss": 1.9861, + "theoretical_loss": 4.631644459736036, + "tokens_seen": 130351104 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048509528585757275, + "loss": 1.8898, + "theoretical_loss": 4.631335660569626, + "tokens_seen": 130416640 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004850852557673019, + "loss": 1.7627, + "theoretical_loss": 4.631027059963953, + "tokens_seen": 130482176 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004850752256770311, + "loss": 1.9026, + "theoretical_loss": 4.630718657691709, + "tokens_seen": 130547712 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048506519558676024, + "loss": 1.7209, + "theoretical_loss": 4.630410453525958, + "tokens_seen": 130613248 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004850551654964895, + "loss": 1.9743, + "theoretical_loss": 4.630102447240138, + "tokens_seen": 130678784 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048504513540621866, + "loss": 1.8388, + "theoretical_loss": 4.629794638608059, + "tokens_seen": 130744320 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048503510531594784, + "loss": 1.9026, + "theoretical_loss": 4.629487027403905, + "tokens_seen": 130809856 + }, + { + "epoch": 0.04, + "learning_rate": 0.000485025075225677, + "loss": 1.82, + "theoretical_loss": 4.629179613402228, + "tokens_seen": 130875392 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048501504513540625, + "loss": 1.8254, + "theoretical_loss": 4.628872396377953, + "tokens_seen": 130940928 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004850050150451354, + "loss": 1.8231, + "theoretical_loss": 4.628565376106371, + "tokens_seen": 131006464 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 50313, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.8566858768463135, + "objective/train/theoretical_loss": 4.628258552363146, + "objective/train/tokens_used": 131072000, + "theoretical_loss": 4.628258552363146, + "tokens_seen": 131072000 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004849949849548646, + "loss": 1.9334, + "theoretical_loss": 4.628258552363146, + "tokens_seen": 131072000 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004849849548645938, + "loss": 1.8919, + "theoretical_loss": 4.627951924924305, + "tokens_seen": 131137536 + }, + { + "epoch": 0.04, + "learning_rate": 0.000484974924774323, + "loss": 1.8538, + "theoretical_loss": 4.627645493566245, + "tokens_seen": 131203072 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004849648946840522, + "loss": 1.8933, + "theoretical_loss": 4.62733925806573, + "tokens_seen": 131268608 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048495486459378134, + "loss": 1.7001, + "theoretical_loss": 4.627033218199886, + "tokens_seen": 131334144 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004849448345035106, + "loss": 1.9711, + "theoretical_loss": 4.626727373746206, + "tokens_seen": 131399680 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004849348044132397, + "loss": 1.7544, + "theoretical_loss": 4.626421724482549, + "tokens_seen": 131465216 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048492477432296894, + "loss": 1.845, + "theoretical_loss": 4.626116270187131, + "tokens_seen": 131530752 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004849147442326981, + "loss": 1.8163, + "theoretical_loss": 4.625811010638538, + "tokens_seen": 131596288 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004849047141424273, + "loss": 1.7738, + "theoretical_loss": 4.62550594561571, + "tokens_seen": 131661824 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004848946840521565, + "loss": 1.7563, + "theoretical_loss": 4.625201074897952, + "tokens_seen": 131727360 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004848846539618857, + "loss": 1.8609, + "theoretical_loss": 4.62489639826493, + "tokens_seen": 131792896 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048487462387161484, + "loss": 1.9311, + "theoretical_loss": 4.6245919154966675, + "tokens_seen": 131858432 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004848645937813441, + "loss": 1.8142, + "theoretical_loss": 4.624287626373545, + "tokens_seen": 131923968 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004848545636910732, + "loss": 1.7462, + "theoretical_loss": 4.623983530676304, + "tokens_seen": 131989504 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048484453360080244, + "loss": 1.9072, + "theoretical_loss": 4.62367962818604, + "tokens_seen": 132055040 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004848345035105316, + "loss": 1.9488, + "theoretical_loss": 4.623375918684207, + "tokens_seen": 132120576 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004848244734202608, + "loss": 1.8458, + "theoretical_loss": 4.623072401952614, + "tokens_seen": 132186112 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048481444332999, + "loss": 1.9467, + "theoretical_loss": 4.622769077773423, + "tokens_seen": 132251648 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048480441323971917, + "loss": 1.9307, + "theoretical_loss": 4.622465945929152, + "tokens_seen": 132317184 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048479438314944835, + "loss": 1.8768, + "theoretical_loss": 4.622163006202671, + "tokens_seen": 132382720 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004847843530591776, + "loss": 1.8229, + "theoretical_loss": 4.621860258377203, + "tokens_seen": 132448256 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004847743229689067, + "loss": 1.9028, + "theoretical_loss": 4.621557702236323, + "tokens_seen": 132513792 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048476429287863594, + "loss": 1.7745, + "theoretical_loss": 4.621255337563955, + "tokens_seen": 132579328 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048475426278836507, + "loss": 1.7841, + "theoretical_loss": 4.620953164144376, + "tokens_seen": 132644864 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004847442326980943, + "loss": 1.9072, + "theoretical_loss": 4.620651181762212, + "tokens_seen": 132710400 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004847342026078235, + "loss": 1.8911, + "theoretical_loss": 4.620349390202436, + "tokens_seen": 132775936 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048472417251755267, + "loss": 1.8958, + "theoretical_loss": 4.6200477892503695, + "tokens_seen": 132841472 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048471414242728185, + "loss": 1.8495, + "theoretical_loss": 4.619746378691682, + "tokens_seen": 132907008 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004847041123370111, + "loss": 1.833, + "theoretical_loss": 4.61944515831239, + "tokens_seen": 132972544 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004846940822467402, + "loss": 1.8433, + "theoretical_loss": 4.6191441278988545, + "tokens_seen": 133038080 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048468405215646945, + "loss": 1.882, + "theoretical_loss": 4.618843287237782, + "tokens_seen": 133103616 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004846740220661986, + "loss": 1.9257, + "theoretical_loss": 4.618542636116224, + "tokens_seen": 133169152 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004846639919759278, + "loss": 1.9284, + "theoretical_loss": 4.618242174321575, + "tokens_seen": 133234688 + }, + { + "epoch": 0.04, + "learning_rate": 0.000484653961885657, + "loss": 1.821, + "theoretical_loss": 4.617941901641573, + "tokens_seen": 133300224 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048464393179538617, + "loss": 1.92, + "theoretical_loss": 4.617641817864296, + "tokens_seen": 133365760 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048463390170511535, + "loss": 1.8381, + "theoretical_loss": 4.617341922778168, + "tokens_seen": 133431296 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048462387161484453, + "loss": 1.8591, + "theoretical_loss": 4.617042216171949, + "tokens_seen": 133496832 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004846138415245737, + "loss": 1.8443, + "theoretical_loss": 4.616742697834742, + "tokens_seen": 133562368 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048460381143430295, + "loss": 1.7957, + "theoretical_loss": 4.616443367555988, + "tokens_seen": 133627904 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004845937813440321, + "loss": 1.8666, + "theoretical_loss": 4.616144225125465, + "tokens_seen": 133693440 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004845837512537613, + "loss": 1.7907, + "theoretical_loss": 4.615845270333294, + "tokens_seen": 133758976 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048457372116349044, + "loss": 1.9184, + "theoretical_loss": 4.615546502969929, + "tokens_seen": 133824512 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004845636910732197, + "loss": 1.9745, + "theoretical_loss": 4.61524792282616, + "tokens_seen": 133890048 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048455366098294886, + "loss": 1.7039, + "theoretical_loss": 4.614949529693115, + "tokens_seen": 133955584 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048454363089267804, + "loss": 1.7382, + "theoretical_loss": 4.614651323362257, + "tokens_seen": 134021120 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004845336008024072, + "loss": 2.0236, + "theoretical_loss": 4.614353303625382, + "tokens_seen": 134086656 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048452357071213645, + "loss": 1.6789, + "theoretical_loss": 4.6140554702746215, + "tokens_seen": 134152192 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004845135406218656, + "loss": 1.8539, + "theoretical_loss": 4.613757823102437, + "tokens_seen": 134217728 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004845035105315948, + "loss": 1.7977, + "theoretical_loss": 4.613460361901627, + "tokens_seen": 134283264 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 51514, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.8250213861465454, + "objective/train/theoretical_loss": 4.613163086465315, + "objective/train/tokens_used": 134348800, + "theoretical_loss": 4.613163086465315, + "tokens_seen": 134348800 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048449348044132394, + "loss": 1.9103, + "theoretical_loss": 4.613163086465315, + "tokens_seen": 134348800 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004844834503510532, + "loss": 1.9225, + "theoretical_loss": 4.612865996586962, + "tokens_seen": 134414336 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048447342026078236, + "loss": 1.9131, + "theoretical_loss": 4.612569092060355, + "tokens_seen": 134479872 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048446339017051154, + "loss": 1.8689, + "theoretical_loss": 4.612272372679611, + "tokens_seen": 134545408 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004844533600802407, + "loss": 1.8516, + "theoretical_loss": 4.611975838239179, + "tokens_seen": 134610944 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004844433299899699, + "loss": 1.9799, + "theoretical_loss": 4.611679488533832, + "tokens_seen": 134676480 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004844332998996991, + "loss": 1.8083, + "theoretical_loss": 4.611383323358671, + "tokens_seen": 134742016 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004844232698094283, + "loss": 1.931, + "theoretical_loss": 4.61108734250913, + "tokens_seen": 134807552 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048441323971915745, + "loss": 1.8204, + "theoretical_loss": 4.610791545780959, + "tokens_seen": 134873088 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004844032096288867, + "loss": 1.8523, + "theoretical_loss": 4.610495932970241, + "tokens_seen": 134938624 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048439317953861586, + "loss": 1.7784, + "theoretical_loss": 4.61020050387338, + "tokens_seen": 135004160 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048438314944834504, + "loss": 1.7767, + "theoretical_loss": 4.609905258287107, + "tokens_seen": 135069696 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004843731193580742, + "loss": 1.8835, + "theoretical_loss": 4.609610196008473, + "tokens_seen": 135135232 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004843630892678034, + "loss": 1.8145, + "theoretical_loss": 4.609315316834855, + "tokens_seen": 135200768 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004843530591775326, + "loss": 1.8311, + "theoretical_loss": 4.60902062056395, + "tokens_seen": 135266304 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004843430290872618, + "loss": 1.8741, + "theoretical_loss": 4.608726106993776, + "tokens_seen": 135331840 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048433299899699095, + "loss": 1.9007, + "theoretical_loss": 4.608431775922673, + "tokens_seen": 135397376 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004843229689067202, + "loss": 1.9052, + "theoretical_loss": 4.608137627149302, + "tokens_seen": 135462912 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004843129388164493, + "loss": 1.7655, + "theoretical_loss": 4.607843660472642, + "tokens_seen": 135528448 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048430290872617855, + "loss": 1.9713, + "theoretical_loss": 4.607549875691989, + "tokens_seen": 135593984 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048429287863590773, + "loss": 1.7746, + "theoretical_loss": 4.607256272606961, + "tokens_seen": 135659520 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004842828485456369, + "loss": 1.73, + "theoretical_loss": 4.606962851017489, + "tokens_seen": 135725056 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004842728184553661, + "loss": 1.8336, + "theoretical_loss": 4.606669610723825, + "tokens_seen": 135790592 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048426278836509527, + "loss": 1.7875, + "theoretical_loss": 4.606376551526536, + "tokens_seen": 135856128 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048425275827482445, + "loss": 1.8947, + "theoretical_loss": 4.6060836732265, + "tokens_seen": 135921664 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004842427281845537, + "loss": 1.7936, + "theoretical_loss": 4.605790975624917, + "tokens_seen": 135987200 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048423269809428287, + "loss": 1.742, + "theoretical_loss": 4.605498458523296, + "tokens_seen": 136052736 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048422266800401205, + "loss": 1.9093, + "theoretical_loss": 4.605206121723461, + "tokens_seen": 136118272 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004842126379137413, + "loss": 1.7814, + "theoretical_loss": 4.604913965027551, + "tokens_seen": 136183808 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004842026078234704, + "loss": 1.8559, + "theoretical_loss": 4.604621988238012, + "tokens_seen": 136249344 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048419257773319965, + "loss": 1.7887, + "theoretical_loss": 4.604330191157608, + "tokens_seen": 136314880 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004841825476429288, + "loss": 1.7841, + "theoretical_loss": 4.604038573589408, + "tokens_seen": 136380416 + }, + { + "epoch": 0.04, + "learning_rate": 0.000484172517552658, + "loss": 1.7175, + "theoretical_loss": 4.603747135336796, + "tokens_seen": 136445952 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004841624874623872, + "loss": 1.7341, + "theoretical_loss": 4.603455876203463, + "tokens_seen": 136511488 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048415245737211637, + "loss": 1.8668, + "theoretical_loss": 4.603164795993409, + "tokens_seen": 136577024 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048414242728184555, + "loss": 1.8564, + "theoretical_loss": 4.602873894510945, + "tokens_seen": 136642560 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048413239719157473, + "loss": 1.7696, + "theoretical_loss": 4.602583171560686, + "tokens_seen": 136708096 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004841223671013039, + "loss": 1.8755, + "theoretical_loss": 4.602292626947556, + "tokens_seen": 136773632 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048411233701103315, + "loss": 1.8206, + "theoretical_loss": 4.602002260476787, + "tokens_seen": 136839168 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004841023069207623, + "loss": 1.8211, + "theoretical_loss": 4.601712071953914, + "tokens_seen": 136904704 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004840922768304915, + "loss": 1.8036, + "theoretical_loss": 4.601422061184778, + "tokens_seen": 136970240 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048408224674022064, + "loss": 1.8879, + "theoretical_loss": 4.601132227975527, + "tokens_seen": 137035776 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004840722166499499, + "loss": 1.9117, + "theoretical_loss": 4.60084257213261, + "tokens_seen": 137101312 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048406218655967906, + "loss": 1.8787, + "theoretical_loss": 4.600553093462781, + "tokens_seen": 137166848 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048405215646940824, + "loss": 1.6697, + "theoretical_loss": 4.600263791773097, + "tokens_seen": 137232384 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004840421263791374, + "loss": 1.7928, + "theoretical_loss": 4.599974666870914, + "tokens_seen": 137297920 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048403209628886665, + "loss": 1.8111, + "theoretical_loss": 4.599685718563894, + "tokens_seen": 137363456 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004840220661985958, + "loss": 1.9271, + "theoretical_loss": 4.599396946659997, + "tokens_seen": 137428992 + }, + { + "epoch": 0.04, + "learning_rate": 0.000484012036108325, + "loss": 1.7873, + "theoretical_loss": 4.5991083509674855, + "tokens_seen": 137494528 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048400200601805414, + "loss": 1.9491, + "theoretical_loss": 4.598819931294919, + "tokens_seen": 137560064 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 52427, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.8075873851776123, + "objective/train/theoretical_loss": 4.598531687451157, + "objective/train/tokens_used": 137625600, + "theoretical_loss": 4.598531687451157, + "tokens_seen": 137625600 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004839919759277834, + "loss": 1.8259, + "theoretical_loss": 4.598531687451157, + "tokens_seen": 137625600 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048398194583751256, + "loss": 1.7618, + "theoretical_loss": 4.598243619245359, + "tokens_seen": 137691136 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048397191574724174, + "loss": 1.6716, + "theoretical_loss": 4.59795572648698, + "tokens_seen": 137756672 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004839618856569709, + "loss": 1.8627, + "theoretical_loss": 4.597668008985774, + "tokens_seen": 137822208 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004839518555667001, + "loss": 1.8523, + "theoretical_loss": 4.59738046655179, + "tokens_seen": 137887744 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004839418254764293, + "loss": 1.9003, + "theoretical_loss": 4.597093098995377, + "tokens_seen": 137953280 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004839317953861585, + "loss": 1.7224, + "theoretical_loss": 4.596805906127173, + "tokens_seen": 138018816 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048392176529588765, + "loss": 1.9068, + "theoretical_loss": 4.596518887758117, + "tokens_seen": 138084352 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004839117352056169, + "loss": 1.8299, + "theoretical_loss": 4.596232043699438, + "tokens_seen": 138149888 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048390170511534606, + "loss": 1.8389, + "theoretical_loss": 4.59594537376266, + "tokens_seen": 138215424 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048389167502507524, + "loss": 1.8709, + "theoretical_loss": 4.595658877759602, + "tokens_seen": 138280960 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004838816449348044, + "loss": 1.808, + "theoretical_loss": 4.595372555502371, + "tokens_seen": 138346496 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004838716148445336, + "loss": 1.8484, + "theoretical_loss": 4.59508640680337, + "tokens_seen": 138412032 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004838615847542628, + "loss": 1.8412, + "theoretical_loss": 4.594800431475292, + "tokens_seen": 138477568 + }, + { + "epoch": 0.04, + "learning_rate": 0.000483851554663992, + "loss": 1.8583, + "theoretical_loss": 4.594514629331121, + "tokens_seen": 138543104 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048384152457372115, + "loss": 1.7597, + "theoretical_loss": 4.594229000184128, + "tokens_seen": 138608640 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004838314944834504, + "loss": 1.7525, + "theoretical_loss": 4.593943543847878, + "tokens_seen": 138674176 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004838214643931795, + "loss": 1.8563, + "theoretical_loss": 4.5936582601362215, + "tokens_seen": 138739712 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048381143430290875, + "loss": 1.8538, + "theoretical_loss": 4.593373148863301, + "tokens_seen": 138805248 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048380140421263793, + "loss": 1.7739, + "theoretical_loss": 4.593088209843543, + "tokens_seen": 138870784 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004837913741223671, + "loss": 1.7609, + "theoretical_loss": 4.592803442891663, + "tokens_seen": 138936320 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004837813440320963, + "loss": 1.7419, + "theoretical_loss": 4.592518847822661, + "tokens_seen": 139001856 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048377131394182547, + "loss": 1.7392, + "theoretical_loss": 4.592234424451828, + "tokens_seen": 139067392 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048376128385155465, + "loss": 1.9285, + "theoretical_loss": 4.591950172594734, + "tokens_seen": 139132928 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004837512537612839, + "loss": 1.9031, + "theoretical_loss": 4.591666092067238, + "tokens_seen": 139198464 + }, + { + "epoch": 0.04, + "learning_rate": 0.000483741223671013, + "loss": 1.7842, + "theoretical_loss": 4.591382182685484, + "tokens_seen": 139264000 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048373119358074225, + "loss": 1.8905, + "theoretical_loss": 4.591098444265896, + "tokens_seen": 139329536 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048372116349047143, + "loss": 1.8468, + "theoretical_loss": 4.590814876625183, + "tokens_seen": 139395072 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004837111334002006, + "loss": 1.9172, + "theoretical_loss": 4.590531479580338, + "tokens_seen": 139460608 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004837011033099298, + "loss": 1.7657, + "theoretical_loss": 4.5902482529486335, + "tokens_seen": 139526144 + }, + { + "epoch": 0.04, + "learning_rate": 0.000483691073219659, + "loss": 1.8885, + "theoretical_loss": 4.589965196547625, + "tokens_seen": 139591680 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048368104312938816, + "loss": 1.8214, + "theoretical_loss": 4.5896823101951485, + "tokens_seen": 139657216 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004836710130391174, + "loss": 1.8361, + "theoretical_loss": 4.58939959370932, + "tokens_seen": 139722752 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004836609829488465, + "loss": 1.8781, + "theoretical_loss": 4.5891170469085365, + "tokens_seen": 139788288 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048365095285857575, + "loss": 1.7787, + "theoretical_loss": 4.588834669611472, + "tokens_seen": 139853824 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004836409227683049, + "loss": 1.7594, + "theoretical_loss": 4.5885524616370805, + "tokens_seen": 139919360 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004836308926780341, + "loss": 1.848, + "theoretical_loss": 4.5882704228045945, + "tokens_seen": 139984896 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004836208625877633, + "loss": 1.7379, + "theoretical_loss": 4.587988552933523, + "tokens_seen": 140050432 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004836108324974925, + "loss": 1.8208, + "theoretical_loss": 4.587706851843652, + "tokens_seen": 140115968 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048360080240722166, + "loss": 1.8139, + "theoretical_loss": 4.587425319355044, + "tokens_seen": 140181504 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048359077231695084, + "loss": 1.834, + "theoretical_loss": 4.587143955288038, + "tokens_seen": 140247040 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048358074222668, + "loss": 1.808, + "theoretical_loss": 4.586862759463248, + "tokens_seen": 140312576 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048357071213640926, + "loss": 1.8202, + "theoretical_loss": 4.586581731701562, + "tokens_seen": 140378112 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004835606820461384, + "loss": 1.8152, + "theoretical_loss": 4.586300871824143, + "tokens_seen": 140443648 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004835506519558676, + "loss": 1.9599, + "theoretical_loss": 4.586020179652427, + "tokens_seen": 140509184 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004835406218655968, + "loss": 1.7534, + "theoretical_loss": 4.585739655008123, + "tokens_seen": 140574720 + }, + { + "epoch": 0.04, + "learning_rate": 0.000483530591775326, + "loss": 1.8764, + "theoretical_loss": 4.585459297713215, + "tokens_seen": 140640256 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048352056168505516, + "loss": 2.0722, + "theoretical_loss": 4.585179107589955, + "tokens_seen": 140705792 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048351053159478434, + "loss": 1.8277, + "theoretical_loss": 4.58489908446087, + "tokens_seen": 140771328 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004835005015045135, + "loss": 1.8669, + "theoretical_loss": 4.584619228148755, + "tokens_seen": 140836864 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 53591, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.8932198286056519, + "objective/train/theoretical_loss": 4.584339538476678, + "objective/train/tokens_used": 140902400, + "theoretical_loss": 4.584339538476678, + "tokens_seen": 140902400 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048349047141424276, + "loss": 1.7802, + "theoretical_loss": 4.584339538476678, + "tokens_seen": 140902400 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048348044132397194, + "loss": 2.0238, + "theoretical_loss": 4.5840600152679745, + "tokens_seen": 140967936 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004834704112337011, + "loss": 1.8351, + "theoretical_loss": 4.583780658346252, + "tokens_seen": 141033472 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004834603811434303, + "loss": 1.9054, + "theoretical_loss": 4.583501467535383, + "tokens_seen": 141099008 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004834503510531595, + "loss": 1.8304, + "theoretical_loss": 4.583222442659514, + "tokens_seen": 141164544 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004834403209628887, + "loss": 1.786, + "theoretical_loss": 4.5829435835430505, + "tokens_seen": 141230080 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048343029087261785, + "loss": 1.7802, + "theoretical_loss": 4.582664890010674, + "tokens_seen": 141295616 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004834202607823471, + "loss": 1.9877, + "theoretical_loss": 4.582386361887329, + "tokens_seen": 141361152 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048341023069207626, + "loss": 1.8171, + "theoretical_loss": 4.582107998998225, + "tokens_seen": 141426688 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048340020060180544, + "loss": 1.7691, + "theoretical_loss": 4.581829801168839, + "tokens_seen": 141492224 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004833901705115346, + "loss": 1.9294, + "theoretical_loss": 4.5815517682249105, + "tokens_seen": 141557760 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004833801404212638, + "loss": 1.7368, + "theoretical_loss": 4.581273899992447, + "tokens_seen": 141623296 + }, + { + "epoch": 0.04, + "learning_rate": 0.000483370110330993, + "loss": 1.7602, + "theoretical_loss": 4.580996196297718, + "tokens_seen": 141688832 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004833600802407222, + "loss": 1.758, + "theoretical_loss": 4.580718656967257, + "tokens_seen": 141754368 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048335005015045135, + "loss": 1.8104, + "theoretical_loss": 4.580441281827858, + "tokens_seen": 141819904 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004833400200601806, + "loss": 1.918, + "theoretical_loss": 4.580164070706584, + "tokens_seen": 141885440 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004833299899699097, + "loss": 1.8828, + "theoretical_loss": 4.579887023430752, + "tokens_seen": 141950976 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048331995987963895, + "loss": 1.9462, + "theoretical_loss": 4.579610139827945, + "tokens_seen": 142016512 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048330992978936813, + "loss": 1.9332, + "theoretical_loss": 4.579333419726007, + "tokens_seen": 142082048 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004832998996990973, + "loss": 1.8508, + "theoretical_loss": 4.579056862953039, + "tokens_seen": 142147584 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004832898696088265, + "loss": 1.8185, + "theoretical_loss": 4.578780469337407, + "tokens_seen": 142213120 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048327983951855567, + "loss": 1.7536, + "theoretical_loss": 4.5785042387077315, + "tokens_seen": 142278656 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048326980942828485, + "loss": 1.8187, + "theoretical_loss": 4.578228170892896, + "tokens_seen": 142344192 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004832597793380141, + "loss": 1.7686, + "theoretical_loss": 4.577952265722038, + "tokens_seen": 142409728 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004832497492477432, + "loss": 2.0272, + "theoretical_loss": 4.5776765230245555, + "tokens_seen": 142475264 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048323971915747245, + "loss": 1.7786, + "theoretical_loss": 4.5774009426301046, + "tokens_seen": 142540800 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048322968906720163, + "loss": 1.8098, + "theoretical_loss": 4.577125524368595, + "tokens_seen": 142606336 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004832196589769308, + "loss": 1.7322, + "theoretical_loss": 4.5768502680701975, + "tokens_seen": 142671872 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048320962888666, + "loss": 1.7618, + "theoretical_loss": 4.576575173565334, + "tokens_seen": 142737408 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004831995987963892, + "loss": 1.8043, + "theoretical_loss": 4.5763002406846836, + "tokens_seen": 142802944 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048318956870611836, + "loss": 1.7778, + "theoretical_loss": 4.57602546925918, + "tokens_seen": 142868480 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004831795386158476, + "loss": 1.7306, + "theoretical_loss": 4.575750859120014, + "tokens_seen": 142934016 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004831695085255767, + "loss": 1.7956, + "theoretical_loss": 4.575476410098624, + "tokens_seen": 142999552 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048315947843530595, + "loss": 1.9206, + "theoretical_loss": 4.575202122026706, + "tokens_seen": 143065088 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004831494483450351, + "loss": 1.7937, + "theoretical_loss": 4.5749279947362105, + "tokens_seen": 143130624 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004831394182547643, + "loss": 1.8366, + "theoretical_loss": 4.574654028059336, + "tokens_seen": 143196160 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004831293881644935, + "loss": 1.8136, + "theoretical_loss": 4.574380221828535, + "tokens_seen": 143261696 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004831193580742227, + "loss": 1.7063, + "theoretical_loss": 4.574106575876511, + "tokens_seen": 143327232 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048310932798395186, + "loss": 1.879, + "theoretical_loss": 4.573833090036219, + "tokens_seen": 143392768 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048309929789368104, + "loss": 1.9331, + "theoretical_loss": 4.573559764140864, + "tokens_seen": 143458304 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004830892678034102, + "loss": 1.8044, + "theoretical_loss": 4.573286598023898, + "tokens_seen": 143523840 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048307923771313946, + "loss": 1.8123, + "theoretical_loss": 4.573013591519029, + "tokens_seen": 143589376 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004830692076228686, + "loss": 1.9006, + "theoretical_loss": 4.572740744460207, + "tokens_seen": 143654912 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004830591775325978, + "loss": 1.763, + "theoretical_loss": 4.572468056681634, + "tokens_seen": 143720448 + }, + { + "epoch": 0.04, + "learning_rate": 0.000483049147442327, + "loss": 1.8118, + "theoretical_loss": 4.57219552801776, + "tokens_seen": 143785984 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004830391173520562, + "loss": 1.7771, + "theoretical_loss": 4.57192315830328, + "tokens_seen": 143851520 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048302908726178536, + "loss": 1.9476, + "theoretical_loss": 4.571650947373138, + "tokens_seen": 143917056 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048301905717151454, + "loss": 1.8083, + "theoretical_loss": 4.571378895062525, + "tokens_seen": 143982592 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004830090270812437, + "loss": 1.7899, + "theoretical_loss": 4.571107001206875, + "tokens_seen": 144048128 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048299899699097296, + "loss": 1.7038, + "theoretical_loss": 4.570835265641872, + "tokens_seen": 144113664 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 54758, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.821224331855774, + "objective/train/theoretical_loss": 4.57056368820344, + "objective/train/tokens_used": 144179200, + "theoretical_loss": 4.57056368820344, + "tokens_seen": 144179200 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004829889669007021, + "loss": 1.9007, + "theoretical_loss": 4.57056368820344, + "tokens_seen": 144179200 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004829789368104313, + "loss": 1.782, + "theoretical_loss": 4.570292268727751, + "tokens_seen": 144244736 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048296890672016045, + "loss": 1.8507, + "theoretical_loss": 4.570021007051221, + "tokens_seen": 144310272 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004829588766298897, + "loss": 1.8814, + "theoretical_loss": 4.569749903010507, + "tokens_seen": 144375808 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048294884653961887, + "loss": 1.911, + "theoretical_loss": 4.569478956442513, + "tokens_seen": 144441344 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048293881644934805, + "loss": 1.7926, + "theoretical_loss": 4.569208167184382, + "tokens_seen": 144506880 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048292878635907723, + "loss": 1.9232, + "theoretical_loss": 4.5689375350735, + "tokens_seen": 144572416 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048291875626880646, + "loss": 1.8171, + "theoretical_loss": 4.568667059947496, + "tokens_seen": 144637952 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004829087261785356, + "loss": 1.7774, + "theoretical_loss": 4.568396741644241, + "tokens_seen": 144703488 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004828986960882648, + "loss": 1.7679, + "theoretical_loss": 4.568126580001843, + "tokens_seen": 144769024 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048288866599799395, + "loss": 1.7784, + "theoretical_loss": 4.567856574858656, + "tokens_seen": 144834560 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004828786359077232, + "loss": 1.837, + "theoretical_loss": 4.567586726053266, + "tokens_seen": 144900096 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048286860581745237, + "loss": 1.8286, + "theoretical_loss": 4.567317033424507, + "tokens_seen": 144965632 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048285857572718155, + "loss": 1.7926, + "theoretical_loss": 4.567047496811445, + "tokens_seen": 145031168 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048284854563691073, + "loss": 1.832, + "theoretical_loss": 4.566778116053389, + "tokens_seen": 145096704 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004828385155466399, + "loss": 1.8608, + "theoretical_loss": 4.566508890989883, + "tokens_seen": 145162240 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004828284854563691, + "loss": 1.6213, + "theoretical_loss": 4.566239821460712, + "tokens_seen": 145227776 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048281845536609833, + "loss": 1.9386, + "theoretical_loss": 4.5659709073058945, + "tokens_seen": 145293312 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048280842527582746, + "loss": 1.8639, + "theoretical_loss": 4.565702148365686, + "tokens_seen": 145358848 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004827983951855567, + "loss": 1.7632, + "theoretical_loss": 4.56543354448058, + "tokens_seen": 145424384 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004827883650952858, + "loss": 1.9325, + "theoretical_loss": 4.565165095491306, + "tokens_seen": 145489920 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048277833500501505, + "loss": 1.8889, + "theoretical_loss": 4.5648968012388265, + "tokens_seen": 145555456 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048276830491474423, + "loss": 1.9137, + "theoretical_loss": 4.56462866156434, + "tokens_seen": 145620992 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004827582748244734, + "loss": 1.7965, + "theoretical_loss": 4.564360676309279, + "tokens_seen": 145686528 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004827482447342026, + "loss": 1.8861, + "theoretical_loss": 4.564092845315312, + "tokens_seen": 145752064 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048273821464393183, + "loss": 1.9325, + "theoretical_loss": 4.563825168424337, + "tokens_seen": 145817600 + }, + { + "epoch": 0.04, + "learning_rate": 0.000482728184553661, + "loss": 1.8035, + "theoretical_loss": 4.563557645478488, + "tokens_seen": 145883136 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004827181544633902, + "loss": 1.8034, + "theoretical_loss": 4.56329027632013, + "tokens_seen": 145948672 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004827081243731194, + "loss": 1.7839, + "theoretical_loss": 4.563023060791862, + "tokens_seen": 146014208 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048269809428284856, + "loss": 1.8709, + "theoretical_loss": 4.562755998736511, + "tokens_seen": 146079744 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004826880641925778, + "loss": 1.885, + "theoretical_loss": 4.562489089997141, + "tokens_seen": 146145280 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004826780341023069, + "loss": 1.8592, + "theoretical_loss": 4.562222334417041, + "tokens_seen": 146210816 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048266800401203615, + "loss": 1.9189, + "theoretical_loss": 4.561955731839734, + "tokens_seen": 146276352 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004826579739217653, + "loss": 1.8208, + "theoretical_loss": 4.561689282108971, + "tokens_seen": 146341888 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004826479438314945, + "loss": 1.7544, + "theoretical_loss": 4.561422985068733, + "tokens_seen": 146407424 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004826379137412237, + "loss": 1.797, + "theoretical_loss": 4.561156840563231, + "tokens_seen": 146472960 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004826278836509529, + "loss": 1.7723, + "theoretical_loss": 4.560890848436902, + "tokens_seen": 146538496 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048261785356068206, + "loss": 1.8985, + "theoretical_loss": 4.560625008534414, + "tokens_seen": 146604032 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048260782347041124, + "loss": 1.8621, + "theoretical_loss": 4.560359320700662, + "tokens_seen": 146669568 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004825977933801404, + "loss": 1.8641, + "theoretical_loss": 4.560093784780766, + "tokens_seen": 146735104 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048258776328986966, + "loss": 1.8293, + "theoretical_loss": 4.559828400620075, + "tokens_seen": 146800640 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004825777331995988, + "loss": 1.8019, + "theoretical_loss": 4.559563168064165, + "tokens_seen": 146866176 + }, + { + "epoch": 0.04, + "learning_rate": 0.000482567703109328, + "loss": 1.9091, + "theoretical_loss": 4.559298086958837, + "tokens_seen": 146931712 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004825576730190572, + "loss": 1.852, + "theoretical_loss": 4.559033157150115, + "tokens_seen": 146997248 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004825476429287864, + "loss": 1.9202, + "theoretical_loss": 4.5587683784842525, + "tokens_seen": 147062784 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048253761283851556, + "loss": 1.7414, + "theoretical_loss": 4.558503750807725, + "tokens_seen": 147128320 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048252758274824474, + "loss": 1.8852, + "theoretical_loss": 4.558239273967234, + "tokens_seen": 147193856 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004825175526579739, + "loss": 1.8845, + "theoretical_loss": 4.5579749478097025, + "tokens_seen": 147259392 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048250752256770316, + "loss": 1.7515, + "theoretical_loss": 4.557710772182278, + "tokens_seen": 147324928 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004824974924774323, + "loss": 1.8905, + "theoretical_loss": 4.5574467469323325, + "tokens_seen": 147390464 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 55735, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.8850185871124268, + "objective/train/theoretical_loss": 4.557182871907457, + "objective/train/tokens_used": 147456000, + "theoretical_loss": 4.557182871907457, + "tokens_seen": 147456000 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004824874623871615, + "loss": 1.8565, + "theoretical_loss": 4.557182871907457, + "tokens_seen": 147456000 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048247743229689065, + "loss": 1.7132, + "theoretical_loss": 4.556919146955469, + "tokens_seen": 147521536 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004824674022066199, + "loss": 1.7954, + "theoretical_loss": 4.556655571924404, + "tokens_seen": 147587072 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048245737211634907, + "loss": 1.8153, + "theoretical_loss": 4.556392146662521, + "tokens_seen": 147652608 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048244734202607825, + "loss": 1.8306, + "theoretical_loss": 4.556128871018297, + "tokens_seen": 147718144 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048243731193580743, + "loss": 1.855, + "theoretical_loss": 4.555865744840434, + "tokens_seen": 147783680 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048242728184553666, + "loss": 1.7391, + "theoretical_loss": 4.55560276797785, + "tokens_seen": 147849216 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004824172517552658, + "loss": 1.9467, + "theoretical_loss": 4.5553399402796835, + "tokens_seen": 147914752 + }, + { + "epoch": 0.04, + "learning_rate": 0.000482407221664995, + "loss": 1.8983, + "theoretical_loss": 4.5550772615952955, + "tokens_seen": 147980288 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048239719157472415, + "loss": 1.7164, + "theoretical_loss": 4.554814731774259, + "tokens_seen": 148045824 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004823871614844534, + "loss": 1.8079, + "theoretical_loss": 4.554552350666373, + "tokens_seen": 148111360 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048237713139418257, + "loss": 1.9013, + "theoretical_loss": 4.554290118121649, + "tokens_seen": 148176896 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048236710130391175, + "loss": 1.8107, + "theoretical_loss": 4.554028033990319, + "tokens_seen": 148242432 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048235707121364093, + "loss": 1.8334, + "theoretical_loss": 4.5537660981228285, + "tokens_seen": 148307968 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004823470411233701, + "loss": 1.7316, + "theoretical_loss": 4.553504310369844, + "tokens_seen": 148373504 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004823370110330993, + "loss": 2.0166, + "theoretical_loss": 4.553242670582245, + "tokens_seen": 148439040 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048232698094282853, + "loss": 1.8282, + "theoretical_loss": 4.55298117861113, + "tokens_seen": 148504576 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048231695085255766, + "loss": 1.8671, + "theoretical_loss": 4.55271983430781, + "tokens_seen": 148570112 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004823069207622869, + "loss": 1.7269, + "theoretical_loss": 4.552458637523815, + "tokens_seen": 148635648 + }, + { + "epoch": 0.05, + "learning_rate": 0.000482296890672016, + "loss": 1.782, + "theoretical_loss": 4.552197588110882, + "tokens_seen": 148701184 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048228686058174525, + "loss": 1.8461, + "theoretical_loss": 4.551936685920971, + "tokens_seen": 148766720 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048227683049147443, + "loss": 1.8871, + "theoretical_loss": 4.551675930806251, + "tokens_seen": 148832256 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004822668004012036, + "loss": 1.8601, + "theoretical_loss": 4.551415322619107, + "tokens_seen": 148897792 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004822567703109328, + "loss": 1.8741, + "theoretical_loss": 4.551154861212133, + "tokens_seen": 148963328 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048224674022066203, + "loss": 1.8656, + "theoretical_loss": 4.550894546438139, + "tokens_seen": 149028864 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048223671013039116, + "loss": 1.8763, + "theoretical_loss": 4.550634378150148, + "tokens_seen": 149094400 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004822266800401204, + "loss": 1.8384, + "theoretical_loss": 4.550374356201392, + "tokens_seen": 149159936 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004822166499498495, + "loss": 1.7988, + "theoretical_loss": 4.550114480445316, + "tokens_seen": 149225472 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048220661985957876, + "loss": 1.7969, + "theoretical_loss": 4.549854750735575, + "tokens_seen": 149291008 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048219658976930794, + "loss": 1.8009, + "theoretical_loss": 4.5495951669260375, + "tokens_seen": 149356544 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004821865596790371, + "loss": 1.8328, + "theoretical_loss": 4.54933572887078, + "tokens_seen": 149422080 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004821765295887663, + "loss": 1.8775, + "theoretical_loss": 4.5490764364240865, + "tokens_seen": 149487616 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004821664994984955, + "loss": 1.9574, + "theoretical_loss": 4.548817289440455, + "tokens_seen": 149553152 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048215646940822466, + "loss": 1.8186, + "theoretical_loss": 4.548558287774592, + "tokens_seen": 149618688 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004821464393179539, + "loss": 1.8647, + "theoretical_loss": 4.548299431281409, + "tokens_seen": 149684224 + }, + { + "epoch": 0.05, + "learning_rate": 0.000482136409227683, + "loss": 1.771, + "theoretical_loss": 4.548040719816029, + "tokens_seen": 149749760 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048212637913741226, + "loss": 1.7568, + "theoretical_loss": 4.547782153233783, + "tokens_seen": 149815296 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004821163490471414, + "loss": 1.8153, + "theoretical_loss": 4.547523731390206, + "tokens_seen": 149880832 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004821063189568706, + "loss": 1.8698, + "theoretical_loss": 4.547265454141046, + "tokens_seen": 149946368 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004820962888665998, + "loss": 1.7548, + "theoretical_loss": 4.547007321342251, + "tokens_seen": 150011904 + }, + { + "epoch": 0.05, + "learning_rate": 0.000482086258776329, + "loss": 1.8572, + "theoretical_loss": 4.546749332849982, + "tokens_seen": 150077440 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048207622868605816, + "loss": 1.6633, + "theoretical_loss": 4.546491488520601, + "tokens_seen": 150142976 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004820661985957874, + "loss": 1.875, + "theoretical_loss": 4.546233788210679, + "tokens_seen": 150208512 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048205616850551653, + "loss": 1.875, + "theoretical_loss": 4.545976231776988, + "tokens_seen": 150274048 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048204613841524576, + "loss": 1.8252, + "theoretical_loss": 4.545718819076509, + "tokens_seen": 150339584 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004820361083249749, + "loss": 1.8585, + "theoretical_loss": 4.545461549966426, + "tokens_seen": 150405120 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004820260782347041, + "loss": 1.8548, + "theoretical_loss": 4.545204424304126, + "tokens_seen": 150470656 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004820160481444333, + "loss": 1.8728, + "theoretical_loss": 4.544947441947201, + "tokens_seen": 150536192 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004820060180541625, + "loss": 1.7235, + "theoretical_loss": 4.544690602753445, + "tokens_seen": 150601728 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048199598796389167, + "loss": 1.887, + "theoretical_loss": 4.544433906580857, + "tokens_seen": 150667264 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 56873, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.7476823329925537, + "objective/train/theoretical_loss": 4.544177353287637, + "objective/train/tokens_used": 150732800, + "theoretical_loss": 4.544177353287637, + "tokens_seen": 150732800 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048198595787362085, + "loss": 1.788, + "theoretical_loss": 4.544177353287637, + "tokens_seen": 150732800 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004819759277833501, + "loss": 1.904, + "theoretical_loss": 4.543920942732187, + "tokens_seen": 150798336 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048196589769307927, + "loss": 1.8692, + "theoretical_loss": 4.543664674773112, + "tokens_seen": 150863872 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048195586760280845, + "loss": 1.7841, + "theoretical_loss": 4.543408549269218, + "tokens_seen": 150929408 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048194583751253763, + "loss": 1.7544, + "theoretical_loss": 4.54315256607951, + "tokens_seen": 150994944 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048193580742226686, + "loss": 1.806, + "theoretical_loss": 4.542896725063195, + "tokens_seen": 151060480 + }, + { + "epoch": 0.05, + "learning_rate": 0.000481925777331996, + "loss": 1.8683, + "theoretical_loss": 4.542641026079683, + "tokens_seen": 151126016 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004819157472417252, + "loss": 1.7521, + "theoretical_loss": 4.542385468988581, + "tokens_seen": 151191552 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048190571715145435, + "loss": 1.7677, + "theoretical_loss": 4.542130053649695, + "tokens_seen": 151257088 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004818956870611836, + "loss": 1.859, + "theoretical_loss": 4.541874779923031, + "tokens_seen": 151322624 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048188565697091277, + "loss": 1.8398, + "theoretical_loss": 4.541619647668796, + "tokens_seen": 151388160 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048187562688064195, + "loss": 1.8141, + "theoretical_loss": 4.541364656747392, + "tokens_seen": 151453696 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048186559679037113, + "loss": 1.8379, + "theoretical_loss": 4.541109807019422, + "tokens_seen": 151519232 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004818555667001003, + "loss": 1.7753, + "theoretical_loss": 4.540855098345683, + "tokens_seen": 151584768 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004818455366098295, + "loss": 1.935, + "theoretical_loss": 4.540600530587175, + "tokens_seen": 151650304 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048183550651955873, + "loss": 1.8225, + "theoretical_loss": 4.5403461036050885, + "tokens_seen": 151715840 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048182547642928786, + "loss": 1.8397, + "theoretical_loss": 4.540091817260815, + "tokens_seen": 151781376 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004818154463390171, + "loss": 1.6938, + "theoretical_loss": 4.539837671415941, + "tokens_seen": 151846912 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004818054162487462, + "loss": 1.7713, + "theoretical_loss": 4.539583665932249, + "tokens_seen": 151912448 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048179538615847545, + "loss": 1.9798, + "theoretical_loss": 4.539329800671718, + "tokens_seen": 151977984 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048178535606820463, + "loss": 1.7454, + "theoretical_loss": 4.539076075496519, + "tokens_seen": 152043520 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004817753259779338, + "loss": 1.8212, + "theoretical_loss": 4.538822490269021, + "tokens_seen": 152109056 + }, + { + "epoch": 0.05, + "learning_rate": 0.000481765295887663, + "loss": 1.8588, + "theoretical_loss": 4.538569044851785, + "tokens_seen": 152174592 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048175526579739223, + "loss": 1.8207, + "theoretical_loss": 4.538315739107571, + "tokens_seen": 152240128 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048174523570712136, + "loss": 1.9064, + "theoretical_loss": 4.5380625728993245, + "tokens_seen": 152305664 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004817352056168506, + "loss": 1.779, + "theoretical_loss": 4.537809546090193, + "tokens_seen": 152371200 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004817251755265797, + "loss": 1.8155, + "theoretical_loss": 4.53755665854351, + "tokens_seen": 152436736 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048171514543630896, + "loss": 1.6735, + "theoretical_loss": 4.537303910122807, + "tokens_seen": 152502272 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048170511534603814, + "loss": 1.7848, + "theoretical_loss": 4.537051300691805, + "tokens_seen": 152567808 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816950852557673, + "loss": 1.8262, + "theoretical_loss": 4.5367988301144155, + "tokens_seen": 152633344 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816850551654965, + "loss": 1.9723, + "theoretical_loss": 4.536546498254746, + "tokens_seen": 152698880 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816750250752257, + "loss": 1.8402, + "theoretical_loss": 4.536294304977092, + "tokens_seen": 152764416 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048166499498495486, + "loss": 1.6831, + "theoretical_loss": 4.536042250145941, + "tokens_seen": 152829952 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816549648946841, + "loss": 1.7282, + "theoretical_loss": 4.53579033362597, + "tokens_seen": 152895488 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816449348044132, + "loss": 1.7312, + "theoretical_loss": 4.535538555282048, + "tokens_seen": 152961024 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048163490471414246, + "loss": 1.7332, + "theoretical_loss": 4.535286914979232, + "tokens_seen": 153026560 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816248746238716, + "loss": 1.8674, + "theoretical_loss": 4.5350354125827685, + "tokens_seen": 153092096 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816148445336008, + "loss": 1.7036, + "theoretical_loss": 4.534784047958096, + "tokens_seen": 153157632 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048160481444333, + "loss": 1.7146, + "theoretical_loss": 4.534532820970839, + "tokens_seen": 153223168 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004815947843530592, + "loss": 1.8438, + "theoretical_loss": 4.534281731486812, + "tokens_seen": 153288704 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048158475426278837, + "loss": 1.8283, + "theoretical_loss": 4.5340307793720145, + "tokens_seen": 153354240 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004815747241725176, + "loss": 1.9017, + "theoretical_loss": 4.533779964492638, + "tokens_seen": 153419776 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048156469408224673, + "loss": 1.6894, + "theoretical_loss": 4.533529286715059, + "tokens_seen": 153485312 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048155466399197596, + "loss": 1.6832, + "theoretical_loss": 4.533278745905843, + "tokens_seen": 153550848 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004815446339017051, + "loss": 1.779, + "theoretical_loss": 4.533028341931738, + "tokens_seen": 153616384 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004815346038114343, + "loss": 1.7661, + "theoretical_loss": 4.532778074659685, + "tokens_seen": 153681920 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004815245737211635, + "loss": 1.7507, + "theoretical_loss": 4.532527943956804, + "tokens_seen": 153747456 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004815145436308927, + "loss": 1.8204, + "theoretical_loss": 4.532277949690407, + "tokens_seen": 153812992 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048150451354062187, + "loss": 1.7807, + "theoretical_loss": 4.532028091727986, + "tokens_seen": 153878528 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048149448345035105, + "loss": 1.8802, + "theoretical_loss": 4.531778369937222, + "tokens_seen": 153944064 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 57712, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.608492136001587, + "objective/train/theoretical_loss": 4.53152878418598, + "objective/train/tokens_used": 154009600, + "theoretical_loss": 4.53152878418598, + "tokens_seen": 154009600 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048148445336008023, + "loss": 1.6938, + "theoretical_loss": 4.53152878418598, + "tokens_seen": 154009600 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048147442326980947, + "loss": 1.8098, + "theoretical_loss": 4.531279334342306, + "tokens_seen": 154075136 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004814643931795386, + "loss": 1.7783, + "theoretical_loss": 4.531030020274436, + "tokens_seen": 154140672 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048145436308926783, + "loss": 1.7575, + "theoretical_loss": 4.530780841850783, + "tokens_seen": 154206208 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048144433299899696, + "loss": 1.7475, + "theoretical_loss": 4.5305317989399505, + "tokens_seen": 154271744 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004814343029087262, + "loss": 1.8969, + "theoretical_loss": 4.530282891410717, + "tokens_seen": 154337280 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048142427281845537, + "loss": 1.7506, + "theoretical_loss": 4.530034119132052, + "tokens_seen": 154402816 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048141424272818455, + "loss": 1.8879, + "theoretical_loss": 4.529785481973102, + "tokens_seen": 154468352 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048140421263791373, + "loss": 1.7256, + "theoretical_loss": 4.529536979803195, + "tokens_seen": 154533888 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048139418254764297, + "loss": 1.7887, + "theoretical_loss": 4.529288612491846, + "tokens_seen": 154599424 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004813841524573721, + "loss": 1.7041, + "theoretical_loss": 4.5290403799087455, + "tokens_seen": 154664960 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048137412236710133, + "loss": 1.7403, + "theoretical_loss": 4.528792281923769, + "tokens_seen": 154730496 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048136409227683046, + "loss": 1.7662, + "theoretical_loss": 4.52854431840697, + "tokens_seen": 154796032 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004813540621865597, + "loss": 1.7778, + "theoretical_loss": 4.528296489228585, + "tokens_seen": 154861568 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004813440320962889, + "loss": 1.9558, + "theoretical_loss": 4.528048794259028, + "tokens_seen": 154927104 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048133400200601806, + "loss": 1.9177, + "theoretical_loss": 4.527801233368896, + "tokens_seen": 154992640 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048132397191574724, + "loss": 1.8535, + "theoretical_loss": 4.527553806428962, + "tokens_seen": 155058176 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004813139418254764, + "loss": 1.6233, + "theoretical_loss": 4.5273065133101795, + "tokens_seen": 155123712 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004813039117352056, + "loss": 1.8061, + "theoretical_loss": 4.527059353883682, + "tokens_seen": 155189248 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048129388164493483, + "loss": 1.8328, + "theoretical_loss": 4.526812328020778, + "tokens_seen": 155254784 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048128385155466396, + "loss": 1.6676, + "theoretical_loss": 4.52656543559296, + "tokens_seen": 155320320 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004812738214643932, + "loss": 1.8653, + "theoretical_loss": 4.526318676471891, + "tokens_seen": 155385856 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004812637913741223, + "loss": 1.7049, + "theoretical_loss": 4.526072050529417, + "tokens_seen": 155451392 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048125376128385156, + "loss": 1.8618, + "theoretical_loss": 4.52582555763756, + "tokens_seen": 155516928 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048124373119358074, + "loss": 1.7094, + "theoretical_loss": 4.525579197668517, + "tokens_seen": 155582464 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004812337011033099, + "loss": 1.7579, + "theoretical_loss": 4.525332970494663, + "tokens_seen": 155648000 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048122367101303916, + "loss": 1.8127, + "theoretical_loss": 4.525086875988549, + "tokens_seen": 155713536 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048121364092276834, + "loss": 1.8043, + "theoretical_loss": 4.524840914022902, + "tokens_seen": 155779072 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004812036108324975, + "loss": 1.9564, + "theoretical_loss": 4.524595084470625, + "tokens_seen": 155844608 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004811935807422267, + "loss": 1.6959, + "theoretical_loss": 4.524349387204794, + "tokens_seen": 155910144 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004811835506519559, + "loss": 1.7996, + "theoretical_loss": 4.524103822098664, + "tokens_seen": 155975680 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048117352056168506, + "loss": 1.8139, + "theoretical_loss": 4.52385838902566, + "tokens_seen": 156041216 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004811634904714143, + "loss": 1.7794, + "theoretical_loss": 4.523613087859386, + "tokens_seen": 156106752 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004811534603811434, + "loss": 1.7285, + "theoretical_loss": 4.523367918473614, + "tokens_seen": 156172288 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048114343029087266, + "loss": 1.8175, + "theoretical_loss": 4.523122880742298, + "tokens_seen": 156237824 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004811334002006018, + "loss": 1.8441, + "theoretical_loss": 4.522877974539556, + "tokens_seen": 156303360 + }, + { + "epoch": 0.05, + "learning_rate": 0.000481123370110331, + "loss": 1.9417, + "theoretical_loss": 4.522633199739687, + "tokens_seen": 156368896 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004811133400200602, + "loss": 1.8211, + "theoretical_loss": 4.522388556217157, + "tokens_seen": 156434432 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004811033099297894, + "loss": 1.7573, + "theoretical_loss": 4.522144043846609, + "tokens_seen": 156499968 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048109327983951857, + "loss": 1.7876, + "theoretical_loss": 4.521899662502856, + "tokens_seen": 156565504 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004810832497492478, + "loss": 1.7704, + "theoretical_loss": 4.5216554120608805, + "tokens_seen": 156631040 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048107321965897693, + "loss": 1.8299, + "theoretical_loss": 4.521411292395841, + "tokens_seen": 156696576 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048106318956870616, + "loss": 1.9476, + "theoretical_loss": 4.521167303383065, + "tokens_seen": 156762112 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004810531594784353, + "loss": 1.8589, + "theoretical_loss": 4.520923444898051, + "tokens_seen": 156827648 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004810431293881645, + "loss": 1.832, + "theoretical_loss": 4.520679716816467, + "tokens_seen": 156893184 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004810330992978937, + "loss": 1.7445, + "theoretical_loss": 4.520436119014155, + "tokens_seen": 156958720 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004810230692076229, + "loss": 1.7647, + "theoretical_loss": 4.520192651367121, + "tokens_seen": 157024256 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048101303911735207, + "loss": 1.8737, + "theoretical_loss": 4.519949313751547, + "tokens_seen": 157089792 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048100300902708125, + "loss": 1.9151, + "theoretical_loss": 4.519706106043779, + "tokens_seen": 157155328 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048099297893681043, + "loss": 1.7246, + "theoretical_loss": 4.519463028120338, + "tokens_seen": 157220864 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 58983, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.9745151996612549, + "objective/train/theoretical_loss": 4.519220079857908, + "objective/train/tokens_used": 157286400, + "theoretical_loss": 4.519220079857908, + "tokens_seen": 157286400 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048098294884653967, + "loss": 1.8145, + "theoretical_loss": 4.519220079857908, + "tokens_seen": 157286400 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004809729187562688, + "loss": 1.8289, + "theoretical_loss": 4.518977261133343, + "tokens_seen": 157351936 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048096288866599803, + "loss": 1.6887, + "theoretical_loss": 4.518734571823668, + "tokens_seen": 157417472 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048095285857572716, + "loss": 1.739, + "theoretical_loss": 4.518492011806073, + "tokens_seen": 157483008 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004809428284854564, + "loss": 1.7648, + "theoretical_loss": 4.518249580957917, + "tokens_seen": 157548544 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048093279839518557, + "loss": 1.8558, + "theoretical_loss": 4.518007279156725, + "tokens_seen": 157614080 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048092276830491475, + "loss": 1.7615, + "theoretical_loss": 4.5177651062801925, + "tokens_seen": 157679616 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048091273821464393, + "loss": 1.8399, + "theoretical_loss": 4.517523062206175, + "tokens_seen": 157745152 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048090270812437317, + "loss": 1.7577, + "theoretical_loss": 4.517281146812701, + "tokens_seen": 157810688 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004808926780341023, + "loss": 1.6627, + "theoretical_loss": 4.517039359977961, + "tokens_seen": 157876224 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048088264794383153, + "loss": 1.8004, + "theoretical_loss": 4.516797701580314, + "tokens_seen": 157941760 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048087261785356066, + "loss": 1.8941, + "theoretical_loss": 4.516556171498284, + "tokens_seen": 158007296 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004808625877632899, + "loss": 1.8548, + "theoretical_loss": 4.516314769610557, + "tokens_seen": 158072832 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004808525576730191, + "loss": 1.8479, + "theoretical_loss": 4.516073495795988, + "tokens_seen": 158138368 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048084252758274826, + "loss": 1.6167, + "theoretical_loss": 4.5158323499335955, + "tokens_seen": 158203904 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048083249749247744, + "loss": 1.812, + "theoretical_loss": 4.515591331902561, + "tokens_seen": 158269440 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004808224674022066, + "loss": 1.7644, + "theoretical_loss": 4.515350441582231, + "tokens_seen": 158334976 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004808124373119358, + "loss": 1.8613, + "theoretical_loss": 4.515109678852117, + "tokens_seen": 158400512 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048080240722166503, + "loss": 1.9329, + "theoretical_loss": 4.514869043591891, + "tokens_seen": 158466048 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048079237713139416, + "loss": 1.8712, + "theoretical_loss": 4.514628535681391, + "tokens_seen": 158531584 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004807823470411234, + "loss": 1.7491, + "theoretical_loss": 4.514388155000615, + "tokens_seen": 158597120 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004807723169508525, + "loss": 1.8397, + "theoretical_loss": 4.5141479014297285, + "tokens_seen": 158662656 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048076228686058176, + "loss": 1.793, + "theoretical_loss": 4.513907774849054, + "tokens_seen": 158728192 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048075225677031094, + "loss": 1.6541, + "theoretical_loss": 4.513667775139078, + "tokens_seen": 158793728 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004807422266800401, + "loss": 1.8723, + "theoretical_loss": 4.5134279021804495, + "tokens_seen": 158859264 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004807321965897693, + "loss": 1.7956, + "theoretical_loss": 4.513188155853978, + "tokens_seen": 158924800 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048072216649949854, + "loss": 1.7874, + "theoretical_loss": 4.512948536040636, + "tokens_seen": 158990336 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048071213640922766, + "loss": 1.8917, + "theoretical_loss": 4.512709042621554, + "tokens_seen": 159055872 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004807021063189569, + "loss": 1.6421, + "theoretical_loss": 4.512469675478026, + "tokens_seen": 159121408 + }, + { + "epoch": 0.05, + "learning_rate": 0.000480692076228686, + "loss": 1.8051, + "theoretical_loss": 4.512230434491503, + "tokens_seen": 159186944 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048068204613841526, + "loss": 1.7515, + "theoretical_loss": 4.511991319543599, + "tokens_seen": 159252480 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048067201604814444, + "loss": 1.8484, + "theoretical_loss": 4.511752330516086, + "tokens_seen": 159318016 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004806619859578736, + "loss": 1.8602, + "theoretical_loss": 4.5115134672908965, + "tokens_seen": 159383552 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004806519558676028, + "loss": 1.8087, + "theoretical_loss": 4.511274729750123, + "tokens_seen": 159449088 + }, + { + "epoch": 0.05, + "learning_rate": 0.000480641925777332, + "loss": 1.7861, + "theoretical_loss": 4.5110361177760145, + "tokens_seen": 159514624 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048063189568706117, + "loss": 1.6795, + "theoretical_loss": 4.5107976312509805, + "tokens_seen": 159580160 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004806218655967904, + "loss": 1.7966, + "theoretical_loss": 4.510559270057587, + "tokens_seen": 159645696 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048061183550651953, + "loss": 1.7123, + "theoretical_loss": 4.51032103407856, + "tokens_seen": 159711232 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048060180541624877, + "loss": 1.6985, + "theoretical_loss": 4.510082923196784, + "tokens_seen": 159776768 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048059177532597795, + "loss": 1.8127, + "theoretical_loss": 4.5098449372952985, + "tokens_seen": 159842304 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048058174523570713, + "loss": 1.668, + "theoretical_loss": 4.5096070762573, + "tokens_seen": 159907840 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004805717151454363, + "loss": 1.8089, + "theoretical_loss": 4.509369339966146, + "tokens_seen": 159973376 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004805616850551655, + "loss": 1.7617, + "theoretical_loss": 4.509131728305347, + "tokens_seen": 160038912 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048055165496489467, + "loss": 1.8743, + "theoretical_loss": 4.50889424115857, + "tokens_seen": 160104448 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004805416248746239, + "loss": 1.8471, + "theoretical_loss": 4.50865687840964, + "tokens_seen": 160169984 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048053159478435303, + "loss": 1.7943, + "theoretical_loss": 4.508419639942538, + "tokens_seen": 160235520 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048052156469408227, + "loss": 1.8348, + "theoretical_loss": 4.5081825256413985, + "tokens_seen": 160301056 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004805115346038114, + "loss": 1.8453, + "theoretical_loss": 4.507945535390512, + "tokens_seen": 160366592 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048050150451354063, + "loss": 1.7901, + "theoretical_loss": 4.507708669074326, + "tokens_seen": 160432128 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004804914744232698, + "loss": 1.6455, + "theoretical_loss": 4.5074719265774394, + "tokens_seen": 160497664 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 59852, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.8841290473937988, + "objective/train/theoretical_loss": 4.507235307784608, + "objective/train/tokens_used": 160563200, + "theoretical_loss": 4.507235307784608, + "tokens_seen": 160563200 + }, + { + "epoch": 0.05, + "learning_rate": 0.000480481444332999, + "loss": 1.9249, + "theoretical_loss": 4.507235307784608, + "tokens_seen": 160563200 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048047141424272823, + "loss": 1.7474, + "theoretical_loss": 4.5069988125807425, + "tokens_seen": 160628736 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048046138415245736, + "loss": 1.8845, + "theoretical_loss": 4.506762440850906, + "tokens_seen": 160694272 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004804513540621866, + "loss": 1.8212, + "theoretical_loss": 4.506526192480315, + "tokens_seen": 160759808 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048044132397191577, + "loss": 1.8573, + "theoretical_loss": 4.506290067354341, + "tokens_seen": 160825344 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048043129388164495, + "loss": 1.7524, + "theoretical_loss": 4.506054065358508, + "tokens_seen": 160890880 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048042126379137413, + "loss": 1.7915, + "theoretical_loss": 4.505818186378491, + "tokens_seen": 160956416 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048041123370110337, + "loss": 1.8093, + "theoretical_loss": 4.505582430300121, + "tokens_seen": 161021952 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004804012036108325, + "loss": 1.7014, + "theoretical_loss": 4.505346797009381, + "tokens_seen": 161087488 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048039117352056173, + "loss": 1.7543, + "theoretical_loss": 4.505111286392402, + "tokens_seen": 161153024 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048038114343029086, + "loss": 1.871, + "theoretical_loss": 4.50487589833547, + "tokens_seen": 161218560 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004803711133400201, + "loss": 1.8723, + "theoretical_loss": 4.504640632725025, + "tokens_seen": 161284096 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004803610832497493, + "loss": 1.6427, + "theoretical_loss": 4.504405489447654, + "tokens_seen": 161349632 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048035105315947846, + "loss": 1.719, + "theoretical_loss": 4.504170468390096, + "tokens_seen": 161415168 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048034102306920764, + "loss": 1.7396, + "theoretical_loss": 4.503935569439242, + "tokens_seen": 161480704 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004803309929789368, + "loss": 1.6693, + "theoretical_loss": 4.5037007924821335, + "tokens_seen": 161546240 + }, + { + "epoch": 0.05, + "learning_rate": 0.000480320962888666, + "loss": 1.8036, + "theoretical_loss": 4.503466137405961, + "tokens_seen": 161611776 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048031093279839523, + "loss": 1.7363, + "theoretical_loss": 4.503231604098065, + "tokens_seen": 161677312 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048030090270812436, + "loss": 1.7308, + "theoretical_loss": 4.50299719244594, + "tokens_seen": 161742848 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004802908726178536, + "loss": 1.7897, + "theoretical_loss": 4.502762902337222, + "tokens_seen": 161808384 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004802808425275827, + "loss": 1.7241, + "theoretical_loss": 4.502528733659704, + "tokens_seen": 161873920 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048027081243731196, + "loss": 1.8281, + "theoretical_loss": 4.502294686301323, + "tokens_seen": 161939456 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048026078234704114, + "loss": 1.8867, + "theoretical_loss": 4.502060760150167, + "tokens_seen": 162004992 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004802507522567703, + "loss": 1.8291, + "theoretical_loss": 4.501826955094472, + "tokens_seen": 162070528 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004802407221664995, + "loss": 1.6886, + "theoretical_loss": 4.501593271022623, + "tokens_seen": 162136064 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048023069207622874, + "loss": 1.6551, + "theoretical_loss": 4.501359707823149, + "tokens_seen": 162201600 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048022066198595786, + "loss": 1.8033, + "theoretical_loss": 4.501126265384732, + "tokens_seen": 162267136 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004802106318956871, + "loss": 1.8109, + "theoretical_loss": 4.500892943596199, + "tokens_seen": 162332672 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048020060180541623, + "loss": 1.858, + "theoretical_loss": 4.500659742346524, + "tokens_seen": 162398208 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048019057171514546, + "loss": 1.7372, + "theoretical_loss": 4.500426661524828, + "tokens_seen": 162463744 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048018054162487464, + "loss": 1.8322, + "theoretical_loss": 4.500193701020379, + "tokens_seen": 162529280 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004801705115346038, + "loss": 1.7341, + "theoretical_loss": 4.499960860722592, + "tokens_seen": 162594816 + }, + { + "epoch": 0.05, + "learning_rate": 0.000480160481444333, + "loss": 1.8099, + "theoretical_loss": 4.499728140521025, + "tokens_seen": 162660352 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004801504513540622, + "loss": 1.8583, + "theoretical_loss": 4.499495540305388, + "tokens_seen": 162725888 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048014042126379137, + "loss": 1.8115, + "theoretical_loss": 4.49926305996553, + "tokens_seen": 162791424 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004801303911735206, + "loss": 1.699, + "theoretical_loss": 4.499030699391449, + "tokens_seen": 162856960 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048012036108324973, + "loss": 1.6313, + "theoretical_loss": 4.49879845847329, + "tokens_seen": 162922496 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048011033099297897, + "loss": 1.8333, + "theoretical_loss": 4.498566337101337, + "tokens_seen": 162988032 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048010030090270815, + "loss": 1.8652, + "theoretical_loss": 4.4983343351660245, + "tokens_seen": 163053568 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048009027081243733, + "loss": 1.8478, + "theoretical_loss": 4.498102452557928, + "tokens_seen": 163119104 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004800802407221665, + "loss": 1.7655, + "theoretical_loss": 4.497870689167771, + "tokens_seen": 163184640 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004800702106318957, + "loss": 1.769, + "theoretical_loss": 4.497639044886412, + "tokens_seen": 163250176 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048006018054162487, + "loss": 1.8492, + "theoretical_loss": 4.497407519604866, + "tokens_seen": 163315712 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004800501504513541, + "loss": 1.7513, + "theoretical_loss": 4.49717611321428, + "tokens_seen": 163381248 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048004012036108323, + "loss": 1.7363, + "theoretical_loss": 4.496944825605951, + "tokens_seen": 163446784 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048003009027081247, + "loss": 1.6737, + "theoretical_loss": 4.496713656671317, + "tokens_seen": 163512320 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004800200601805416, + "loss": 1.7439, + "theoretical_loss": 4.496482606301957, + "tokens_seen": 163577856 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048001003009027083, + "loss": 1.5944, + "theoretical_loss": 4.496251674389596, + "tokens_seen": 163643392 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048, + "loss": 1.7285, + "theoretical_loss": 4.496020860826098, + "tokens_seen": 163708928 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004799899699097292, + "loss": 1.8778, + "theoretical_loss": 4.49579016550347, + "tokens_seen": 163774464 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 60958, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.814664602279663, + "objective/train/theoretical_loss": 4.49555958831386, + "objective/train/tokens_used": 163840000, + "theoretical_loss": 4.49555958831386, + "tokens_seen": 163840000 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004799799398194584, + "loss": 1.7879, + "theoretical_loss": 4.49555958831386, + "tokens_seen": 163840000 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047996990972918756, + "loss": 1.7001, + "theoretical_loss": 4.49532912914956, + "tokens_seen": 163905536 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047995987963891674, + "loss": 1.7373, + "theoretical_loss": 4.495098787903, + "tokens_seen": 163971072 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047994984954864597, + "loss": 1.8127, + "theoretical_loss": 4.4948685644667545, + "tokens_seen": 164036608 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004799398194583751, + "loss": 1.7129, + "theoretical_loss": 4.494638458733535, + "tokens_seen": 164102144 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047992978936810433, + "loss": 1.8363, + "theoretical_loss": 4.494408470596195, + "tokens_seen": 164167680 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004799197592778335, + "loss": 1.7219, + "theoretical_loss": 4.494178599947729, + "tokens_seen": 164233216 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004799097291875627, + "loss": 1.813, + "theoretical_loss": 4.4939488466812705, + "tokens_seen": 164298752 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004798996990972919, + "loss": 1.7728, + "theoretical_loss": 4.493719210690093, + "tokens_seen": 164364288 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047988966900702106, + "loss": 1.7117, + "theoretical_loss": 4.493489691867612, + "tokens_seen": 164429824 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047987963891675024, + "loss": 1.7989, + "theoretical_loss": 4.493260290107379, + "tokens_seen": 164495360 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004798696088264795, + "loss": 1.721, + "theoretical_loss": 4.493031005303084, + "tokens_seen": 164560896 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004798595787362086, + "loss": 1.6745, + "theoretical_loss": 4.49280183734856, + "tokens_seen": 164626432 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047984954864593784, + "loss": 1.8175, + "theoretical_loss": 4.492572786137774, + "tokens_seen": 164691968 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047983951855566696, + "loss": 1.8136, + "theoretical_loss": 4.492343851564835, + "tokens_seen": 164757504 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004798294884653962, + "loss": 1.7254, + "theoretical_loss": 4.492115033523987, + "tokens_seen": 164823040 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004798194583751254, + "loss": 1.8009, + "theoretical_loss": 4.491886331909615, + "tokens_seen": 164888576 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047980942828485456, + "loss": 1.5964, + "theoretical_loss": 4.491657746616241, + "tokens_seen": 164954112 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047979939819458374, + "loss": 1.7591, + "theoretical_loss": 4.49142927753852, + "tokens_seen": 165019648 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004797893681043129, + "loss": 1.8361, + "theoretical_loss": 4.491200924571251, + "tokens_seen": 165085184 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004797793380140421, + "loss": 1.7305, + "theoretical_loss": 4.490972687609364, + "tokens_seen": 165150720 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047976930792377134, + "loss": 1.7417, + "theoretical_loss": 4.4907445665479315, + "tokens_seen": 165216256 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047975927783350047, + "loss": 1.8325, + "theoretical_loss": 4.490516561282157, + "tokens_seen": 165281792 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004797492477432297, + "loss": 1.722, + "theoretical_loss": 4.4902886717073835, + "tokens_seen": 165347328 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004797392176529589, + "loss": 1.725, + "theoretical_loss": 4.49006089771909, + "tokens_seen": 165412864 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047972918756268807, + "loss": 1.6992, + "theoretical_loss": 4.489833239212888, + "tokens_seen": 165478400 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004797191574724173, + "loss": 1.9327, + "theoretical_loss": 4.48960569608453, + "tokens_seen": 165543936 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047970912738214643, + "loss": 1.8207, + "theoretical_loss": 4.489378268229897, + "tokens_seen": 165609472 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047969909729187566, + "loss": 1.6969, + "theoretical_loss": 4.489150955545012, + "tokens_seen": 165675008 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047968906720160484, + "loss": 1.7355, + "theoretical_loss": 4.488923757926031, + "tokens_seen": 165740544 + }, + { + "epoch": 0.05, + "learning_rate": 0.000479679037111334, + "loss": 1.7411, + "theoretical_loss": 4.4886966752692405, + "tokens_seen": 165806080 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004796690070210632, + "loss": 1.7963, + "theoretical_loss": 4.488469707471066, + "tokens_seen": 165871616 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004796589769307924, + "loss": 1.7764, + "theoretical_loss": 4.488242854428064, + "tokens_seen": 165937152 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047964894684052157, + "loss": 1.7635, + "theoretical_loss": 4.488016116036929, + "tokens_seen": 166002688 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004796389167502508, + "loss": 1.6976, + "theoretical_loss": 4.487789492194485, + "tokens_seen": 166068224 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047962888665997993, + "loss": 1.7895, + "theoretical_loss": 4.487562982797693, + "tokens_seen": 166133760 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047961885656970917, + "loss": 1.6893, + "theoretical_loss": 4.487336587743643, + "tokens_seen": 166199296 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047960882647943835, + "loss": 1.8206, + "theoretical_loss": 4.487110306929562, + "tokens_seen": 166264832 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047959879638916753, + "loss": 1.8627, + "theoretical_loss": 4.48688414025281, + "tokens_seen": 166330368 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004795887662988967, + "loss": 1.7055, + "theoretical_loss": 4.486658087610875, + "tokens_seen": 166395904 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004795787362086259, + "loss": 1.8153, + "theoretical_loss": 4.486432148901384, + "tokens_seen": 166461440 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047956870611835507, + "loss": 1.7496, + "theoretical_loss": 4.48620632402209, + "tokens_seen": 166526976 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004795586760280843, + "loss": 1.7712, + "theoretical_loss": 4.485980612870883, + "tokens_seen": 166592512 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047954864593781343, + "loss": 1.8321, + "theoretical_loss": 4.48575501534578, + "tokens_seen": 166658048 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047953861584754267, + "loss": 1.6686, + "theoretical_loss": 4.485529531344932, + "tokens_seen": 166723584 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004795285857572718, + "loss": 1.8087, + "theoretical_loss": 4.485304160766624, + "tokens_seen": 166789120 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047951855566700103, + "loss": 1.6878, + "theoretical_loss": 4.485078903509266, + "tokens_seen": 166854656 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004795085255767302, + "loss": 1.6987, + "theoretical_loss": 4.484853759471403, + "tokens_seen": 166920192 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004794984954864594, + "loss": 1.7291, + "theoretical_loss": 4.484628728551711, + "tokens_seen": 166985728 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004794884653961886, + "loss": 1.7313, + "theoretical_loss": 4.484403810648994, + "tokens_seen": 167051264 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 62196, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.7473068237304688, + "objective/train/theoretical_loss": 4.484179005662186, + "objective/train/tokens_used": 167116800, + "theoretical_loss": 4.484179005662186, + "tokens_seen": 167116800 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047947843530591776, + "loss": 1.7142, + "theoretical_loss": 4.484179005662186, + "tokens_seen": 167116800 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047946840521564694, + "loss": 1.8358, + "theoretical_loss": 4.483954313490354, + "tokens_seen": 167182336 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047945837512537617, + "loss": 1.6403, + "theoretical_loss": 4.483729734032694, + "tokens_seen": 167247872 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004794483450351053, + "loss": 1.6965, + "theoretical_loss": 4.483505267188528, + "tokens_seen": 167313408 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047943831494483453, + "loss": 1.8336, + "theoretical_loss": 4.48328091285731, + "tokens_seen": 167378944 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004794282848545637, + "loss": 1.7714, + "theoretical_loss": 4.483056670938625, + "tokens_seen": 167444480 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004794182547642929, + "loss": 1.7332, + "theoretical_loss": 4.482832541332183, + "tokens_seen": 167510016 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004794082246740221, + "loss": 1.8637, + "theoretical_loss": 4.482608523937824, + "tokens_seen": 167575552 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047939819458375126, + "loss": 1.6779, + "theoretical_loss": 4.4823846186555185, + "tokens_seen": 167641088 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047938816449348044, + "loss": 1.6831, + "theoretical_loss": 4.482160825385363, + "tokens_seen": 167706624 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004793781344032097, + "loss": 1.9025, + "theoretical_loss": 4.481937144027583, + "tokens_seen": 167772160 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004793681043129388, + "loss": 1.7615, + "theoretical_loss": 4.48171357448253, + "tokens_seen": 167837696 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047935807422266804, + "loss": 1.8034, + "theoretical_loss": 4.481490116650686, + "tokens_seen": 167903232 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047934804413239716, + "loss": 1.745, + "theoretical_loss": 4.48126677043266, + "tokens_seen": 167968768 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004793380140421264, + "loss": 1.8021, + "theoretical_loss": 4.481043535729183, + "tokens_seen": 168034304 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004793279839518556, + "loss": 1.7256, + "theoretical_loss": 4.480820412441122, + "tokens_seen": 168099840 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047931795386158476, + "loss": 1.7623, + "theoretical_loss": 4.4805974004694615, + "tokens_seen": 168165376 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047930792377131394, + "loss": 1.7014, + "theoretical_loss": 4.48037449971532, + "tokens_seen": 168230912 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004792978936810431, + "loss": 1.765, + "theoretical_loss": 4.480151710079937, + "tokens_seen": 168296448 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004792878635907723, + "loss": 1.8491, + "theoretical_loss": 4.479929031464682, + "tokens_seen": 168361984 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047927783350050154, + "loss": 1.6465, + "theoretical_loss": 4.479706463771047, + "tokens_seen": 168427520 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047926780341023067, + "loss": 1.8109, + "theoretical_loss": 4.479484006900652, + "tokens_seen": 168493056 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004792577733199599, + "loss": 1.818, + "theoretical_loss": 4.479261660755242, + "tokens_seen": 168558592 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004792477432296891, + "loss": 1.7685, + "theoretical_loss": 4.479039425236687, + "tokens_seen": 168624128 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047923771313941827, + "loss": 1.8796, + "theoretical_loss": 4.478817300246982, + "tokens_seen": 168689664 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047922768304914745, + "loss": 1.7191, + "theoretical_loss": 4.478595285688247, + "tokens_seen": 168755200 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047921765295887663, + "loss": 1.7265, + "theoretical_loss": 4.478373381462728, + "tokens_seen": 168820736 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004792076228686058, + "loss": 1.757, + "theoretical_loss": 4.478151587472791, + "tokens_seen": 168886272 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047919759277833504, + "loss": 1.7565, + "theoretical_loss": 4.477929903620931, + "tokens_seen": 168951808 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047918756268806417, + "loss": 1.7686, + "theoretical_loss": 4.477708329809766, + "tokens_seen": 169017344 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004791775325977934, + "loss": 1.7891, + "theoretical_loss": 4.477486865942036, + "tokens_seen": 169082880 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047916750250752253, + "loss": 1.6079, + "theoretical_loss": 4.477265511920607, + "tokens_seen": 169148416 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047915747241725177, + "loss": 1.7793, + "theoretical_loss": 4.477044267648466, + "tokens_seen": 169213952 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047914744232698095, + "loss": 1.8025, + "theoretical_loss": 4.476823133028725, + "tokens_seen": 169279488 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047913741223671013, + "loss": 1.7462, + "theoretical_loss": 4.47660210796462, + "tokens_seen": 169345024 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004791273821464393, + "loss": 1.8382, + "theoretical_loss": 4.476381192359504, + "tokens_seen": 169410560 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047911735205616855, + "loss": 1.7828, + "theoretical_loss": 4.476160386116861, + "tokens_seen": 169476096 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004791073219658977, + "loss": 1.6977, + "theoretical_loss": 4.475939689140291, + "tokens_seen": 169541632 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004790972918756269, + "loss": 1.8201, + "theoretical_loss": 4.475719101333519, + "tokens_seen": 169607168 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047908726178535604, + "loss": 1.7552, + "theoretical_loss": 4.475498622600391, + "tokens_seen": 169672704 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047907723169508527, + "loss": 1.6436, + "theoretical_loss": 4.475278252844875, + "tokens_seen": 169738240 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047906720160481445, + "loss": 1.7742, + "theoretical_loss": 4.475057991971062, + "tokens_seen": 169803776 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047905717151454363, + "loss": 1.7988, + "theoretical_loss": 4.474837839883162, + "tokens_seen": 169869312 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004790471414242728, + "loss": 1.8448, + "theoretical_loss": 4.474617796485507, + "tokens_seen": 169934848 + }, + { + "epoch": 0.05, + "learning_rate": 0.000479037111334002, + "loss": 1.6676, + "theoretical_loss": 4.474397861682552, + "tokens_seen": 170000384 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004790270812437312, + "loss": 1.6759, + "theoretical_loss": 4.474178035378869, + "tokens_seen": 170065920 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004790170511534604, + "loss": 1.6593, + "theoretical_loss": 4.473958317479154, + "tokens_seen": 170131456 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047900702106318954, + "loss": 1.8276, + "theoretical_loss": 4.473738707888221, + "tokens_seen": 170196992 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004789969909729188, + "loss": 1.7558, + "theoretical_loss": 4.473519206511006, + "tokens_seen": 170262528 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004789869608826479, + "loss": 1.9309, + "theoretical_loss": 4.473299813252565, + "tokens_seen": 170328064 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 63049, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6509816646575928, + "objective/train/theoretical_loss": 4.473080528018071, + "objective/train/tokens_used": 170393600, + "theoretical_loss": 4.473080528018071, + "tokens_seen": 170393600 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047897693079237714, + "loss": 1.6221, + "theoretical_loss": 4.473080528018071, + "tokens_seen": 170393600 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047896690070210637, + "loss": 1.7405, + "theoretical_loss": 4.4728613507128205, + "tokens_seen": 170459136 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004789568706118355, + "loss": 1.6334, + "theoretical_loss": 4.472642281242226, + "tokens_seen": 170524672 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047894684052156473, + "loss": 1.735, + "theoretical_loss": 4.472423319511822, + "tokens_seen": 170590208 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004789368104312939, + "loss": 1.7371, + "theoretical_loss": 4.472204465427261, + "tokens_seen": 170655744 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004789267803410231, + "loss": 1.7412, + "theoretical_loss": 4.471985718894312, + "tokens_seen": 170721280 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004789167502507523, + "loss": 1.6436, + "theoretical_loss": 4.471767079818866, + "tokens_seen": 170786816 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047890672016048146, + "loss": 1.7148, + "theoretical_loss": 4.471548548106932, + "tokens_seen": 170852352 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047889669007021064, + "loss": 1.7169, + "theoretical_loss": 4.471330123664634, + "tokens_seen": 170917888 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004788866599799399, + "loss": 1.6741, + "theoretical_loss": 4.471111806398218, + "tokens_seen": 170983424 + }, + { + "epoch": 0.05, + "learning_rate": 0.000478876629889669, + "loss": 1.6195, + "theoretical_loss": 4.4708935962140455, + "tokens_seen": 171048960 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047886659979939824, + "loss": 1.7416, + "theoretical_loss": 4.4706754930185975, + "tokens_seen": 171114496 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047885656970912736, + "loss": 1.6202, + "theoretical_loss": 4.47045749671847, + "tokens_seen": 171180032 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004788465396188566, + "loss": 1.7588, + "theoretical_loss": 4.470239607220377, + "tokens_seen": 171245568 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004788365095285858, + "loss": 1.736, + "theoretical_loss": 4.470021824431152, + "tokens_seen": 171311104 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047882647943831496, + "loss": 1.7982, + "theoretical_loss": 4.469804148257742, + "tokens_seen": 171376640 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047881644934804414, + "loss": 1.5785, + "theoretical_loss": 4.469586578607213, + "tokens_seen": 171442176 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004788064192577733, + "loss": 1.9307, + "theoretical_loss": 4.469369115386746, + "tokens_seen": 171507712 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004787963891675025, + "loss": 1.7518, + "theoretical_loss": 4.46915175850364, + "tokens_seen": 171573248 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047878635907723174, + "loss": 1.9191, + "theoretical_loss": 4.468934507865308, + "tokens_seen": 171638784 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047877632898696087, + "loss": 1.8216, + "theoretical_loss": 4.46871736337928, + "tokens_seen": 171704320 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004787662988966901, + "loss": 1.6854, + "theoretical_loss": 4.468500324953202, + "tokens_seen": 171769856 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004787562688064193, + "loss": 1.6654, + "theoretical_loss": 4.468283392494836, + "tokens_seen": 171835392 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047874623871614847, + "loss": 1.6985, + "theoretical_loss": 4.468066565912057, + "tokens_seen": 171900928 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047873620862587765, + "loss": 1.7941, + "theoretical_loss": 4.467849845112859, + "tokens_seen": 171966464 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047872617853560683, + "loss": 1.7861, + "theoretical_loss": 4.467633230005347, + "tokens_seen": 172032000 + }, + { + "epoch": 0.05, + "learning_rate": 0.000478716148445336, + "loss": 1.8398, + "theoretical_loss": 4.467416720497743, + "tokens_seen": 172097536 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047870611835506524, + "loss": 1.8446, + "theoretical_loss": 4.467200316498385, + "tokens_seen": 172163072 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047869608826479437, + "loss": 1.8295, + "theoretical_loss": 4.4669840179157205, + "tokens_seen": 172228608 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004786860581745236, + "loss": 1.7404, + "theoretical_loss": 4.466767824658317, + "tokens_seen": 172294144 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047867602808425273, + "loss": 1.8683, + "theoretical_loss": 4.466551736634852, + "tokens_seen": 172359680 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047866599799398197, + "loss": 1.6781, + "theoretical_loss": 4.466335753754118, + "tokens_seen": 172425216 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047865596790371115, + "loss": 1.5626, + "theoretical_loss": 4.466119875925022, + "tokens_seen": 172490752 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047864593781344033, + "loss": 1.8035, + "theoretical_loss": 4.465904103056586, + "tokens_seen": 172556288 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004786359077231695, + "loss": 1.6792, + "theoretical_loss": 4.465688435057938, + "tokens_seen": 172621824 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047862587763289875, + "loss": 1.9781, + "theoretical_loss": 4.46547287183833, + "tokens_seen": 172687360 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004786158475426279, + "loss": 1.7554, + "theoretical_loss": 4.465257413307119, + "tokens_seen": 172752896 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004786058174523571, + "loss": 1.8021, + "theoretical_loss": 4.465042059373776, + "tokens_seen": 172818432 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047859578736208624, + "loss": 1.7989, + "theoretical_loss": 4.464826809947887, + "tokens_seen": 172883968 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047858575727181547, + "loss": 1.7861, + "theoretical_loss": 4.464611664939149, + "tokens_seen": 172949504 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047857572718154465, + "loss": 1.6611, + "theoretical_loss": 4.46439662425737, + "tokens_seen": 173015040 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047856569709127383, + "loss": 1.7747, + "theoretical_loss": 4.464181687812474, + "tokens_seen": 173080576 + }, + { + "epoch": 0.05, + "learning_rate": 0.000478555667001003, + "loss": 1.7891, + "theoretical_loss": 4.4639668555144905, + "tokens_seen": 173146112 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004785456369107322, + "loss": 1.6157, + "theoretical_loss": 4.463752127273567, + "tokens_seen": 173211648 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004785356068204614, + "loss": 1.6935, + "theoretical_loss": 4.463537502999959, + "tokens_seen": 173277184 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004785255767301906, + "loss": 1.6833, + "theoretical_loss": 4.4633229826040335, + "tokens_seen": 173342720 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047851554663991974, + "loss": 1.8438, + "theoretical_loss": 4.46310856599627, + "tokens_seen": 173408256 + }, + { + "epoch": 0.05, + "learning_rate": 0.000478505516549649, + "loss": 1.6537, + "theoretical_loss": 4.462894253087258, + "tokens_seen": 173473792 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004784954864593781, + "loss": 1.6894, + "theoretical_loss": 4.462680043787699, + "tokens_seen": 173539328 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047848545636910734, + "loss": 1.8058, + "theoretical_loss": 4.4624659380084015, + "tokens_seen": 173604864 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 64190, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.905247688293457, + "objective/train/theoretical_loss": 4.462251935660287, + "objective/train/tokens_used": 173670400, + "theoretical_loss": 4.462251935660287, + "tokens_seen": 173670400 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004784754262788365, + "loss": 1.7999, + "theoretical_loss": 4.462251935660287, + "tokens_seen": 173670400 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004784653961885657, + "loss": 1.8107, + "theoretical_loss": 4.46203803665439, + "tokens_seen": 173735936 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004784553660982949, + "loss": 1.7495, + "theoretical_loss": 4.4618242409018505, + "tokens_seen": 173801472 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004784453360080241, + "loss": 1.7006, + "theoretical_loss": 4.461610548313919, + "tokens_seen": 173867008 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047843530591775324, + "loss": 1.6927, + "theoretical_loss": 4.461396958801959, + "tokens_seen": 173932544 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004784252758274825, + "loss": 1.6038, + "theoretical_loss": 4.461183472277439, + "tokens_seen": 173998080 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004784152457372116, + "loss": 1.8207, + "theoretical_loss": 4.460970088651941, + "tokens_seen": 174063616 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047840521564694084, + "loss": 1.7033, + "theoretical_loss": 4.460756807837154, + "tokens_seen": 174129152 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047839518555667, + "loss": 1.8663, + "theoretical_loss": 4.460543629744874, + "tokens_seen": 174194688 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004783851554663992, + "loss": 1.6682, + "theoretical_loss": 4.460330554287011, + "tokens_seen": 174260224 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004783751253761284, + "loss": 1.8143, + "theoretical_loss": 4.460117581375579, + "tokens_seen": 174325760 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047836509528585756, + "loss": 1.728, + "theoretical_loss": 4.459904710922704, + "tokens_seen": 174391296 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047835506519558675, + "loss": 1.6472, + "theoretical_loss": 4.459691942840616, + "tokens_seen": 174456832 + }, + { + "epoch": 0.05, + "learning_rate": 0.000478345035105316, + "loss": 1.7599, + "theoretical_loss": 4.459479277041657, + "tokens_seen": 174522368 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004783350050150451, + "loss": 1.765, + "theoretical_loss": 4.459266713438277, + "tokens_seen": 174587904 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047832497492477434, + "loss": 1.8646, + "theoretical_loss": 4.459054251943029, + "tokens_seen": 174653440 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047831494483450347, + "loss": 1.7044, + "theoretical_loss": 4.4588418924685795, + "tokens_seen": 174718976 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004783049147442327, + "loss": 1.7061, + "theoretical_loss": 4.458629634927698, + "tokens_seen": 174784512 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004782948846539619, + "loss": 1.7504, + "theoretical_loss": 4.458417479233265, + "tokens_seen": 174850048 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047828485456369107, + "loss": 1.8174, + "theoretical_loss": 4.4582054252982655, + "tokens_seen": 174915584 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047827482447342025, + "loss": 1.6781, + "theoretical_loss": 4.457993473035792, + "tokens_seen": 174981120 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004782647943831495, + "loss": 1.7393, + "theoretical_loss": 4.457781622359043, + "tokens_seen": 175046656 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004782547642928786, + "loss": 1.8036, + "theoretical_loss": 4.457569873181326, + "tokens_seen": 175112192 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047824473420260785, + "loss": 1.7415, + "theoretical_loss": 4.457358225416052, + "tokens_seen": 175177728 + }, + { + "epoch": 0.05, + "learning_rate": 0.000478234704112337, + "loss": 1.8478, + "theoretical_loss": 4.4571466789767396, + "tokens_seen": 175243264 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004782246740220662, + "loss": 1.8129, + "theoretical_loss": 4.456935233777013, + "tokens_seen": 175308800 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047821464393179544, + "loss": 1.692, + "theoretical_loss": 4.456723889730605, + "tokens_seen": 175374336 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047820461384152457, + "loss": 1.7632, + "theoretical_loss": 4.456512646751349, + "tokens_seen": 175439872 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004781945837512538, + "loss": 1.8257, + "theoretical_loss": 4.456301504753188, + "tokens_seen": 175505408 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047818455366098293, + "loss": 1.7706, + "theoretical_loss": 4.456090463650169, + "tokens_seen": 175570944 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047817452357071217, + "loss": 1.7469, + "theoretical_loss": 4.455879523356444, + "tokens_seen": 175636480 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047816449348044135, + "loss": 1.7731, + "theoretical_loss": 4.4556686837862705, + "tokens_seen": 175702016 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047815446339017053, + "loss": 1.7062, + "theoretical_loss": 4.455457944854011, + "tokens_seen": 175767552 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004781444332998997, + "loss": 1.6288, + "theoretical_loss": 4.455247306474131, + "tokens_seen": 175833088 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047813440320962895, + "loss": 1.7359, + "theoretical_loss": 4.455036768561204, + "tokens_seen": 175898624 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004781243731193581, + "loss": 1.736, + "theoretical_loss": 4.4548263310299046, + "tokens_seen": 175964160 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004781143430290873, + "loss": 1.6102, + "theoretical_loss": 4.454615993795015, + "tokens_seen": 176029696 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047810431293881644, + "loss": 1.6857, + "theoretical_loss": 4.454405756771415, + "tokens_seen": 176095232 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047809428284854567, + "loss": 1.8232, + "theoretical_loss": 4.454195619874096, + "tokens_seen": 176160768 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047808425275827485, + "loss": 1.628, + "theoretical_loss": 4.453985583018149, + "tokens_seen": 176226304 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047807422266800403, + "loss": 1.8036, + "theoretical_loss": 4.4537756461187685, + "tokens_seen": 176291840 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004780641925777332, + "loss": 1.6914, + "theoretical_loss": 4.453565809091254, + "tokens_seen": 176357376 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004780541624874624, + "loss": 1.6869, + "theoretical_loss": 4.453356071851006, + "tokens_seen": 176422912 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004780441323971916, + "loss": 1.7117, + "theoretical_loss": 4.453146434313531, + "tokens_seen": 176488448 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004780341023069208, + "loss": 1.6997, + "theoretical_loss": 4.452936896394435, + "tokens_seen": 176553984 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047802407221664994, + "loss": 1.6754, + "theoretical_loss": 4.452727458009431, + "tokens_seen": 176619520 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004780140421263792, + "loss": 1.6841, + "theoretical_loss": 4.45251811907433, + "tokens_seen": 176685056 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004780040120361083, + "loss": 1.7351, + "theoretical_loss": 4.452308879505047, + "tokens_seen": 176750592 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047799398194583754, + "loss": 1.7362, + "theoretical_loss": 4.452099739217601, + "tokens_seen": 176816128 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004779839518555667, + "loss": 1.7321, + "theoretical_loss": 4.451890698128112, + "tokens_seen": 176881664 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 65168, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6575524806976318, + "objective/train/theoretical_loss": 4.4516817561528015, + "objective/train/tokens_used": 176947200, + "theoretical_loss": 4.4516817561528015, + "tokens_seen": 176947200 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004779739217652959, + "loss": 1.755, + "theoretical_loss": 4.4516817561528015, + "tokens_seen": 176947200 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004779638916750251, + "loss": 1.7322, + "theoretical_loss": 4.451472913207992, + "tokens_seen": 177012736 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004779538615847543, + "loss": 1.8008, + "theoretical_loss": 4.451264169210107, + "tokens_seen": 177078272 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047794383149448344, + "loss": 1.7325, + "theoretical_loss": 4.451055524075677, + "tokens_seen": 177143808 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004779338014042127, + "loss": 1.7064, + "theoretical_loss": 4.450846977721326, + "tokens_seen": 177209344 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004779237713139418, + "loss": 1.7734, + "theoretical_loss": 4.450638530063786, + "tokens_seen": 177274880 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047791374122367104, + "loss": 1.6175, + "theoretical_loss": 4.450430181019884, + "tokens_seen": 177340416 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004779037111334002, + "loss": 1.6961, + "theoretical_loss": 4.450221930506553, + "tokens_seen": 177405952 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004778936810431294, + "loss": 1.8276, + "theoretical_loss": 4.450013778440822, + "tokens_seen": 177471488 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004778836509528586, + "loss": 1.6936, + "theoretical_loss": 4.449805724739824, + "tokens_seen": 177537024 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047787362086258776, + "loss": 1.7107, + "theoretical_loss": 4.449597769320791, + "tokens_seen": 177602560 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047786359077231695, + "loss": 1.8828, + "theoretical_loss": 4.4493899121010525, + "tokens_seen": 177668096 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004778535606820462, + "loss": 1.7375, + "theoretical_loss": 4.449182152998044, + "tokens_seen": 177733632 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004778435305917753, + "loss": 1.8682, + "theoretical_loss": 4.448974491929297, + "tokens_seen": 177799168 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047783350050150454, + "loss": 1.7464, + "theoretical_loss": 4.448766928812441, + "tokens_seen": 177864704 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047782347041123367, + "loss": 1.7386, + "theoretical_loss": 4.448559463565209, + "tokens_seen": 177930240 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004778134403209629, + "loss": 1.8498, + "theoretical_loss": 4.44835209610543, + "tokens_seen": 177995776 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004778034102306921, + "loss": 1.7064, + "theoretical_loss": 4.448144826351036, + "tokens_seen": 178061312 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047779338014042127, + "loss": 1.9112, + "theoretical_loss": 4.447937654220054, + "tokens_seen": 178126848 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047778335005015045, + "loss": 1.8114, + "theoretical_loss": 4.447730579630612, + "tokens_seen": 178192384 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004777733199598797, + "loss": 1.8313, + "theoretical_loss": 4.447523602500936, + "tokens_seen": 178257920 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004777632898696088, + "loss": 1.8404, + "theoretical_loss": 4.447316722749351, + "tokens_seen": 178323456 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047775325977933805, + "loss": 1.7107, + "theoretical_loss": 4.447109940294283, + "tokens_seen": 178388992 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004777432296890672, + "loss": 1.8407, + "theoretical_loss": 4.44690325505425, + "tokens_seen": 178454528 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004777331995987964, + "loss": 1.803, + "theoretical_loss": 4.446696666947874, + "tokens_seen": 178520064 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004777231695085256, + "loss": 1.732, + "theoretical_loss": 4.446490175893872, + "tokens_seen": 178585600 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047771313941825477, + "loss": 1.7809, + "theoretical_loss": 4.44628378181106, + "tokens_seen": 178651136 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047770310932798395, + "loss": 1.6496, + "theoretical_loss": 4.4460774846183515, + "tokens_seen": 178716672 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047769307923771313, + "loss": 1.5935, + "theoretical_loss": 4.445871284234757, + "tokens_seen": 178782208 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004776830491474423, + "loss": 1.6116, + "theoretical_loss": 4.445665180579383, + "tokens_seen": 178847744 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047767301905717155, + "loss": 1.7819, + "theoretical_loss": 4.445459173571438, + "tokens_seen": 178913280 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004776629889669007, + "loss": 1.7001, + "theoretical_loss": 4.445253263130223, + "tokens_seen": 178978816 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004776529588766299, + "loss": 1.7038, + "theoretical_loss": 4.445047449175136, + "tokens_seen": 179044352 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047764292878635904, + "loss": 1.8076, + "theoretical_loss": 4.444841731625674, + "tokens_seen": 179109888 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004776328986960883, + "loss": 1.6094, + "theoretical_loss": 4.444636110401431, + "tokens_seen": 179175424 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047762286860581746, + "loss": 1.7365, + "theoretical_loss": 4.444430585422094, + "tokens_seen": 179240960 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047761283851554664, + "loss": 1.6876, + "theoretical_loss": 4.444225156607449, + "tokens_seen": 179306496 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004776028084252758, + "loss": 1.7106, + "theoretical_loss": 4.444019823877378, + "tokens_seen": 179372032 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047759277833500505, + "loss": 1.7366, + "theoretical_loss": 4.4438145871518575, + "tokens_seen": 179437568 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004775827482447342, + "loss": 1.7992, + "theoretical_loss": 4.443609446350962, + "tokens_seen": 179503104 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004775727181544634, + "loss": 1.6042, + "theoretical_loss": 4.44340440139486, + "tokens_seen": 179568640 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047756268806419254, + "loss": 1.5676, + "theoretical_loss": 4.443199452203816, + "tokens_seen": 179634176 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004775526579739218, + "loss": 1.7668, + "theoretical_loss": 4.442994598698189, + "tokens_seen": 179699712 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047754262788365096, + "loss": 1.6371, + "theoretical_loss": 4.4427898407984365, + "tokens_seen": 179765248 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047753259779338014, + "loss": 1.7349, + "theoretical_loss": 4.442585178425107, + "tokens_seen": 179830784 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004775225677031093, + "loss": 1.7594, + "theoretical_loss": 4.442380611498846, + "tokens_seen": 179896320 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004775125376128385, + "loss": 1.6032, + "theoretical_loss": 4.442176139940393, + "tokens_seen": 179961856 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004775025075225677, + "loss": 1.5516, + "theoretical_loss": 4.441971763670585, + "tokens_seen": 180027392 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004774924774322969, + "loss": 1.7311, + "theoretical_loss": 4.441767482610349, + "tokens_seen": 180092928 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047748244734202605, + "loss": 1.7219, + "theoretical_loss": 4.441563296680709, + "tokens_seen": 180158464 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 66455, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4508898258209229, + "objective/train/theoretical_loss": 4.441359205802783, + "objective/train/tokens_used": 180224000, + "theoretical_loss": 4.441359205802783, + "tokens_seen": 180224000 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004774724172517553, + "loss": 1.5525, + "theoretical_loss": 4.441359205802783, + "tokens_seen": 180224000 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004774623871614845, + "loss": 1.739, + "theoretical_loss": 4.441155209897782, + "tokens_seen": 180289536 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047745235707121364, + "loss": 1.704, + "theoretical_loss": 4.440951308887014, + "tokens_seen": 180355072 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004774423269809429, + "loss": 1.7423, + "theoretical_loss": 4.440747502691876, + "tokens_seen": 180420608 + }, + { + "epoch": 0.05, + "learning_rate": 0.000477432296890672, + "loss": 1.6482, + "theoretical_loss": 4.440543791233862, + "tokens_seen": 180486144 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047742226680040124, + "loss": 1.5773, + "theoretical_loss": 4.440340174434558, + "tokens_seen": 180551680 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004774122367101304, + "loss": 1.7478, + "theoretical_loss": 4.440136652215646, + "tokens_seen": 180617216 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004774022066198596, + "loss": 1.6937, + "theoretical_loss": 4.439933224498898, + "tokens_seen": 180682752 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004773921765295888, + "loss": 1.75, + "theoretical_loss": 4.43972989120618, + "tokens_seen": 180748288 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047738214643931797, + "loss": 1.8268, + "theoretical_loss": 4.439526652259451, + "tokens_seen": 180813824 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047737211634904715, + "loss": 1.7219, + "theoretical_loss": 4.439323507580761, + "tokens_seen": 180879360 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004773620862587764, + "loss": 1.7118, + "theoretical_loss": 4.439120457092258, + "tokens_seen": 180944896 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004773520561685055, + "loss": 1.7026, + "theoretical_loss": 4.438917500716178, + "tokens_seen": 181010432 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047734202607823474, + "loss": 1.6956, + "theoretical_loss": 4.438714638374849, + "tokens_seen": 181075968 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047733199598796387, + "loss": 1.5764, + "theoretical_loss": 4.438511869990693, + "tokens_seen": 181141504 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004773219658976931, + "loss": 1.9034, + "theoretical_loss": 4.438309195486223, + "tokens_seen": 181207040 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004773119358074223, + "loss": 1.5355, + "theoretical_loss": 4.438106614784045, + "tokens_seen": 181272576 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047730190571715147, + "loss": 1.7821, + "theoretical_loss": 4.437904127806856, + "tokens_seen": 181338112 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047729187562688065, + "loss": 1.7275, + "theoretical_loss": 4.437701734477445, + "tokens_seen": 181403648 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004772818455366099, + "loss": 1.7284, + "theoretical_loss": 4.437499434718692, + "tokens_seen": 181469184 + }, + { + "epoch": 0.06, + "learning_rate": 0.000477271815446339, + "loss": 1.5954, + "theoretical_loss": 4.437297228453567, + "tokens_seen": 181534720 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047726178535606825, + "loss": 1.7116, + "theoretical_loss": 4.4370951156051355, + "tokens_seen": 181600256 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004772517552657974, + "loss": 1.7987, + "theoretical_loss": 4.43689309609655, + "tokens_seen": 181665792 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004772417251755266, + "loss": 1.653, + "theoretical_loss": 4.436691169851055, + "tokens_seen": 181731328 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004772316950852558, + "loss": 1.765, + "theoretical_loss": 4.436489336791985, + "tokens_seen": 181796864 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047722166499498497, + "loss": 1.7664, + "theoretical_loss": 4.436287596842769, + "tokens_seen": 181862400 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047721163490471415, + "loss": 1.7614, + "theoretical_loss": 4.4360859499269205, + "tokens_seen": 181927936 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047720160481444333, + "loss": 1.7193, + "theoretical_loss": 4.435884395968047, + "tokens_seen": 181993472 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004771915747241725, + "loss": 1.7326, + "theoretical_loss": 4.435682934889847, + "tokens_seen": 182059008 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047718154463390175, + "loss": 1.5868, + "theoretical_loss": 4.435481566616108, + "tokens_seen": 182124544 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004771715145436309, + "loss": 1.6393, + "theoretical_loss": 4.435280291070705, + "tokens_seen": 182190080 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004771614844533601, + "loss": 1.8356, + "theoretical_loss": 4.435079108177606, + "tokens_seen": 182255616 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047715145436308924, + "loss": 1.5808, + "theoretical_loss": 4.434878017860869, + "tokens_seen": 182321152 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004771414242728185, + "loss": 1.6308, + "theoretical_loss": 4.434677020044637, + "tokens_seen": 182386688 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047713139418254766, + "loss": 1.663, + "theoretical_loss": 4.4344761146531475, + "tokens_seen": 182452224 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047712136409227684, + "loss": 1.6122, + "theoretical_loss": 4.434275301610725, + "tokens_seen": 182517760 + }, + { + "epoch": 0.06, + "learning_rate": 0.000477111334002006, + "loss": 1.7023, + "theoretical_loss": 4.434074580841784, + "tokens_seen": 182583296 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047710130391173525, + "loss": 1.8381, + "theoretical_loss": 4.433873952270826, + "tokens_seen": 182648832 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004770912738214644, + "loss": 1.6709, + "theoretical_loss": 4.433673415822443, + "tokens_seen": 182714368 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004770812437311936, + "loss": 1.7333, + "theoretical_loss": 4.433472971421315, + "tokens_seen": 182779904 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047707121364092274, + "loss": 1.7341, + "theoretical_loss": 4.433272618992213, + "tokens_seen": 182845440 + }, + { + "epoch": 0.06, + "learning_rate": 0.000477061183550652, + "loss": 1.7785, + "theoretical_loss": 4.433072358459991, + "tokens_seen": 182910976 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047705115346038116, + "loss": 1.732, + "theoretical_loss": 4.432872189749596, + "tokens_seen": 182976512 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047704112337011034, + "loss": 1.7458, + "theoretical_loss": 4.432672112786063, + "tokens_seen": 183042048 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004770310932798395, + "loss": 1.8166, + "theoretical_loss": 4.4324721274945125, + "tokens_seen": 183107584 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004770210631895687, + "loss": 1.7133, + "theoretical_loss": 4.432272233800154, + "tokens_seen": 183173120 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004770110330992979, + "loss": 1.6666, + "theoretical_loss": 4.432072431628286, + "tokens_seen": 183238656 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004770010030090271, + "loss": 1.6839, + "theoretical_loss": 4.431872720904292, + "tokens_seen": 183304192 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047699097291875625, + "loss": 1.7471, + "theoretical_loss": 4.4316731015536455, + "tokens_seen": 183369728 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004769809428284855, + "loss": 1.6988, + "theoretical_loss": 4.431473573501906, + "tokens_seen": 183435264 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 67310, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5619434118270874, + "objective/train/theoretical_loss": 4.431274136674721, + "objective/train/tokens_used": 183500800, + "theoretical_loss": 4.431274136674721, + "tokens_seen": 183500800 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047697091273821466, + "loss": 1.6536, + "theoretical_loss": 4.431274136674721, + "tokens_seen": 183500800 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047696088264794384, + "loss": 1.7419, + "theoretical_loss": 4.431074790997823, + "tokens_seen": 183566336 + }, + { + "epoch": 0.06, + "learning_rate": 0.000476950852557673, + "loss": 1.6826, + "theoretical_loss": 4.430875536397036, + "tokens_seen": 183631872 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004769408224674022, + "loss": 1.6499, + "theoretical_loss": 4.430676372798265, + "tokens_seen": 183697408 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004769307923771314, + "loss": 1.7645, + "theoretical_loss": 4.430477300127507, + "tokens_seen": 183762944 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004769207622868606, + "loss": 1.6285, + "theoretical_loss": 4.43027831831084, + "tokens_seen": 183828480 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047691073219658975, + "loss": 1.7464, + "theoretical_loss": 4.430079427274435, + "tokens_seen": 183894016 + }, + { + "epoch": 0.06, + "learning_rate": 0.000476900702106319, + "loss": 1.7222, + "theoretical_loss": 4.429880626944543, + "tokens_seen": 183959552 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004768906720160481, + "loss": 1.5522, + "theoretical_loss": 4.429681917247506, + "tokens_seen": 184025088 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047688064192577735, + "loss": 1.591, + "theoretical_loss": 4.429483298109749, + "tokens_seen": 184090624 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047687061183550653, + "loss": 1.6761, + "theoretical_loss": 4.429284769457784, + "tokens_seen": 184156160 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004768605817452357, + "loss": 1.6563, + "theoretical_loss": 4.429086331218208, + "tokens_seen": 184221696 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004768505516549649, + "loss": 1.7502, + "theoretical_loss": 4.428887983317706, + "tokens_seen": 184287232 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047684052156469407, + "loss": 1.6602, + "theoretical_loss": 4.4286897256830455, + "tokens_seen": 184352768 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047683049147442325, + "loss": 1.6507, + "theoretical_loss": 4.428491558241081, + "tokens_seen": 184418304 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004768204613841525, + "loss": 1.6385, + "theoretical_loss": 4.428293480918751, + "tokens_seen": 184483840 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004768104312938816, + "loss": 1.6467, + "theoretical_loss": 4.428095493643081, + "tokens_seen": 184549376 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047680040120361085, + "loss": 1.7385, + "theoretical_loss": 4.427897596341181, + "tokens_seen": 184614912 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047679037111334003, + "loss": 1.6381, + "theoretical_loss": 4.427699788940243, + "tokens_seen": 184680448 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004767803410230692, + "loss": 1.8051, + "theoretical_loss": 4.427502071367549, + "tokens_seen": 184745984 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004767703109327984, + "loss": 1.6393, + "theoretical_loss": 4.42730444355046, + "tokens_seen": 184811520 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004767602808425276, + "loss": 1.6886, + "theoretical_loss": 4.4271069054164265, + "tokens_seen": 184877056 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047675025075225676, + "loss": 1.7484, + "theoretical_loss": 4.426909456892979, + "tokens_seen": 184942592 + }, + { + "epoch": 0.06, + "learning_rate": 0.000476740220661986, + "loss": 1.7021, + "theoretical_loss": 4.426712097907735, + "tokens_seen": 185008128 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004767301905717151, + "loss": 1.794, + "theoretical_loss": 4.426514828388394, + "tokens_seen": 185073664 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047672016048144435, + "loss": 1.6625, + "theoretical_loss": 4.426317648262743, + "tokens_seen": 185139200 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047671013039117353, + "loss": 1.8044, + "theoretical_loss": 4.426120557458649, + "tokens_seen": 185204736 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004767001003009027, + "loss": 1.7012, + "theoretical_loss": 4.425923555904065, + "tokens_seen": 185270272 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047669007021063195, + "loss": 1.6919, + "theoretical_loss": 4.4257266435270255, + "tokens_seen": 185335808 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004766800401203611, + "loss": 1.7579, + "theoretical_loss": 4.425529820255651, + "tokens_seen": 185401344 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004766700100300903, + "loss": 1.692, + "theoretical_loss": 4.425333086018144, + "tokens_seen": 185466880 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047665997993981944, + "loss": 1.6692, + "theoretical_loss": 4.425136440742788, + "tokens_seen": 185532416 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004766499498495487, + "loss": 1.7819, + "theoretical_loss": 4.424939884357956, + "tokens_seen": 185597952 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047663991975927786, + "loss": 1.5786, + "theoretical_loss": 4.424743416792096, + "tokens_seen": 185663488 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047662988966900704, + "loss": 1.6545, + "theoretical_loss": 4.424547037973745, + "tokens_seen": 185729024 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004766198595787362, + "loss": 1.7548, + "theoretical_loss": 4.424350747831518, + "tokens_seen": 185794560 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047660982948846545, + "loss": 1.6611, + "theoretical_loss": 4.424154546294117, + "tokens_seen": 185860096 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765997993981946, + "loss": 1.7165, + "theoretical_loss": 4.4239584332903235, + "tokens_seen": 185925632 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765897693079238, + "loss": 1.7062, + "theoretical_loss": 4.423762408749001, + "tokens_seen": 185991168 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047657973921765294, + "loss": 1.7579, + "theoretical_loss": 4.423566472599099, + "tokens_seen": 186056704 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765697091273822, + "loss": 1.6467, + "theoretical_loss": 4.423370624769643, + "tokens_seen": 186122240 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047655967903711136, + "loss": 1.7319, + "theoretical_loss": 4.423174865189747, + "tokens_seen": 186187776 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047654964894684054, + "loss": 1.6317, + "theoretical_loss": 4.422979193788601, + "tokens_seen": 186253312 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765396188565697, + "loss": 1.7207, + "theoretical_loss": 4.422783610495481, + "tokens_seen": 186318848 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765295887662989, + "loss": 1.6619, + "theoretical_loss": 4.422588115239742, + "tokens_seen": 186384384 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765195586760281, + "loss": 1.7285, + "theoretical_loss": 4.422392707950824, + "tokens_seen": 186449920 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765095285857573, + "loss": 1.7306, + "theoretical_loss": 4.42219738855824, + "tokens_seen": 186515456 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047649949849548645, + "loss": 1.801, + "theoretical_loss": 4.422002156991597, + "tokens_seen": 186580992 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004764894684052157, + "loss": 1.603, + "theoretical_loss": 4.421807013180571, + "tokens_seen": 186646528 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047647943831494486, + "loss": 1.7809, + "theoretical_loss": 4.421611957054925, + "tokens_seen": 186712064 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 68623, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.7997204065322876, + "objective/train/theoretical_loss": 4.421416988544503, + "objective/train/tokens_used": 186777600, + "theoretical_loss": 4.421416988544503, + "tokens_seen": 186777600 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047646940822467404, + "loss": 1.7404, + "theoretical_loss": 4.421416988544503, + "tokens_seen": 186777600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004764593781344032, + "loss": 1.6726, + "theoretical_loss": 4.4212221075792275, + "tokens_seen": 186843136 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004764493480441324, + "loss": 1.571, + "theoretical_loss": 4.421027314089103, + "tokens_seen": 186908672 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004764393179538616, + "loss": 1.5455, + "theoretical_loss": 4.420832608004215, + "tokens_seen": 186974208 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004764292878635908, + "loss": 1.6379, + "theoretical_loss": 4.420637989254726, + "tokens_seen": 187039744 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047641925777331995, + "loss": 1.6, + "theoretical_loss": 4.420443457770885, + "tokens_seen": 187105280 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004764092276830492, + "loss": 1.7122, + "theoretical_loss": 4.420249013483014, + "tokens_seen": 187170816 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004763991975927783, + "loss": 1.6883, + "theoretical_loss": 4.42005465632152, + "tokens_seen": 187236352 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047638916750250755, + "loss": 1.8098, + "theoretical_loss": 4.419860386216888, + "tokens_seen": 187301888 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047637913741223673, + "loss": 1.7608, + "theoretical_loss": 4.419666203099683, + "tokens_seen": 187367424 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004763691073219659, + "loss": 1.6717, + "theoretical_loss": 4.41947210690055, + "tokens_seen": 187432960 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004763590772316951, + "loss": 1.7517, + "theoretical_loss": 4.419278097550212, + "tokens_seen": 187498496 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047634904714142427, + "loss": 1.7132, + "theoretical_loss": 4.419084174979473, + "tokens_seen": 187564032 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047633901705115345, + "loss": 1.6675, + "theoretical_loss": 4.4188903391192165, + "tokens_seen": 187629568 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004763289869608827, + "loss": 1.6299, + "theoretical_loss": 4.418696589900405, + "tokens_seen": 187695104 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004763189568706118, + "loss": 1.7402, + "theoretical_loss": 4.418502927254078, + "tokens_seen": 187760640 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047630892678034105, + "loss": 1.9435, + "theoretical_loss": 4.418309351111356, + "tokens_seen": 187826176 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047629889669007023, + "loss": 1.7037, + "theoretical_loss": 4.418115861403439, + "tokens_seen": 187891712 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762888665997994, + "loss": 1.7489, + "theoretical_loss": 4.417922458061603, + "tokens_seen": 187957248 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762788365095286, + "loss": 1.7387, + "theoretical_loss": 4.417729141017205, + "tokens_seen": 188022784 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762688064192578, + "loss": 1.6941, + "theoretical_loss": 4.4175359102016785, + "tokens_seen": 188088320 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047625877632898696, + "loss": 1.789, + "theoretical_loss": 4.417342765546538, + "tokens_seen": 188153856 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762487462387162, + "loss": 1.7749, + "theoretical_loss": 4.417149706983373, + "tokens_seen": 188219392 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762387161484453, + "loss": 1.7125, + "theoretical_loss": 4.416956734443854, + "tokens_seen": 188284928 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047622868605817455, + "loss": 1.7209, + "theoretical_loss": 4.416763847859728, + "tokens_seen": 188350464 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762186559679037, + "loss": 1.7939, + "theoretical_loss": 4.41657104716282, + "tokens_seen": 188416000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762086258776329, + "loss": 1.6192, + "theoretical_loss": 4.416378332285031, + "tokens_seen": 188481536 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761985957873621, + "loss": 1.726, + "theoretical_loss": 4.4161857031583445, + "tokens_seen": 188547072 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761885656970913, + "loss": 1.7253, + "theoretical_loss": 4.415993159714818, + "tokens_seen": 188612608 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047617853560682046, + "loss": 1.7822, + "theoretical_loss": 4.415800701886584, + "tokens_seen": 188678144 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047616850551654964, + "loss": 1.7532, + "theoretical_loss": 4.415608329605859, + "tokens_seen": 188743680 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761584754262788, + "loss": 1.6733, + "theoretical_loss": 4.415416042804931, + "tokens_seen": 188809216 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047614844533600806, + "loss": 1.843, + "theoretical_loss": 4.415223841416167, + "tokens_seen": 188874752 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761384152457372, + "loss": 1.7357, + "theoretical_loss": 4.415031725372011, + "tokens_seen": 188940288 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761283851554664, + "loss": 1.683, + "theoretical_loss": 4.414839694604985, + "tokens_seen": 189005824 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761183550651956, + "loss": 1.7561, + "theoretical_loss": 4.414647749047685, + "tokens_seen": 189071360 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761083249749248, + "loss": 1.881, + "theoretical_loss": 4.414455888632786, + "tokens_seen": 189136896 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047609829488465396, + "loss": 1.6126, + "theoretical_loss": 4.414264113293038, + "tokens_seen": 189202432 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047608826479438314, + "loss": 1.6555, + "theoretical_loss": 4.41407242296127, + "tokens_seen": 189267968 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004760782347041123, + "loss": 1.7649, + "theoretical_loss": 4.413880817570382, + "tokens_seen": 189333504 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047606820461384156, + "loss": 1.7084, + "theoretical_loss": 4.413689297053356, + "tokens_seen": 189399040 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004760581745235707, + "loss": 1.8044, + "theoretical_loss": 4.413497861343247, + "tokens_seen": 189464576 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004760481444332999, + "loss": 1.775, + "theoretical_loss": 4.413306510373185, + "tokens_seen": 189530112 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047603811434302905, + "loss": 1.6831, + "theoretical_loss": 4.41311524407638, + "tokens_seen": 189595648 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004760280842527583, + "loss": 1.7577, + "theoretical_loss": 4.412924062386112, + "tokens_seen": 189661184 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047601805416248746, + "loss": 1.7337, + "theoretical_loss": 4.412732965235742, + "tokens_seen": 189726720 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047600802407221665, + "loss": 1.7742, + "theoretical_loss": 4.412541952558703, + "tokens_seen": 189792256 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004759979939819458, + "loss": 1.6197, + "theoretical_loss": 4.412351024288504, + "tokens_seen": 189857792 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047598796389167506, + "loss": 1.6957, + "theoretical_loss": 4.412160180358731, + "tokens_seen": 189923328 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004759779338014042, + "loss": 1.656, + "theoretical_loss": 4.411969420703042, + "tokens_seen": 189988864 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 69528, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6714653968811035, + "objective/train/theoretical_loss": 4.411778745255174, + "objective/train/tokens_used": 190054400, + "theoretical_loss": 4.411778745255174, + "tokens_seen": 190054400 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004759679037111334, + "loss": 1.6452, + "theoretical_loss": 4.411778745255174, + "tokens_seen": 190054400 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004759578736208626, + "loss": 1.6004, + "theoretical_loss": 4.411588153948935, + "tokens_seen": 190119936 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004759478435305918, + "loss": 1.6545, + "theoretical_loss": 4.411397646718211, + "tokens_seen": 190185472 + }, + { + "epoch": 0.06, + "learning_rate": 0.000475937813440321, + "loss": 1.6654, + "theoretical_loss": 4.411207223496962, + "tokens_seen": 190251008 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047592778335005015, + "loss": 1.6958, + "theoretical_loss": 4.41101688421922, + "tokens_seen": 190316544 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004759177532597794, + "loss": 1.6786, + "theoretical_loss": 4.410826628819094, + "tokens_seen": 190382080 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004759077231695085, + "loss": 1.711, + "theoretical_loss": 4.410636457230767, + "tokens_seen": 190447616 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047589769307923775, + "loss": 1.6261, + "theoretical_loss": 4.410446369388497, + "tokens_seen": 190513152 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047588766298896693, + "loss": 1.7213, + "theoretical_loss": 4.410256365226614, + "tokens_seen": 190578688 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004758776328986961, + "loss": 1.7746, + "theoretical_loss": 4.410066444679524, + "tokens_seen": 190644224 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004758676028084253, + "loss": 1.7249, + "theoretical_loss": 4.409876607681706, + "tokens_seen": 190709760 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047585757271815447, + "loss": 1.7782, + "theoretical_loss": 4.409686854167714, + "tokens_seen": 190775296 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047584754262788365, + "loss": 1.7312, + "theoretical_loss": 4.4094971840721735, + "tokens_seen": 190840832 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004758375125376129, + "loss": 1.7164, + "theoretical_loss": 4.409307597329786, + "tokens_seen": 190906368 + }, + { + "epoch": 0.06, + "learning_rate": 0.000475827482447342, + "loss": 1.7488, + "theoretical_loss": 4.409118093875324, + "tokens_seen": 190971904 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047581745235707125, + "loss": 1.6334, + "theoretical_loss": 4.408928673643636, + "tokens_seen": 191037440 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047580742226680043, + "loss": 1.7199, + "theoretical_loss": 4.408739336569644, + "tokens_seen": 191102976 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757973921765296, + "loss": 1.7496, + "theoretical_loss": 4.408550082588339, + "tokens_seen": 191168512 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757873620862588, + "loss": 1.7719, + "theoretical_loss": 4.408360911634789, + "tokens_seen": 191234048 + }, + { + "epoch": 0.06, + "learning_rate": 0.000475777331995988, + "loss": 1.6524, + "theoretical_loss": 4.408171823644135, + "tokens_seen": 191299584 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047576730190571716, + "loss": 1.6494, + "theoretical_loss": 4.407982818551589, + "tokens_seen": 191365120 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757572718154464, + "loss": 1.7035, + "theoretical_loss": 4.407793896292437, + "tokens_seen": 191430656 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757472417251755, + "loss": 1.8007, + "theoretical_loss": 4.407605056802035, + "tokens_seen": 191496192 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047573721163490475, + "loss": 1.6864, + "theoretical_loss": 4.407416300015816, + "tokens_seen": 191561728 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757271815446339, + "loss": 1.7633, + "theoretical_loss": 4.407227625869283, + "tokens_seen": 191627264 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757171514543631, + "loss": 1.7855, + "theoretical_loss": 4.407039034298011, + "tokens_seen": 191692800 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757071213640923, + "loss": 1.6685, + "theoretical_loss": 4.4068505252376475, + "tokens_seen": 191758336 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004756970912738215, + "loss": 1.7185, + "theoretical_loss": 4.406662098623913, + "tokens_seen": 191823872 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047568706118355066, + "loss": 1.7441, + "theoretical_loss": 4.406473754392598, + "tokens_seen": 191889408 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047567703109327984, + "loss": 1.7468, + "theoretical_loss": 4.406285492479569, + "tokens_seen": 191954944 + }, + { + "epoch": 0.06, + "learning_rate": 0.000475667001003009, + "loss": 1.7867, + "theoretical_loss": 4.406097312820759, + "tokens_seen": 192020480 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047565697091273826, + "loss": 1.6915, + "theoretical_loss": 4.4059092153521755, + "tokens_seen": 192086016 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004756469408224674, + "loss": 1.7357, + "theoretical_loss": 4.405721200009898, + "tokens_seen": 192151552 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004756369107321966, + "loss": 1.7076, + "theoretical_loss": 4.405533266730077, + "tokens_seen": 192217088 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004756268806419258, + "loss": 1.8311, + "theoretical_loss": 4.405345415448934, + "tokens_seen": 192282624 + }, + { + "epoch": 0.06, + "learning_rate": 0.000475616850551655, + "loss": 1.7459, + "theoretical_loss": 4.405157646102762, + "tokens_seen": 192348160 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047560682046138416, + "loss": 1.5306, + "theoretical_loss": 4.404969958627927, + "tokens_seen": 192413696 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047559679037111334, + "loss": 1.6856, + "theoretical_loss": 4.4047823529608605, + "tokens_seen": 192479232 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004755867602808425, + "loss": 1.7038, + "theoretical_loss": 4.404594829038071, + "tokens_seen": 192544768 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047557673019057176, + "loss": 1.6167, + "theoretical_loss": 4.404407386796137, + "tokens_seen": 192610304 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004755667001003009, + "loss": 1.7299, + "theoretical_loss": 4.404220026171704, + "tokens_seen": 192675840 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004755566700100301, + "loss": 1.613, + "theoretical_loss": 4.404032747101493, + "tokens_seen": 192741376 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047554663991975925, + "loss": 1.7188, + "theoretical_loss": 4.4038455495222895, + "tokens_seen": 192806912 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004755366098294885, + "loss": 1.7373, + "theoretical_loss": 4.403658433370957, + "tokens_seen": 192872448 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047552657973921766, + "loss": 1.7778, + "theoretical_loss": 4.403471398584422, + "tokens_seen": 192937984 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047551654964894685, + "loss": 1.5914, + "theoretical_loss": 4.403284445099688, + "tokens_seen": 193003520 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047550651955867603, + "loss": 1.7295, + "theoretical_loss": 4.403097572853823, + "tokens_seen": 193069056 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047549648946840526, + "loss": 1.8016, + "theoretical_loss": 4.402910781783968, + "tokens_seen": 193134592 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004754864593781344, + "loss": 1.563, + "theoretical_loss": 4.4027240718273335, + "tokens_seen": 193200128 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004754764292878636, + "loss": 1.6926, + "theoretical_loss": 4.4025374429212, + "tokens_seen": 193265664 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 70720, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.8319575786590576, + "objective/train/theoretical_loss": 4.402350895002916, + "objective/train/tokens_used": 193331200, + "theoretical_loss": 4.402350895002916, + "tokens_seen": 193331200 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047546639919759275, + "loss": 1.7492, + "theoretical_loss": 4.402350895002916, + "tokens_seen": 193331200 + }, + { + "epoch": 0.06, + "learning_rate": 0.000475456369107322, + "loss": 1.7407, + "theoretical_loss": 4.402164428009904, + "tokens_seen": 193396736 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047544633901705117, + "loss": 1.676, + "theoretical_loss": 4.40197804187965, + "tokens_seen": 193462272 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047543630892678035, + "loss": 1.7343, + "theoretical_loss": 4.401791736549714, + "tokens_seen": 193527808 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047542627883650953, + "loss": 1.6373, + "theoretical_loss": 4.401605511957724, + "tokens_seen": 193593344 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004754162487462387, + "loss": 1.7712, + "theoretical_loss": 4.401419368041377, + "tokens_seen": 193658880 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004754062186559679, + "loss": 1.7983, + "theoretical_loss": 4.401233304738438, + "tokens_seen": 193724416 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047539618856569713, + "loss": 1.671, + "theoretical_loss": 4.401047321986745, + "tokens_seen": 193789952 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047538615847542625, + "loss": 1.8115, + "theoretical_loss": 4.4008614197242, + "tokens_seen": 193855488 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004753761283851555, + "loss": 1.7297, + "theoretical_loss": 4.400675597888777, + "tokens_seen": 193921024 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004753660982948846, + "loss": 1.6737, + "theoretical_loss": 4.400489856418518, + "tokens_seen": 193986560 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047535606820461385, + "loss": 1.7843, + "theoretical_loss": 4.4003041952515325, + "tokens_seen": 194052096 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047534603811434303, + "loss": 1.6999, + "theoretical_loss": 4.400118614326001, + "tokens_seen": 194117632 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004753360080240722, + "loss": 1.718, + "theoretical_loss": 4.39993311358017, + "tokens_seen": 194183168 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004753259779338014, + "loss": 1.8586, + "theoretical_loss": 4.399747692952355, + "tokens_seen": 194248704 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047531594784353063, + "loss": 1.6923, + "theoretical_loss": 4.399562352380941, + "tokens_seen": 194314240 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047530591775325976, + "loss": 1.6086, + "theoretical_loss": 4.399377091804379, + "tokens_seen": 194379776 + }, + { + "epoch": 0.06, + "learning_rate": 0.000475295887662989, + "loss": 1.7978, + "theoretical_loss": 4.399191911161191, + "tokens_seen": 194445312 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004752858575727181, + "loss": 1.7759, + "theoretical_loss": 4.399006810389963, + "tokens_seen": 194510848 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047527582748244736, + "loss": 1.6863, + "theoretical_loss": 4.398821789429354, + "tokens_seen": 194576384 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047526579739217654, + "loss": 1.6709, + "theoretical_loss": 4.398636848218084, + "tokens_seen": 194641920 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004752557673019057, + "loss": 1.8086, + "theoretical_loss": 4.398451986694948, + "tokens_seen": 194707456 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004752457372116349, + "loss": 1.7107, + "theoretical_loss": 4.398267204798803, + "tokens_seen": 194772992 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004752357071213641, + "loss": 1.6338, + "theoretical_loss": 4.398082502468577, + "tokens_seen": 194838528 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047522567703109326, + "loss": 1.6675, + "theoretical_loss": 4.397897879643262, + "tokens_seen": 194904064 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004752156469408225, + "loss": 1.6497, + "theoretical_loss": 4.39771333626192, + "tokens_seen": 194969600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004752056168505517, + "loss": 1.6962, + "theoretical_loss": 4.397528872263679, + "tokens_seen": 195035136 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047519558676028086, + "loss": 1.696, + "theoretical_loss": 4.397344487587736, + "tokens_seen": 195100672 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047518555667001004, + "loss": 1.6758, + "theoretical_loss": 4.3971601821733515, + "tokens_seen": 195166208 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004751755265797392, + "loss": 1.7019, + "theoretical_loss": 4.396975955959856, + "tokens_seen": 195231744 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047516549648946846, + "loss": 1.7086, + "theoretical_loss": 4.396791808886643, + "tokens_seen": 195297280 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004751554663991976, + "loss": 1.7057, + "theoretical_loss": 4.396607740893179, + "tokens_seen": 195362816 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004751454363089268, + "loss": 1.6434, + "theoretical_loss": 4.396423751918991, + "tokens_seen": 195428352 + }, + { + "epoch": 0.06, + "learning_rate": 0.000475135406218656, + "loss": 1.7288, + "theoretical_loss": 4.396239841903674, + "tokens_seen": 195493888 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004751253761283852, + "loss": 1.8359, + "theoretical_loss": 4.396056010786893, + "tokens_seen": 195559424 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047511534603811436, + "loss": 1.8149, + "theoretical_loss": 4.3958722585083745, + "tokens_seen": 195624960 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047510531594784354, + "loss": 1.6431, + "theoretical_loss": 4.395688585007913, + "tokens_seen": 195690496 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004750952858575727, + "loss": 1.6545, + "theoretical_loss": 4.395504990225371, + "tokens_seen": 195756032 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047508525576730196, + "loss": 1.6416, + "theoretical_loss": 4.395321474100673, + "tokens_seen": 195821568 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004750752256770311, + "loss": 1.688, + "theoretical_loss": 4.395138036573814, + "tokens_seen": 195887104 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004750651955867603, + "loss": 1.7664, + "theoretical_loss": 4.3949546775848525, + "tokens_seen": 195952640 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047505516549648945, + "loss": 1.6936, + "theoretical_loss": 4.394771397073912, + "tokens_seen": 196018176 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004750451354062187, + "loss": 1.7042, + "theoretical_loss": 4.394588194981182, + "tokens_seen": 196083712 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047503510531594787, + "loss": 1.8497, + "theoretical_loss": 4.394405071246919, + "tokens_seen": 196149248 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047502507522567705, + "loss": 1.5912, + "theoretical_loss": 4.394222025811445, + "tokens_seen": 196214784 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047501504513540623, + "loss": 1.6127, + "theoretical_loss": 4.394039058615144, + "tokens_seen": 196280320 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047500501504513546, + "loss": 1.7235, + "theoretical_loss": 4.39385616959847, + "tokens_seen": 196345856 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004749949849548646, + "loss": 1.6861, + "theoretical_loss": 4.393673358701939, + "tokens_seen": 196411392 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004749849548645938, + "loss": 1.7209, + "theoretical_loss": 4.393490625866132, + "tokens_seen": 196476928 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047497492477432295, + "loss": 1.6369, + "theoretical_loss": 4.393307971031697, + "tokens_seen": 196542464 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 71643, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5311357975006104, + "objective/train/theoretical_loss": 4.3931253941393456, + "objective/train/tokens_used": 196608000, + "theoretical_loss": 4.3931253941393456, + "tokens_seen": 196608000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004749648946840522, + "loss": 1.6469, + "theoretical_loss": 4.3931253941393456, + "tokens_seen": 196608000 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047495486459378137, + "loss": 1.6068, + "theoretical_loss": 4.392942895129854, + "tokens_seen": 196673536 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047494483450351055, + "loss": 1.7213, + "theoretical_loss": 4.392760473944064, + "tokens_seen": 196739072 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047493480441323973, + "loss": 1.7185, + "theoretical_loss": 4.392578130522883, + "tokens_seen": 196804608 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004749247743229689, + "loss": 1.7414, + "theoretical_loss": 4.392395864807279, + "tokens_seen": 196870144 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004749147442326981, + "loss": 1.6034, + "theoretical_loss": 4.392213676738288, + "tokens_seen": 196935680 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047490471414242733, + "loss": 1.7452, + "theoretical_loss": 4.3920315662570095, + "tokens_seen": 197001216 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047489468405215646, + "loss": 1.6476, + "theoretical_loss": 4.391849533304607, + "tokens_seen": 197066752 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004748846539618857, + "loss": 1.7657, + "theoretical_loss": 4.391667577822308, + "tokens_seen": 197132288 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004748746238716148, + "loss": 1.6457, + "theoretical_loss": 4.391485699751406, + "tokens_seen": 197197824 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047486459378134405, + "loss": 1.7032, + "theoretical_loss": 4.391303899033253, + "tokens_seen": 197263360 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047485456369107323, + "loss": 1.6725, + "theoretical_loss": 4.391122175609272, + "tokens_seen": 197328896 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004748445336008024, + "loss": 1.8053, + "theoretical_loss": 4.390940529420946, + "tokens_seen": 197394432 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004748345035105316, + "loss": 1.6763, + "theoretical_loss": 4.39075896040982, + "tokens_seen": 197459968 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047482447342026083, + "loss": 1.715, + "theoretical_loss": 4.390577468517508, + "tokens_seen": 197525504 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047481444332998996, + "loss": 1.7271, + "theoretical_loss": 4.390396053685682, + "tokens_seen": 197591040 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004748044132397192, + "loss": 1.6668, + "theoretical_loss": 4.39021471585608, + "tokens_seen": 197656576 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004747943831494483, + "loss": 1.8048, + "theoretical_loss": 4.390033454970504, + "tokens_seen": 197722112 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047478435305917756, + "loss": 1.7181, + "theoretical_loss": 4.389852270970818, + "tokens_seen": 197787648 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047477432296890674, + "loss": 1.64, + "theoretical_loss": 4.389671163798949, + "tokens_seen": 197853184 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004747642928786359, + "loss": 1.8471, + "theoretical_loss": 4.3894901333968885, + "tokens_seen": 197918720 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004747542627883651, + "loss": 1.63, + "theoretical_loss": 4.38930917970669, + "tokens_seen": 197984256 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004747442326980943, + "loss": 1.6462, + "theoretical_loss": 4.3891283026704695, + "tokens_seen": 198049792 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047473420260782346, + "loss": 1.738, + "theoretical_loss": 4.388947502230407, + "tokens_seen": 198115328 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004747241725175527, + "loss": 1.629, + "theoretical_loss": 4.388766778328744, + "tokens_seen": 198180864 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004747141424272818, + "loss": 1.7306, + "theoretical_loss": 4.388586130907786, + "tokens_seen": 198246400 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047470411233701106, + "loss": 1.6492, + "theoretical_loss": 4.388405559909899, + "tokens_seen": 198311936 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746940822467402, + "loss": 1.5679, + "theoretical_loss": 4.3882250652775125, + "tokens_seen": 198377472 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746840521564694, + "loss": 1.8372, + "theoretical_loss": 4.38804464695312, + "tokens_seen": 198443008 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746740220661986, + "loss": 1.6599, + "theoretical_loss": 4.387864304879275, + "tokens_seen": 198508544 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746639919759278, + "loss": 1.7287, + "theoretical_loss": 4.387684038998595, + "tokens_seen": 198574080 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047465396188565696, + "loss": 1.6337, + "theoretical_loss": 4.387503849253757, + "tokens_seen": 198639616 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746439317953862, + "loss": 1.8451, + "theoretical_loss": 4.387323735587502, + "tokens_seen": 198705152 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746339017051153, + "loss": 1.7176, + "theoretical_loss": 4.387143697942634, + "tokens_seen": 198770688 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047462387161484456, + "loss": 1.8277, + "theoretical_loss": 4.386963736262016, + "tokens_seen": 198836224 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746138415245737, + "loss": 1.6693, + "theoretical_loss": 4.386783850488575, + "tokens_seen": 198901760 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746038114343029, + "loss": 1.7425, + "theoretical_loss": 4.386604040565299, + "tokens_seen": 198967296 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004745937813440321, + "loss": 1.7736, + "theoretical_loss": 4.386424306435236, + "tokens_seen": 199032832 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004745837512537613, + "loss": 1.698, + "theoretical_loss": 4.386244648041498, + "tokens_seen": 199098368 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047457372116349047, + "loss": 1.8408, + "theoretical_loss": 4.386065065327258, + "tokens_seen": 199163904 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047456369107321965, + "loss": 1.7212, + "theoretical_loss": 4.385885558235747, + "tokens_seen": 199229440 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047455366098294883, + "loss": 1.7335, + "theoretical_loss": 4.385706126710264, + "tokens_seen": 199294976 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047454363089267807, + "loss": 1.6681, + "theoretical_loss": 4.385526770694161, + "tokens_seen": 199360512 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004745336008024072, + "loss": 1.6427, + "theoretical_loss": 4.385347490130858, + "tokens_seen": 199426048 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047452357071213643, + "loss": 1.7109, + "theoretical_loss": 4.385168284963832, + "tokens_seen": 199491584 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047451354062186555, + "loss": 1.7562, + "theoretical_loss": 4.384989155136621, + "tokens_seen": 199557120 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004745035105315948, + "loss": 1.7367, + "theoretical_loss": 4.384810100592826, + "tokens_seen": 199622656 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047449348044132397, + "loss": 1.7037, + "theoretical_loss": 4.384631121276108, + "tokens_seen": 199688192 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047448345035105315, + "loss": 1.7201, + "theoretical_loss": 4.384452217130187, + "tokens_seen": 199753728 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004744734202607824, + "loss": 1.7224, + "theoretical_loss": 4.384273388098845, + "tokens_seen": 199819264 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 72845, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.7876832485198975, + "objective/train/theoretical_loss": 4.384094634125924, + "objective/train/tokens_used": 199884800, + "theoretical_loss": 4.384094634125924, + "tokens_seen": 199884800 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047446339017051157, + "loss": 1.7449, + "theoretical_loss": 4.384094634125924, + "tokens_seen": 199884800 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047445336008024075, + "loss": 1.8, + "theoretical_loss": 4.383915955155326, + "tokens_seen": 199950336 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047444332998996993, + "loss": 1.6605, + "theoretical_loss": 4.383737351131016, + "tokens_seen": 200015872 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004744332998996991, + "loss": 1.6387, + "theoretical_loss": 4.383558821997014, + "tokens_seen": 200081408 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004744232698094283, + "loss": 1.681, + "theoretical_loss": 4.383380367697406, + "tokens_seen": 200146944 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047441323971915753, + "loss": 1.8012, + "theoretical_loss": 4.383201988176333, + "tokens_seen": 200212480 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047440320962888666, + "loss": 1.6288, + "theoretical_loss": 4.383023683377999, + "tokens_seen": 200278016 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004743931795386159, + "loss": 1.6246, + "theoretical_loss": 4.382845453246668, + "tokens_seen": 200343552 + }, + { + "epoch": 0.06, + "learning_rate": 0.000474383149448345, + "loss": 1.7322, + "theoretical_loss": 4.382667297726661, + "tokens_seen": 200409088 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047437311935807425, + "loss": 1.6289, + "theoretical_loss": 4.3824892167623615, + "tokens_seen": 200474624 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047436308926780343, + "loss": 1.6866, + "theoretical_loss": 4.382311210298212, + "tokens_seen": 200540160 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004743530591775326, + "loss": 1.6424, + "theoretical_loss": 4.3821332782787135, + "tokens_seen": 200605696 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004743430290872618, + "loss": 1.6305, + "theoretical_loss": 4.381955420648428, + "tokens_seen": 200671232 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047433299899699103, + "loss": 1.6239, + "theoretical_loss": 4.381777637351974, + "tokens_seen": 200736768 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047432296890672016, + "loss": 1.6581, + "theoretical_loss": 4.381599928334033, + "tokens_seen": 200802304 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004743129388164494, + "loss": 1.8226, + "theoretical_loss": 4.381422293539345, + "tokens_seen": 200867840 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004743029087261785, + "loss": 1.7631, + "theoretical_loss": 4.381244732912706, + "tokens_seen": 200933376 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047429287863590776, + "loss": 1.8238, + "theoretical_loss": 4.381067246398975, + "tokens_seen": 200998912 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047428284854563694, + "loss": 1.6437, + "theoretical_loss": 4.380889833943066, + "tokens_seen": 201064448 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004742728184553661, + "loss": 1.7506, + "theoretical_loss": 4.380712495489957, + "tokens_seen": 201129984 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004742627883650953, + "loss": 1.6803, + "theoretical_loss": 4.380535230984679, + "tokens_seen": 201195520 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004742527582748245, + "loss": 1.8016, + "theoretical_loss": 4.380358040372325, + "tokens_seen": 201261056 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047424272818455366, + "loss": 1.7156, + "theoretical_loss": 4.380180923598047, + "tokens_seen": 201326592 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004742326980942829, + "loss": 1.7707, + "theoretical_loss": 4.380003880607055, + "tokens_seen": 201392128 + }, + { + "epoch": 0.06, + "learning_rate": 0.000474222668004012, + "loss": 1.798, + "theoretical_loss": 4.379826911344617, + "tokens_seen": 201457664 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047421263791374126, + "loss": 1.8779, + "theoretical_loss": 4.3796500157560585, + "tokens_seen": 201523200 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004742026078234704, + "loss": 1.8034, + "theoretical_loss": 4.379473193786765, + "tokens_seen": 201588736 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741925777331996, + "loss": 1.6262, + "theoretical_loss": 4.37929644538218, + "tokens_seen": 201654272 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741825476429288, + "loss": 1.7743, + "theoretical_loss": 4.379119770487805, + "tokens_seen": 201719808 + }, + { + "epoch": 0.06, + "learning_rate": 0.000474172517552658, + "loss": 1.7078, + "theoretical_loss": 4.378943169049198, + "tokens_seen": 201785344 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047416248746238716, + "loss": 1.8556, + "theoretical_loss": 4.378766641011978, + "tokens_seen": 201850880 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741524573721164, + "loss": 1.6133, + "theoretical_loss": 4.378590186321819, + "tokens_seen": 201916416 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741424272818455, + "loss": 1.7137, + "theoretical_loss": 4.378413804924454, + "tokens_seen": 201981952 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047413239719157476, + "loss": 1.6859, + "theoretical_loss": 4.378237496765674, + "tokens_seen": 202047488 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741223671013039, + "loss": 1.6207, + "theoretical_loss": 4.378061261791328, + "tokens_seen": 202113024 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741123370110331, + "loss": 1.7902, + "theoretical_loss": 4.377885099947321, + "tokens_seen": 202178560 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741023069207623, + "loss": 1.7425, + "theoretical_loss": 4.3777090111796175, + "tokens_seen": 202244096 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004740922768304915, + "loss": 1.6321, + "theoretical_loss": 4.377532995434238, + "tokens_seen": 202309632 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047408224674022067, + "loss": 1.6798, + "theoretical_loss": 4.37735705265726, + "tokens_seen": 202375168 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047407221664994985, + "loss": 1.7394, + "theoretical_loss": 4.3771811827948195, + "tokens_seen": 202440704 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047406218655967903, + "loss": 1.7732, + "theoretical_loss": 4.37700538579311, + "tokens_seen": 202506240 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047405215646940827, + "loss": 1.7193, + "theoretical_loss": 4.37682966159838, + "tokens_seen": 202571776 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004740421263791374, + "loss": 1.6956, + "theoretical_loss": 4.376654010156939, + "tokens_seen": 202637312 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047403209628886663, + "loss": 1.6824, + "theoretical_loss": 4.376478431415148, + "tokens_seen": 202702848 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047402206619859575, + "loss": 1.6404, + "theoretical_loss": 4.376302925319427, + "tokens_seen": 202768384 + }, + { + "epoch": 0.06, + "learning_rate": 0.000474012036108325, + "loss": 1.5418, + "theoretical_loss": 4.3761274918162565, + "tokens_seen": 202833920 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047400200601805417, + "loss": 1.6848, + "theoretical_loss": 4.375952130852169, + "tokens_seen": 202899456 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047399197592778335, + "loss": 1.7129, + "theoretical_loss": 4.375776842373755, + "tokens_seen": 202964992 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047398194583751253, + "loss": 1.6785, + "theoretical_loss": 4.375601626327662, + "tokens_seen": 203030528 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047397191574724177, + "loss": 1.5921, + "theoretical_loss": 4.375426482660595, + "tokens_seen": 203096064 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 73746, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5321800708770752, + "objective/train/theoretical_loss": 4.375251411319312, + "objective/train/tokens_used": 203161600, + "theoretical_loss": 4.375251411319312, + "tokens_seen": 203161600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004739618856569709, + "loss": 1.6384, + "theoretical_loss": 4.375251411319312, + "tokens_seen": 203161600 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047395185556670013, + "loss": 1.5824, + "theoretical_loss": 4.375076412250632, + "tokens_seen": 203227136 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047394182547642926, + "loss": 1.7724, + "theoretical_loss": 4.374901485401426, + "tokens_seen": 203292672 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004739317953861585, + "loss": 1.682, + "theoretical_loss": 4.374726630718624, + "tokens_seen": 203358208 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004739217652958877, + "loss": 1.7429, + "theoretical_loss": 4.37455184814921, + "tokens_seen": 203423744 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047391173520561686, + "loss": 1.6953, + "theoretical_loss": 4.374377137640225, + "tokens_seen": 203489280 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047390170511534604, + "loss": 1.7394, + "theoretical_loss": 4.374202499138768, + "tokens_seen": 203554816 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004738916750250752, + "loss": 1.8442, + "theoretical_loss": 4.374027932591989, + "tokens_seen": 203620352 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004738816449348044, + "loss": 1.6713, + "theoretical_loss": 4.373853437947099, + "tokens_seen": 203685888 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047387161484453363, + "loss": 1.7383, + "theoretical_loss": 4.37367901515136, + "tokens_seen": 203751424 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047386158475426276, + "loss": 1.6558, + "theoretical_loss": 4.373504664152094, + "tokens_seen": 203816960 + }, + { + "epoch": 0.06, + "learning_rate": 0.000473851554663992, + "loss": 1.7584, + "theoretical_loss": 4.373330384896676, + "tokens_seen": 203882496 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004738415245737211, + "loss": 1.59, + "theoretical_loss": 4.373156177332536, + "tokens_seen": 203948032 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047383149448345036, + "loss": 1.7916, + "theoretical_loss": 4.372982041407161, + "tokens_seen": 204013568 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047382146439317954, + "loss": 1.7313, + "theoretical_loss": 4.372807977068092, + "tokens_seen": 204079104 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004738114343029087, + "loss": 1.6904, + "theoretical_loss": 4.372633984262928, + "tokens_seen": 204144640 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004738014042126379, + "loss": 1.6574, + "theoretical_loss": 4.372460062939318, + "tokens_seen": 204210176 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047379137412236714, + "loss": 1.8324, + "theoretical_loss": 4.372286213044972, + "tokens_seen": 204275712 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047378134403209626, + "loss": 1.8401, + "theoretical_loss": 4.37211243452765, + "tokens_seen": 204341248 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004737713139418255, + "loss": 1.6437, + "theoretical_loss": 4.37193872733517, + "tokens_seen": 204406784 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004737612838515546, + "loss": 1.6367, + "theoretical_loss": 4.371765091415404, + "tokens_seen": 204472320 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047375125376128386, + "loss": 1.7984, + "theoretical_loss": 4.371591526716279, + "tokens_seen": 204537856 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047374122367101304, + "loss": 1.6643, + "theoretical_loss": 4.371418033185777, + "tokens_seen": 204603392 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004737311935807422, + "loss": 1.7176, + "theoretical_loss": 4.3712446107719325, + "tokens_seen": 204668928 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047372116349047146, + "loss": 1.7464, + "theoretical_loss": 4.371071259422838, + "tokens_seen": 204734464 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004737111334002006, + "loss": 1.6972, + "theoretical_loss": 4.370897979086636, + "tokens_seen": 204800000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004737011033099298, + "loss": 1.8033, + "theoretical_loss": 4.370724769711529, + "tokens_seen": 204865536 + }, + { + "epoch": 0.06, + "learning_rate": 0.000473691073219659, + "loss": 1.6828, + "theoretical_loss": 4.370551631245769, + "tokens_seen": 204931072 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736810431293882, + "loss": 1.7258, + "theoretical_loss": 4.370378563637667, + "tokens_seen": 204996608 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047367101303911736, + "loss": 1.6029, + "theoretical_loss": 4.3702055668355815, + "tokens_seen": 205062144 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736609829488466, + "loss": 1.8507, + "theoretical_loss": 4.370032640787931, + "tokens_seen": 205127680 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736509528585757, + "loss": 1.6345, + "theoretical_loss": 4.369859785443188, + "tokens_seen": 205193216 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047364092276830496, + "loss": 1.7128, + "theoretical_loss": 4.369687000749873, + "tokens_seen": 205258752 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736308926780341, + "loss": 1.5754, + "theoretical_loss": 4.3695142866565675, + "tokens_seen": 205324288 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736208625877633, + "loss": 1.6753, + "theoretical_loss": 4.369341643111902, + "tokens_seen": 205389824 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736108324974925, + "loss": 1.7857, + "theoretical_loss": 4.369169070064563, + "tokens_seen": 205455360 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736008024072217, + "loss": 1.6239, + "theoretical_loss": 4.368996567463292, + "tokens_seen": 205520896 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047359077231695087, + "loss": 1.5632, + "theoretical_loss": 4.3688241352568795, + "tokens_seen": 205586432 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047358074222668005, + "loss": 1.5989, + "theoretical_loss": 4.368651773394173, + "tokens_seen": 205651968 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047357071213640923, + "loss": 1.7331, + "theoretical_loss": 4.368479481824075, + "tokens_seen": 205717504 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047356068204613847, + "loss": 1.6568, + "theoretical_loss": 4.368307260495536, + "tokens_seen": 205783040 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004735506519558676, + "loss": 1.7231, + "theoretical_loss": 4.368135109357564, + "tokens_seen": 205848576 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047354062186559683, + "loss": 1.7595, + "theoretical_loss": 4.36796302835922, + "tokens_seen": 205914112 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047353059177532595, + "loss": 1.6061, + "theoretical_loss": 4.367791017449618, + "tokens_seen": 205979648 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004735205616850552, + "loss": 1.7001, + "theoretical_loss": 4.367619076577923, + "tokens_seen": 206045184 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047351053159478437, + "loss": 1.7348, + "theoretical_loss": 4.367447205693354, + "tokens_seen": 206110720 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047350050150451355, + "loss": 1.7153, + "theoretical_loss": 4.367275404745186, + "tokens_seen": 206176256 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047349047141424273, + "loss": 1.6629, + "theoretical_loss": 4.367103673682742, + "tokens_seen": 206241792 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047348044132397197, + "loss": 1.6778, + "theoretical_loss": 4.3669320124554, + "tokens_seen": 206307328 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004734704112337011, + "loss": 1.7242, + "theoretical_loss": 4.366760421012592, + "tokens_seen": 206372864 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 74963, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.634135365486145, + "objective/train/theoretical_loss": 4.366588899303801, + "objective/train/tokens_used": 206438400, + "theoretical_loss": 4.366588899303801, + "tokens_seen": 206438400 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047346038114343033, + "loss": 1.7017, + "theoretical_loss": 4.366588899303801, + "tokens_seen": 206438400 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047345035105315946, + "loss": 1.551, + "theoretical_loss": 4.366417447278565, + "tokens_seen": 206503936 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004734403209628887, + "loss": 1.7075, + "theoretical_loss": 4.366246064886468, + "tokens_seen": 206569472 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004734302908726179, + "loss": 1.7007, + "theoretical_loss": 4.3660747520771555, + "tokens_seen": 206635008 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047342026078234706, + "loss": 1.8394, + "theoretical_loss": 4.36590350880032, + "tokens_seen": 206700544 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047341023069207624, + "loss": 1.7068, + "theoretical_loss": 4.365732335005706, + "tokens_seen": 206766080 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004734002006018054, + "loss": 1.6969, + "theoretical_loss": 4.365561230643112, + "tokens_seen": 206831616 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004733901705115346, + "loss": 1.789, + "theoretical_loss": 4.365390195662389, + "tokens_seen": 206897152 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047338014042126383, + "loss": 1.5922, + "theoretical_loss": 4.365219230013438, + "tokens_seen": 206962688 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047337011033099296, + "loss": 1.7479, + "theoretical_loss": 4.3650483336462145, + "tokens_seen": 207028224 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004733600802407222, + "loss": 1.6529, + "theoretical_loss": 4.364877506510723, + "tokens_seen": 207093760 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004733500501504513, + "loss": 1.8486, + "theoretical_loss": 4.3647067485570235, + "tokens_seen": 207159296 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047334002006018056, + "loss": 1.8142, + "theoretical_loss": 4.364536059735224, + "tokens_seen": 207224832 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047332998996990974, + "loss": 1.7425, + "theoretical_loss": 4.364365439995488, + "tokens_seen": 207290368 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004733199598796389, + "loss": 1.7126, + "theoretical_loss": 4.3641948892880285, + "tokens_seen": 207355904 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004733099297893681, + "loss": 1.6743, + "theoretical_loss": 4.364024407563109, + "tokens_seen": 207421440 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047329989969909734, + "loss": 1.9469, + "theoretical_loss": 4.363853994771048, + "tokens_seen": 207486976 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047328986960882646, + "loss": 1.6529, + "theoretical_loss": 4.363683650862212, + "tokens_seen": 207552512 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004732798395185557, + "loss": 1.7089, + "theoretical_loss": 4.36351337578702, + "tokens_seen": 207618048 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004732698094282848, + "loss": 1.615, + "theoretical_loss": 4.363343169495945, + "tokens_seen": 207683584 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047325977933801406, + "loss": 1.6754, + "theoretical_loss": 4.363173031939506, + "tokens_seen": 207749120 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047324974924774324, + "loss": 1.6669, + "theoretical_loss": 4.363002963068277, + "tokens_seen": 207814656 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004732397191574724, + "loss": 1.685, + "theoretical_loss": 4.362832962832883, + "tokens_seen": 207880192 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004732296890672016, + "loss": 1.6864, + "theoretical_loss": 4.362663031183999, + "tokens_seen": 207945728 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004732196589769308, + "loss": 1.7646, + "theoretical_loss": 4.36249316807235, + "tokens_seen": 208011264 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047320962888665997, + "loss": 1.7781, + "theoretical_loss": 4.3623233734487155, + "tokens_seen": 208076800 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004731995987963892, + "loss": 1.6829, + "theoretical_loss": 4.362153647263921, + "tokens_seen": 208142336 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047318956870611833, + "loss": 1.593, + "theoretical_loss": 4.361983989468847, + "tokens_seen": 208207872 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047317953861584757, + "loss": 1.7742, + "theoretical_loss": 4.3618144000144214, + "tokens_seen": 208273408 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047316950852557675, + "loss": 1.8271, + "theoretical_loss": 4.361644878851626, + "tokens_seen": 208338944 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047315947843530593, + "loss": 1.7696, + "theoretical_loss": 4.36147542593149, + "tokens_seen": 208404480 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004731494483450351, + "loss": 1.6717, + "theoretical_loss": 4.361306041205095, + "tokens_seen": 208470016 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004731394182547643, + "loss": 1.7427, + "theoretical_loss": 4.361136724623573, + "tokens_seen": 208535552 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047312938816449347, + "loss": 1.8352, + "theoretical_loss": 4.360967476138105, + "tokens_seen": 208601088 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004731193580742227, + "loss": 1.8023, + "theoretical_loss": 4.360798295699925, + "tokens_seen": 208666624 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047310932798395183, + "loss": 1.6197, + "theoretical_loss": 4.360629183260313, + "tokens_seen": 208732160 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047309929789368107, + "loss": 1.7333, + "theoretical_loss": 4.360460138770604, + "tokens_seen": 208797696 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004730892678034102, + "loss": 1.8447, + "theoretical_loss": 4.360291162182179, + "tokens_seen": 208863232 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047307923771313943, + "loss": 1.8036, + "theoretical_loss": 4.360122253446472, + "tokens_seen": 208928768 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004730692076228686, + "loss": 1.7655, + "theoretical_loss": 4.359953412514965, + "tokens_seen": 208994304 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004730591775325978, + "loss": 1.7263, + "theoretical_loss": 4.359784639339191, + "tokens_seen": 209059840 + }, + { + "epoch": 0.06, + "learning_rate": 0.000473049147442327, + "loss": 1.6313, + "theoretical_loss": 4.359615933870732, + "tokens_seen": 209125376 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047303911735205615, + "loss": 1.6989, + "theoretical_loss": 4.359447296061219, + "tokens_seen": 209190912 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047302908726178534, + "loss": 1.6719, + "theoretical_loss": 4.359278725862336, + "tokens_seen": 209256448 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047301905717151457, + "loss": 1.8476, + "theoretical_loss": 4.359110223225814, + "tokens_seen": 209321984 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004730090270812437, + "loss": 1.8312, + "theoretical_loss": 4.358941788103433, + "tokens_seen": 209387520 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047299899699097293, + "loss": 1.6791, + "theoretical_loss": 4.358773420447024, + "tokens_seen": 209453056 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004729889669007021, + "loss": 1.8442, + "theoretical_loss": 4.358605120208469, + "tokens_seen": 209518592 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004729789368104313, + "loss": 1.607, + "theoretical_loss": 4.358436887339694, + "tokens_seen": 209584128 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047296890672016053, + "loss": 1.6331, + "theoretical_loss": 4.35826872179268, + "tokens_seen": 209649664 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 76100, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.7140634059906006, + "objective/train/theoretical_loss": 4.358100623519453, + "objective/train/tokens_used": 209715200, + "theoretical_loss": 4.358100623519453, + "tokens_seen": 209715200 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047295887662988966, + "loss": 1.6511, + "theoretical_loss": 4.358100623519453, + "tokens_seen": 209715200 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004729488465396189, + "loss": 1.7834, + "theoretical_loss": 4.357932592472092, + "tokens_seen": 209780736 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004729388164493481, + "loss": 1.752, + "theoretical_loss": 4.357764628602721, + "tokens_seen": 209846272 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047292878635907726, + "loss": 1.7403, + "theoretical_loss": 4.357596731863517, + "tokens_seen": 209911808 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047291875626880644, + "loss": 1.8053, + "theoretical_loss": 4.357428902206704, + "tokens_seen": 209977344 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004729087261785356, + "loss": 1.6382, + "theoretical_loss": 4.357261139584553, + "tokens_seen": 210042880 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004728986960882648, + "loss": 1.6434, + "theoretical_loss": 4.357093443949388, + "tokens_seen": 210108416 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047288866599799403, + "loss": 1.7374, + "theoretical_loss": 4.356925815253579, + "tokens_seen": 210173952 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047287863590772316, + "loss": 1.6047, + "theoretical_loss": 4.356758253449544, + "tokens_seen": 210239488 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004728686058174524, + "loss": 1.6913, + "theoretical_loss": 4.356590758489753, + "tokens_seen": 210305024 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004728585757271815, + "loss": 1.6411, + "theoretical_loss": 4.356423330326722, + "tokens_seen": 210370560 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047284854563691076, + "loss": 1.6241, + "theoretical_loss": 4.356255968913015, + "tokens_seen": 210436096 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047283851554663994, + "loss": 1.584, + "theoretical_loss": 4.356088674201246, + "tokens_seen": 210501632 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004728284854563691, + "loss": 1.7592, + "theoretical_loss": 4.355921446144077, + "tokens_seen": 210567168 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004728184553660983, + "loss": 1.7344, + "theoretical_loss": 4.355754284694218, + "tokens_seen": 210632704 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047280842527582754, + "loss": 1.7415, + "theoretical_loss": 4.355587189804427, + "tokens_seen": 210698240 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047279839518555666, + "loss": 1.7257, + "theoretical_loss": 4.355420161427512, + "tokens_seen": 210763776 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004727883650952859, + "loss": 1.8175, + "theoretical_loss": 4.355253199516326, + "tokens_seen": 210829312 + }, + { + "epoch": 0.06, + "learning_rate": 0.000472778335005015, + "loss": 1.6529, + "theoretical_loss": 4.355086304023773, + "tokens_seen": 210894848 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047276830491474426, + "loss": 1.7009, + "theoretical_loss": 4.354919474902804, + "tokens_seen": 210960384 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047275827482447344, + "loss": 1.5581, + "theoretical_loss": 4.354752712106417, + "tokens_seen": 211025920 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004727482447342026, + "loss": 1.8127, + "theoretical_loss": 4.354586015587659, + "tokens_seen": 211091456 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004727382146439318, + "loss": 1.6569, + "theoretical_loss": 4.354419385299623, + "tokens_seen": 211156992 + }, + { + "epoch": 0.06, + "learning_rate": 0.000472728184553661, + "loss": 1.7355, + "theoretical_loss": 4.354252821195453, + "tokens_seen": 211222528 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047271815446339017, + "loss": 1.6305, + "theoretical_loss": 4.354086323228338, + "tokens_seen": 211288064 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004727081243731194, + "loss": 1.6337, + "theoretical_loss": 4.353919891351516, + "tokens_seen": 211353600 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047269809428284853, + "loss": 1.6398, + "theoretical_loss": 4.35375352551827, + "tokens_seen": 211419136 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047268806419257777, + "loss": 1.7322, + "theoretical_loss": 4.353587225681935, + "tokens_seen": 211484672 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047267803410230695, + "loss": 1.7447, + "theoretical_loss": 4.35342099179589, + "tokens_seen": 211550208 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047266800401203613, + "loss": 1.5947, + "theoretical_loss": 4.353254823813561, + "tokens_seen": 211615744 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004726579739217653, + "loss": 1.6954, + "theoretical_loss": 4.353088721688424, + "tokens_seen": 211681280 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004726479438314945, + "loss": 1.676, + "theoretical_loss": 4.352922685374001, + "tokens_seen": 211746816 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047263791374122367, + "loss": 1.54, + "theoretical_loss": 4.352756714823859, + "tokens_seen": 211812352 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004726278836509529, + "loss": 1.5945, + "theoretical_loss": 4.352590809991616, + "tokens_seen": 211877888 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047261785356068203, + "loss": 1.6854, + "theoretical_loss": 4.352424970830935, + "tokens_seen": 211943424 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047260782347041127, + "loss": 1.8027, + "theoretical_loss": 4.352259197295525, + "tokens_seen": 212008960 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004725977933801404, + "loss": 1.6308, + "theoretical_loss": 4.352093489339143, + "tokens_seen": 212074496 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047258776328986963, + "loss": 1.8024, + "theoretical_loss": 4.351927846915595, + "tokens_seen": 212140032 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004725777331995988, + "loss": 1.6655, + "theoretical_loss": 4.35176226997873, + "tokens_seen": 212205568 + }, + { + "epoch": 0.06, + "learning_rate": 0.000472567703109328, + "loss": 1.7383, + "theoretical_loss": 4.351596758482445, + "tokens_seen": 212271104 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004725576730190572, + "loss": 1.7104, + "theoretical_loss": 4.351431312380684, + "tokens_seen": 212336640 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047254764292878636, + "loss": 1.7439, + "theoretical_loss": 4.351265931627439, + "tokens_seen": 212402176 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047253761283851554, + "loss": 1.7151, + "theoretical_loss": 4.351100616176747, + "tokens_seen": 212467712 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047252758274824477, + "loss": 1.7292, + "theoretical_loss": 4.350935365982691, + "tokens_seen": 212533248 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004725175526579739, + "loss": 1.7258, + "theoretical_loss": 4.350770180999402, + "tokens_seen": 212598784 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047250752256770313, + "loss": 1.6215, + "theoretical_loss": 4.3506050611810565, + "tokens_seen": 212664320 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004724974924774323, + "loss": 1.6248, + "theoretical_loss": 4.350440006481877, + "tokens_seen": 212729856 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004724874623871615, + "loss": 1.6693, + "theoretical_loss": 4.350275016856132, + "tokens_seen": 212795392 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004724774322968907, + "loss": 1.8046, + "theoretical_loss": 4.350110092258138, + "tokens_seen": 212860928 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047246740220661986, + "loss": 1.6024, + "theoretical_loss": 4.349945232642256, + "tokens_seen": 212926464 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 77027, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6623857021331787, + "objective/train/theoretical_loss": 4.3497804379628935, + "objective/train/tokens_used": 212992000, + "theoretical_loss": 4.3497804379628935, + "tokens_seen": 212992000 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047245737211634904, + "loss": 1.671, + "theoretical_loss": 4.3497804379628935, + "tokens_seen": 212992000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004724473420260783, + "loss": 1.74, + "theoretical_loss": 4.349615708174504, + "tokens_seen": 213057536 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004724373119358074, + "loss": 1.7088, + "theoretical_loss": 4.3494510432315865, + "tokens_seen": 213123072 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047242728184553664, + "loss": 1.6269, + "theoretical_loss": 4.349286443088687, + "tokens_seen": 213188608 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047241725175526576, + "loss": 1.6231, + "theoretical_loss": 4.349121907700396, + "tokens_seen": 213254144 + }, + { + "epoch": 0.06, + "learning_rate": 0.000472407221664995, + "loss": 1.691, + "theoretical_loss": 4.348957437021351, + "tokens_seen": 213319680 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004723971915747242, + "loss": 1.6605, + "theoretical_loss": 4.348793031006235, + "tokens_seen": 213385216 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047238716148445336, + "loss": 1.5757, + "theoretical_loss": 4.348628689609775, + "tokens_seen": 213450752 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047237713139418254, + "loss": 1.6711, + "theoretical_loss": 4.348464412786746, + "tokens_seen": 213516288 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004723671013039117, + "loss": 1.6072, + "theoretical_loss": 4.348300200491966, + "tokens_seen": 213581824 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004723570712136409, + "loss": 1.6947, + "theoretical_loss": 4.348136052680301, + "tokens_seen": 213647360 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047234704112337014, + "loss": 1.6839, + "theoretical_loss": 4.347971969306662, + "tokens_seen": 213712896 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047233701103309927, + "loss": 1.7587, + "theoretical_loss": 4.347807950326002, + "tokens_seen": 213778432 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004723269809428285, + "loss": 1.676, + "theoretical_loss": 4.347643995693325, + "tokens_seen": 213843968 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004723169508525577, + "loss": 1.644, + "theoretical_loss": 4.347480105363674, + "tokens_seen": 213909504 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047230692076228686, + "loss": 1.7224, + "theoretical_loss": 4.3473162792921425, + "tokens_seen": 213975040 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047229689067201605, + "loss": 1.7709, + "theoretical_loss": 4.347152517433866, + "tokens_seen": 214040576 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004722868605817452, + "loss": 1.6459, + "theoretical_loss": 4.346988819744026, + "tokens_seen": 214106112 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004722768304914744, + "loss": 1.7301, + "theoretical_loss": 4.346825186177848, + "tokens_seen": 214171648 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047226680040120364, + "loss": 1.6931, + "theoretical_loss": 4.346661616690605, + "tokens_seen": 214237184 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047225677031093277, + "loss": 1.6482, + "theoretical_loss": 4.346498111237612, + "tokens_seen": 214302720 + }, + { + "epoch": 0.06, + "learning_rate": 0.000472246740220662, + "loss": 1.6861, + "theoretical_loss": 4.346334669774231, + "tokens_seen": 214368256 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047223671013039113, + "loss": 1.6323, + "theoretical_loss": 4.346171292255867, + "tokens_seen": 214433792 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047222668004012037, + "loss": 1.6069, + "theoretical_loss": 4.34600797863797, + "tokens_seen": 214499328 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004722166499498496, + "loss": 1.555, + "theoretical_loss": 4.345844728876036, + "tokens_seen": 214564864 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047220661985957873, + "loss": 1.6293, + "theoretical_loss": 4.345681542925604, + "tokens_seen": 214630400 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047219658976930797, + "loss": 1.7763, + "theoretical_loss": 4.345518420742259, + "tokens_seen": 214695936 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047218655967903715, + "loss": 1.5174, + "theoretical_loss": 4.345355362281628, + "tokens_seen": 214761472 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047217652958876633, + "loss": 1.7207, + "theoretical_loss": 4.345192367499386, + "tokens_seen": 214827008 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004721664994984955, + "loss": 1.6085, + "theoretical_loss": 4.3450294363512505, + "tokens_seen": 214892544 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004721564694082247, + "loss": 1.6146, + "theoretical_loss": 4.34486656879298, + "tokens_seen": 214958080 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047214643931795387, + "loss": 1.7515, + "theoretical_loss": 4.344703764780386, + "tokens_seen": 215023616 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004721364092276831, + "loss": 1.6677, + "theoretical_loss": 4.344541024269312, + "tokens_seen": 215089152 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047212637913741223, + "loss": 1.6228, + "theoretical_loss": 4.344378347215657, + "tokens_seen": 215154688 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047211634904714147, + "loss": 1.7143, + "theoretical_loss": 4.344215733575357, + "tokens_seen": 215220224 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004721063189568706, + "loss": 1.5993, + "theoretical_loss": 4.344053183304396, + "tokens_seen": 215285760 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047209628886659983, + "loss": 1.7059, + "theoretical_loss": 4.343890696358798, + "tokens_seen": 215351296 + }, + { + "epoch": 0.07, + "learning_rate": 0.000472086258776329, + "loss": 1.7065, + "theoretical_loss": 4.343728272694635, + "tokens_seen": 215416832 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004720762286860582, + "loss": 1.6889, + "theoretical_loss": 4.34356591226802, + "tokens_seen": 215482368 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004720661985957874, + "loss": 1.5507, + "theoretical_loss": 4.343403615035111, + "tokens_seen": 215547904 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047205616850551656, + "loss": 1.6456, + "theoretical_loss": 4.343241380952109, + "tokens_seen": 215613440 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047204613841524574, + "loss": 1.608, + "theoretical_loss": 4.34307920997526, + "tokens_seen": 215678976 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047203610832497497, + "loss": 1.5927, + "theoretical_loss": 4.34291710206085, + "tokens_seen": 215744512 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004720260782347041, + "loss": 1.7173, + "theoretical_loss": 4.342755057165214, + "tokens_seen": 215810048 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047201604814443333, + "loss": 1.6788, + "theoretical_loss": 4.342593075244728, + "tokens_seen": 215875584 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004720060180541625, + "loss": 1.5327, + "theoretical_loss": 4.342431156255809, + "tokens_seen": 215941120 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004719959879638917, + "loss": 1.7129, + "theoretical_loss": 4.3422693001549195, + "tokens_seen": 216006656 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004719859578736209, + "loss": 1.5607, + "theoretical_loss": 4.342107506898567, + "tokens_seen": 216072192 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047197592778335006, + "loss": 1.6627, + "theoretical_loss": 4.3419457764433, + "tokens_seen": 216137728 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047196589769307924, + "loss": 1.7395, + "theoretical_loss": 4.34178410874571, + "tokens_seen": 216203264 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 78159, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.7622928619384766, + "objective/train/theoretical_loss": 4.341622503762434, + "objective/train/tokens_used": 216268800, + "theoretical_loss": 4.341622503762434, + "tokens_seen": 216268800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004719558676028085, + "loss": 1.7845, + "theoretical_loss": 4.341622503762434, + "tokens_seen": 216268800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004719458375125376, + "loss": 1.7383, + "theoretical_loss": 4.341460961450148, + "tokens_seen": 216334336 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047193580742226684, + "loss": 1.7009, + "theoretical_loss": 4.341299481765575, + "tokens_seen": 216399872 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047192577733199596, + "loss": 1.7292, + "theoretical_loss": 4.34113806466548, + "tokens_seen": 216465408 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004719157472417252, + "loss": 1.694, + "theoretical_loss": 4.340976710106671, + "tokens_seen": 216530944 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004719057171514544, + "loss": 1.6399, + "theoretical_loss": 4.340815418045995, + "tokens_seen": 216596480 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047189568706118356, + "loss": 1.4747, + "theoretical_loss": 4.340654188440349, + "tokens_seen": 216662016 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047188565697091274, + "loss": 1.595, + "theoretical_loss": 4.340493021246668, + "tokens_seen": 216727552 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004718756268806419, + "loss": 1.6706, + "theoretical_loss": 4.3403319164219285, + "tokens_seen": 216793088 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004718655967903711, + "loss": 1.7071, + "theoretical_loss": 4.340170873923155, + "tokens_seen": 216858624 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047185556670010034, + "loss": 1.6805, + "theoretical_loss": 4.340009893707408, + "tokens_seen": 216924160 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047184553660982947, + "loss": 1.616, + "theoretical_loss": 4.339848975731796, + "tokens_seen": 216989696 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004718355065195587, + "loss": 1.6792, + "theoretical_loss": 4.3396881199534665, + "tokens_seen": 217055232 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004718254764292879, + "loss": 1.6154, + "theoretical_loss": 4.339527326329613, + "tokens_seen": 217120768 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047181544633901706, + "loss": 1.7016, + "theoretical_loss": 4.3393665948174664, + "tokens_seen": 217186304 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047180541624874625, + "loss": 1.7092, + "theoretical_loss": 4.339205925374305, + "tokens_seen": 217251840 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004717953861584754, + "loss": 1.5578, + "theoretical_loss": 4.339045317957446, + "tokens_seen": 217317376 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004717853560682046, + "loss": 1.603, + "theoretical_loss": 4.33888477252425, + "tokens_seen": 217382912 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047177532597793384, + "loss": 1.5928, + "theoretical_loss": 4.33872428903212, + "tokens_seen": 217448448 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047176529588766297, + "loss": 1.5806, + "theoretical_loss": 4.338563867438499, + "tokens_seen": 217513984 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004717552657973922, + "loss": 1.6988, + "theoretical_loss": 4.338403507700877, + "tokens_seen": 217579520 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047174523570712133, + "loss": 1.6893, + "theoretical_loss": 4.33824320977678, + "tokens_seen": 217645056 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047173520561685057, + "loss": 1.6013, + "theoretical_loss": 4.33808297362378, + "tokens_seen": 217710592 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047172517552657975, + "loss": 1.7586, + "theoretical_loss": 4.337922799199489, + "tokens_seen": 217776128 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047171514543630893, + "loss": 1.6769, + "theoretical_loss": 4.337762686461561, + "tokens_seen": 217841664 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004717051153460381, + "loss": 1.6397, + "theoretical_loss": 4.337602635367694, + "tokens_seen": 217907200 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047169508525576735, + "loss": 1.6896, + "theoretical_loss": 4.337442645875624, + "tokens_seen": 217972736 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004716850551654965, + "loss": 1.7286, + "theoretical_loss": 4.337282717943132, + "tokens_seen": 218038272 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004716750250752257, + "loss": 1.7309, + "theoretical_loss": 4.337122851528037, + "tokens_seen": 218103808 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047166499498495484, + "loss": 1.8042, + "theoretical_loss": 4.336963046588204, + "tokens_seen": 218169344 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047165496489468407, + "loss": 1.7959, + "theoretical_loss": 4.336803303081538, + "tokens_seen": 218234880 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047164493480441325, + "loss": 1.6666, + "theoretical_loss": 4.336643620965982, + "tokens_seen": 218300416 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047163490471414243, + "loss": 1.7245, + "theoretical_loss": 4.336484000199524, + "tokens_seen": 218365952 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004716248746238716, + "loss": 1.6564, + "theoretical_loss": 4.336324440740195, + "tokens_seen": 218431488 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004716148445336008, + "loss": 1.7496, + "theoretical_loss": 4.336164942546061, + "tokens_seen": 218497024 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047160481444333, + "loss": 1.6226, + "theoretical_loss": 4.336005505575235, + "tokens_seen": 218562560 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715947843530592, + "loss": 1.5408, + "theoretical_loss": 4.335846129785869, + "tokens_seen": 218628096 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047158475426278834, + "loss": 1.6896, + "theoretical_loss": 4.335686815136157, + "tokens_seen": 218693632 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715747241725176, + "loss": 1.612, + "theoretical_loss": 4.335527561584333, + "tokens_seen": 218759168 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715646940822467, + "loss": 1.7758, + "theoretical_loss": 4.3353683690886715, + "tokens_seen": 218824704 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047155466399197594, + "loss": 1.7073, + "theoretical_loss": 4.33520923760749, + "tokens_seen": 218890240 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715446339017051, + "loss": 1.7081, + "theoretical_loss": 4.335050167099145, + "tokens_seen": 218955776 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715346038114343, + "loss": 1.7913, + "theoretical_loss": 4.334891157522035, + "tokens_seen": 219021312 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715245737211635, + "loss": 1.7755, + "theoretical_loss": 4.3347322088346, + "tokens_seen": 219086848 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715145436308927, + "loss": 1.6664, + "theoretical_loss": 4.33457332099532, + "tokens_seen": 219152384 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047150451354062184, + "loss": 1.6373, + "theoretical_loss": 4.334414493962714, + "tokens_seen": 219217920 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004714944834503511, + "loss": 1.4418, + "theoretical_loss": 4.334255727695344, + "tokens_seen": 219283456 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004714844533600802, + "loss": 1.6616, + "theoretical_loss": 4.334097022151812, + "tokens_seen": 219348992 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047147442326980944, + "loss": 1.7177, + "theoretical_loss": 4.33393837729076, + "tokens_seen": 219414528 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004714643931795387, + "loss": 1.6431, + "theoretical_loss": 4.333779793070871, + "tokens_seen": 219480064 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 79131, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.7466304302215576, + "objective/train/theoretical_loss": 4.33362126945087, + "objective/train/tokens_used": 219545600, + "theoretical_loss": 4.33362126945087, + "tokens_seen": 219545600 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004714543630892678, + "loss": 1.7365, + "theoretical_loss": 4.33362126945087, + "tokens_seen": 219545600 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047144433299899704, + "loss": 1.5938, + "theoretical_loss": 4.333462806389517, + "tokens_seen": 219611136 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047143430290872616, + "loss": 1.6979, + "theoretical_loss": 4.33330440384562, + "tokens_seen": 219676672 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004714242728184554, + "loss": 1.6235, + "theoretical_loss": 4.333146061778021, + "tokens_seen": 219742208 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004714142427281846, + "loss": 1.6621, + "theoretical_loss": 4.332987780145606, + "tokens_seen": 219807744 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047140421263791376, + "loss": 1.556, + "theoretical_loss": 4.3328295589073, + "tokens_seen": 219873280 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047139418254764294, + "loss": 1.6747, + "theoretical_loss": 4.332671398022067, + "tokens_seen": 219938816 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004713841524573721, + "loss": 1.5807, + "theoretical_loss": 4.332513297448912, + "tokens_seen": 220004352 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004713741223671013, + "loss": 1.7739, + "theoretical_loss": 4.332355257146881, + "tokens_seen": 220069888 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047136409227683054, + "loss": 1.571, + "theoretical_loss": 4.3321972770750605, + "tokens_seen": 220135424 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047135406218655967, + "loss": 1.6896, + "theoretical_loss": 4.332039357192572, + "tokens_seen": 220200960 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004713440320962889, + "loss": 1.6291, + "theoretical_loss": 4.331881497458584, + "tokens_seen": 220266496 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004713340020060181, + "loss": 1.6428, + "theoretical_loss": 4.3317236978323, + "tokens_seen": 220332032 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047132397191574726, + "loss": 1.6634, + "theoretical_loss": 4.3315659582729635, + "tokens_seen": 220397568 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047131394182547645, + "loss": 1.6632, + "theoretical_loss": 4.331408278739861, + "tokens_seen": 220463104 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047130391173520563, + "loss": 1.5946, + "theoretical_loss": 4.331250659192316, + "tokens_seen": 220528640 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004712938816449348, + "loss": 1.7317, + "theoretical_loss": 4.33109309958969, + "tokens_seen": 220594176 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047128385155466404, + "loss": 1.6715, + "theoretical_loss": 4.330935599891389, + "tokens_seen": 220659712 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047127382146439317, + "loss": 1.7646, + "theoretical_loss": 4.330778160056855, + "tokens_seen": 220725248 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004712637913741224, + "loss": 1.6464, + "theoretical_loss": 4.330620780045571, + "tokens_seen": 220790784 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047125376128385153, + "loss": 1.5344, + "theoretical_loss": 4.330463459817057, + "tokens_seen": 220856320 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047124373119358077, + "loss": 1.7299, + "theoretical_loss": 4.3303061993308765, + "tokens_seen": 220921856 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047123370110330995, + "loss": 1.6199, + "theoretical_loss": 4.330148998546628, + "tokens_seen": 220987392 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047122367101303913, + "loss": 1.7898, + "theoretical_loss": 4.329991857423953, + "tokens_seen": 221052928 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004712136409227683, + "loss": 1.5997, + "theoretical_loss": 4.329834775922529, + "tokens_seen": 221118464 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047120361083249755, + "loss": 1.7113, + "theoretical_loss": 4.329677754002074, + "tokens_seen": 221184000 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004711935807422267, + "loss": 1.6299, + "theoretical_loss": 4.329520791622348, + "tokens_seen": 221249536 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004711835506519559, + "loss": 1.7171, + "theoretical_loss": 4.329363888743145, + "tokens_seen": 221315072 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047117352056168504, + "loss": 1.5941, + "theoretical_loss": 4.329207045324301, + "tokens_seen": 221380608 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047116349047141427, + "loss": 1.6697, + "theoretical_loss": 4.3290502613256905, + "tokens_seen": 221446144 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047115346038114345, + "loss": 1.6362, + "theoretical_loss": 4.328893536707228, + "tokens_seen": 221511680 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047114343029087263, + "loss": 1.6472, + "theoretical_loss": 4.328736871428864, + "tokens_seen": 221577216 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004711334002006018, + "loss": 1.627, + "theoretical_loss": 4.328580265450592, + "tokens_seen": 221642752 + }, + { + "epoch": 0.07, + "learning_rate": 0.000471123370110331, + "loss": 1.7101, + "theoretical_loss": 4.32842371873244, + "tokens_seen": 221708288 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004711133400200602, + "loss": 1.8139, + "theoretical_loss": 4.328267231234476, + "tokens_seen": 221773824 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004711033099297894, + "loss": 1.824, + "theoretical_loss": 4.328110802916811, + "tokens_seen": 221839360 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047109327983951854, + "loss": 1.6214, + "theoretical_loss": 4.327954433739588, + "tokens_seen": 221904896 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004710832497492478, + "loss": 1.5895, + "theoretical_loss": 4.327798123662992, + "tokens_seen": 221970432 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004710732196589769, + "loss": 1.5849, + "theoretical_loss": 4.327641872647248, + "tokens_seen": 222035968 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047106318956870614, + "loss": 1.72, + "theoretical_loss": 4.327485680652615, + "tokens_seen": 222101504 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004710531594784353, + "loss": 1.6308, + "theoretical_loss": 4.327329547639396, + "tokens_seen": 222167040 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004710431293881645, + "loss": 1.8127, + "theoretical_loss": 4.327173473567927, + "tokens_seen": 222232576 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004710330992978937, + "loss": 1.7682, + "theoretical_loss": 4.327017458398587, + "tokens_seen": 222298112 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004710230692076229, + "loss": 1.717, + "theoretical_loss": 4.32686150209179, + "tokens_seen": 222363648 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047101303911735204, + "loss": 1.5562, + "theoretical_loss": 4.32670560460799, + "tokens_seen": 222429184 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004710030090270813, + "loss": 1.5184, + "theoretical_loss": 4.326549765907679, + "tokens_seen": 222494720 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004709929789368104, + "loss": 1.675, + "theoretical_loss": 4.326393985951386, + "tokens_seen": 222560256 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047098294884653964, + "loss": 1.7405, + "theoretical_loss": 4.326238264699679, + "tokens_seen": 222625792 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004709729187562688, + "loss": 1.6768, + "theoretical_loss": 4.326082602113164, + "tokens_seen": 222691328 + }, + { + "epoch": 0.07, + "learning_rate": 0.000470962888665998, + "loss": 1.5918, + "theoretical_loss": 4.325926998152487, + "tokens_seen": 222756864 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 80178, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.8502051830291748, + "objective/train/theoretical_loss": 4.325771452778328, + "objective/train/tokens_used": 222822400, + "theoretical_loss": 4.325771452778328, + "tokens_seen": 222822400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004709528585757272, + "loss": 1.7929, + "theoretical_loss": 4.325771452778328, + "tokens_seen": 222822400 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047094282848545636, + "loss": 1.5346, + "theoretical_loss": 4.325615965951408, + "tokens_seen": 222887936 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047093279839518555, + "loss": 1.6992, + "theoretical_loss": 4.325460537632483, + "tokens_seen": 222953472 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004709227683049148, + "loss": 1.6143, + "theoretical_loss": 4.325305167782352, + "tokens_seen": 223019008 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004709127382146439, + "loss": 1.6642, + "theoretical_loss": 4.325149856361845, + "tokens_seen": 223084544 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047090270812437314, + "loss": 1.8127, + "theoretical_loss": 4.3249946033318345, + "tokens_seen": 223150080 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047089267803410227, + "loss": 1.5982, + "theoretical_loss": 4.32483940865323, + "tokens_seen": 223215616 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004708826479438315, + "loss": 1.6061, + "theoretical_loss": 4.324684272286978, + "tokens_seen": 223281152 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004708726178535607, + "loss": 1.6414, + "theoretical_loss": 4.324529194194062, + "tokens_seen": 223346688 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047086258776328987, + "loss": 1.6916, + "theoretical_loss": 4.324374174335503, + "tokens_seen": 223412224 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047085255767301905, + "loss": 1.6595, + "theoretical_loss": 4.324219212672363, + "tokens_seen": 223477760 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004708425275827483, + "loss": 1.6899, + "theoretical_loss": 4.324064309165734, + "tokens_seen": 223543296 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004708324974924774, + "loss": 1.5896, + "theoretical_loss": 4.3239094637767534, + "tokens_seen": 223608832 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047082246740220665, + "loss": 1.6743, + "theoretical_loss": 4.323754676466592, + "tokens_seen": 223674368 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004708124373119358, + "loss": 1.7039, + "theoretical_loss": 4.323599947196458, + "tokens_seen": 223739904 + }, + { + "epoch": 0.07, + "learning_rate": 0.000470802407221665, + "loss": 1.6486, + "theoretical_loss": 4.323445275927597, + "tokens_seen": 223805440 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004707923771313942, + "loss": 1.667, + "theoretical_loss": 4.323290662621292, + "tokens_seen": 223870976 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047078234704112337, + "loss": 1.6557, + "theoretical_loss": 4.323136107238865, + "tokens_seen": 223936512 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047077231695085255, + "loss": 1.788, + "theoretical_loss": 4.322981609741671, + "tokens_seen": 224002048 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047076228686058173, + "loss": 1.7363, + "theoretical_loss": 4.322827170091106, + "tokens_seen": 224067584 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004707522567703109, + "loss": 1.6085, + "theoretical_loss": 4.322672788248601, + "tokens_seen": 224133120 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047074222668004015, + "loss": 1.7159, + "theoretical_loss": 4.3225184641756265, + "tokens_seen": 224198656 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004707321965897693, + "loss": 1.705, + "theoretical_loss": 4.322364197833684, + "tokens_seen": 224264192 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004707221664994985, + "loss": 1.6383, + "theoretical_loss": 4.322209989184319, + "tokens_seen": 224329728 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047071213640922775, + "loss": 1.6435, + "theoretical_loss": 4.32205583818911, + "tokens_seen": 224395264 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004707021063189569, + "loss": 1.7285, + "theoretical_loss": 4.321901744809672, + "tokens_seen": 224460800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004706920762286861, + "loss": 1.711, + "theoretical_loss": 4.321747709007658, + "tokens_seen": 224526336 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047068204613841524, + "loss": 1.7201, + "theoretical_loss": 4.321593730744759, + "tokens_seen": 224591872 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047067201604814447, + "loss": 1.6397, + "theoretical_loss": 4.3214398099826985, + "tokens_seen": 224657408 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047066198595787365, + "loss": 1.7167, + "theoretical_loss": 4.321285946683242, + "tokens_seen": 224722944 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047065195586760283, + "loss": 1.5423, + "theoretical_loss": 4.321132140808186, + "tokens_seen": 224788480 + }, + { + "epoch": 0.07, + "learning_rate": 0.000470641925777332, + "loss": 1.7135, + "theoretical_loss": 4.320978392319368, + "tokens_seen": 224854016 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004706318956870612, + "loss": 1.6109, + "theoretical_loss": 4.320824701178661, + "tokens_seen": 224919552 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004706218655967904, + "loss": 1.7456, + "theoretical_loss": 4.320671067347972, + "tokens_seen": 224985088 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004706118355065196, + "loss": 1.5999, + "theoretical_loss": 4.320517490789246, + "tokens_seen": 225050624 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047060180541624874, + "loss": 1.6461, + "theoretical_loss": 4.320363971464466, + "tokens_seen": 225116160 + }, + { + "epoch": 0.07, + "learning_rate": 0.000470591775325978, + "loss": 1.5591, + "theoretical_loss": 4.320210509335649, + "tokens_seen": 225181696 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004705817452357071, + "loss": 1.6741, + "theoretical_loss": 4.320057104364848, + "tokens_seen": 225247232 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047057171514543634, + "loss": 1.6479, + "theoretical_loss": 4.3199037565141545, + "tokens_seen": 225312768 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004705616850551655, + "loss": 1.7342, + "theoretical_loss": 4.319750465745694, + "tokens_seen": 225378304 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004705516549648947, + "loss": 1.6121, + "theoretical_loss": 4.31959723202163, + "tokens_seen": 225443840 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004705416248746239, + "loss": 1.6391, + "theoretical_loss": 4.319444055304161, + "tokens_seen": 225509376 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004705315947843531, + "loss": 1.732, + "theoretical_loss": 4.319290935555521, + "tokens_seen": 225574912 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047052156469408224, + "loss": 1.7277, + "theoretical_loss": 4.319137872737979, + "tokens_seen": 225640448 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004705115346038115, + "loss": 1.6741, + "theoretical_loss": 4.318984866813844, + "tokens_seen": 225705984 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004705015045135406, + "loss": 1.7276, + "theoretical_loss": 4.318831917745458, + "tokens_seen": 225771520 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047049147442326984, + "loss": 1.5895, + "theoretical_loss": 4.318679025495197, + "tokens_seen": 225837056 + }, + { + "epoch": 0.07, + "learning_rate": 0.000470481444332999, + "loss": 1.6964, + "theoretical_loss": 4.318526190025478, + "tokens_seen": 225902592 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004704714142427282, + "loss": 1.7262, + "theoretical_loss": 4.318373411298749, + "tokens_seen": 225968128 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004704613841524574, + "loss": 1.7377, + "theoretical_loss": 4.318220689277496, + "tokens_seen": 226033664 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 81349, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6773018836975098, + "objective/train/theoretical_loss": 4.318068023924241, + "objective/train/tokens_used": 226099200, + "theoretical_loss": 4.318068023924241, + "tokens_seen": 226099200 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047045135406218656, + "loss": 1.6685, + "theoretical_loss": 4.318068023924241, + "tokens_seen": 226099200 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047044132397191575, + "loss": 1.5509, + "theoretical_loss": 4.317915415201538, + "tokens_seen": 226164736 + }, + { + "epoch": 0.07, + "learning_rate": 0.000470431293881645, + "loss": 1.6875, + "theoretical_loss": 4.317762863071982, + "tokens_seen": 226230272 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004704212637913741, + "loss": 1.6082, + "theoretical_loss": 4.317610367498201, + "tokens_seen": 226295808 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047041123370110334, + "loss": 1.6034, + "theoretical_loss": 4.317457928442856, + "tokens_seen": 226361344 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047040120361083247, + "loss": 1.7008, + "theoretical_loss": 4.317305545868648, + "tokens_seen": 226426880 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004703911735205617, + "loss": 1.7113, + "theoretical_loss": 4.317153219738311, + "tokens_seen": 226492416 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004703811434302909, + "loss": 1.6807, + "theoretical_loss": 4.317000950014615, + "tokens_seen": 226557952 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047037111334002007, + "loss": 1.6318, + "theoretical_loss": 4.316848736660363, + "tokens_seen": 226623488 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047036108324974925, + "loss": 1.5554, + "theoretical_loss": 4.316696579638396, + "tokens_seen": 226689024 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004703510531594785, + "loss": 1.7297, + "theoretical_loss": 4.316544478911592, + "tokens_seen": 226754560 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004703410230692076, + "loss": 1.6658, + "theoretical_loss": 4.316392434442858, + "tokens_seen": 226820096 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047033099297893685, + "loss": 1.6882, + "theoretical_loss": 4.316240446195142, + "tokens_seen": 226885632 + }, + { + "epoch": 0.07, + "learning_rate": 0.000470320962888666, + "loss": 1.5902, + "theoretical_loss": 4.3160885141314225, + "tokens_seen": 226951168 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004703109327983952, + "loss": 1.7629, + "theoretical_loss": 4.315936638214718, + "tokens_seen": 227016704 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004703009027081244, + "loss": 1.6976, + "theoretical_loss": 4.315784818408079, + "tokens_seen": 227082240 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047029087261785357, + "loss": 1.641, + "theoretical_loss": 4.31563305467459, + "tokens_seen": 227147776 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047028084252758275, + "loss": 1.5817, + "theoretical_loss": 4.315481346977374, + "tokens_seen": 227213312 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047027081243731193, + "loss": 1.6858, + "theoretical_loss": 4.315329695279584, + "tokens_seen": 227278848 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004702607823470411, + "loss": 1.6276, + "theoretical_loss": 4.315178099544413, + "tokens_seen": 227344384 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047025075225677035, + "loss": 1.6314, + "theoretical_loss": 4.315026559735085, + "tokens_seen": 227409920 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004702407221664995, + "loss": 1.6111, + "theoretical_loss": 4.314875075814861, + "tokens_seen": 227475456 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004702306920762287, + "loss": 1.6073, + "theoretical_loss": 4.314723647747035, + "tokens_seen": 227540992 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047022066198595784, + "loss": 1.7355, + "theoretical_loss": 4.314572275494937, + "tokens_seen": 227606528 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004702106318956871, + "loss": 1.6426, + "theoretical_loss": 4.3144209590219305, + "tokens_seen": 227672064 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047020060180541626, + "loss": 1.6409, + "theoretical_loss": 4.314269698291415, + "tokens_seen": 227737600 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047019057171514544, + "loss": 1.641, + "theoretical_loss": 4.3141184932668235, + "tokens_seen": 227803136 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004701805416248746, + "loss": 1.7065, + "theoretical_loss": 4.3139673439116235, + "tokens_seen": 227868672 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047017051153460385, + "loss": 1.7189, + "theoretical_loss": 4.313816250189317, + "tokens_seen": 227934208 + }, + { + "epoch": 0.07, + "learning_rate": 0.000470160481444333, + "loss": 1.7125, + "theoretical_loss": 4.313665212063441, + "tokens_seen": 227999744 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004701504513540622, + "loss": 1.6333, + "theoretical_loss": 4.313514229497566, + "tokens_seen": 228065280 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047014042126379134, + "loss": 1.5272, + "theoretical_loss": 4.313363302455299, + "tokens_seen": 228130816 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004701303911735206, + "loss": 1.6885, + "theoretical_loss": 4.313212430900277, + "tokens_seen": 228196352 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047012036108324976, + "loss": 1.676, + "theoretical_loss": 4.313061614796176, + "tokens_seen": 228261888 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047011033099297894, + "loss": 1.7251, + "theoretical_loss": 4.312910854106702, + "tokens_seen": 228327424 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004701003009027081, + "loss": 1.6291, + "theoretical_loss": 4.3127601487956, + "tokens_seen": 228392960 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004700902708124373, + "loss": 1.6642, + "theoretical_loss": 4.312609498826644, + "tokens_seen": 228458496 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004700802407221665, + "loss": 1.6523, + "theoretical_loss": 4.312458904163645, + "tokens_seen": 228524032 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004700702106318957, + "loss": 1.6394, + "theoretical_loss": 4.312308364770448, + "tokens_seen": 228589568 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047006018054162484, + "loss": 1.6659, + "theoretical_loss": 4.312157880610931, + "tokens_seen": 228655104 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004700501504513541, + "loss": 1.5922, + "theoretical_loss": 4.312007451649006, + "tokens_seen": 228720640 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047004012036108326, + "loss": 1.7624, + "theoretical_loss": 4.31185707784862, + "tokens_seen": 228786176 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047003009027081244, + "loss": 1.5406, + "theoretical_loss": 4.311706759173753, + "tokens_seen": 228851712 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004700200601805416, + "loss": 1.7269, + "theoretical_loss": 4.311556495588419, + "tokens_seen": 228917248 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004700100300902708, + "loss": 1.4432, + "theoretical_loss": 4.3114062870566645, + "tokens_seen": 228982784 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047, + "loss": 1.6525, + "theoretical_loss": 4.311256133542573, + "tokens_seen": 229048320 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004699899699097292, + "loss": 1.5568, + "theoretical_loss": 4.311106035010259, + "tokens_seen": 229113856 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046997993981945835, + "loss": 1.6526, + "theoretical_loss": 4.310955991423871, + "tokens_seen": 229179392 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004699699097291876, + "loss": 1.6722, + "theoretical_loss": 4.310806002747592, + "tokens_seen": 229244928 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046995987963891676, + "loss": 1.7114, + "theoretical_loss": 4.310656068945637, + "tokens_seen": 229310464 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 82233, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6203224658966064, + "objective/train/theoretical_loss": 4.310506189982257, + "objective/train/tokens_used": 229376000, + "theoretical_loss": 4.310506189982257, + "tokens_seen": 229376000 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046994984954864595, + "loss": 1.5882, + "theoretical_loss": 4.310506189982257, + "tokens_seen": 229376000 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004699398194583752, + "loss": 1.535, + "theoretical_loss": 4.310356365821734, + "tokens_seen": 229441536 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004699297893681043, + "loss": 1.7425, + "theoretical_loss": 4.310206596428387, + "tokens_seen": 229507072 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046991975927783354, + "loss": 1.6566, + "theoretical_loss": 4.310056881766561, + "tokens_seen": 229572608 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046990972918756267, + "loss": 1.6813, + "theoretical_loss": 4.309907221800645, + "tokens_seen": 229638144 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698996990972919, + "loss": 1.7211, + "theoretical_loss": 4.309757616495053, + "tokens_seen": 229703680 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698896690070211, + "loss": 1.6048, + "theoretical_loss": 4.309608065814234, + "tokens_seen": 229769216 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046987963891675027, + "loss": 1.6914, + "theoretical_loss": 4.309458569722673, + "tokens_seen": 229834752 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046986960882647945, + "loss": 1.6351, + "theoretical_loss": 4.309309128184886, + "tokens_seen": 229900288 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698595787362087, + "loss": 1.6682, + "theoretical_loss": 4.309159741165422, + "tokens_seen": 229965824 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698495486459378, + "loss": 1.4752, + "theoretical_loss": 4.309010408628865, + "tokens_seen": 230031360 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046983951855566705, + "loss": 1.7503, + "theoretical_loss": 4.308861130539829, + "tokens_seen": 230096896 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698294884653962, + "loss": 1.7162, + "theoretical_loss": 4.308711906862965, + "tokens_seen": 230162432 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698194583751254, + "loss": 1.6479, + "theoretical_loss": 4.308562737562953, + "tokens_seen": 230227968 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698094282848546, + "loss": 1.5634, + "theoretical_loss": 4.30841362260451, + "tokens_seen": 230293504 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046979939819458377, + "loss": 1.571, + "theoretical_loss": 4.308264561952381, + "tokens_seen": 230359040 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046978936810431295, + "loss": 1.7595, + "theoretical_loss": 4.30811555557135, + "tokens_seen": 230424576 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046977933801404213, + "loss": 1.6559, + "theoretical_loss": 4.307966603426227, + "tokens_seen": 230490112 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004697693079237713, + "loss": 1.709, + "theoretical_loss": 4.307817705481861, + "tokens_seen": 230555648 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046975927783350055, + "loss": 1.713, + "theoretical_loss": 4.307668861703131, + "tokens_seen": 230621184 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004697492477432297, + "loss": 1.6568, + "theoretical_loss": 4.307520072054947, + "tokens_seen": 230686720 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004697392176529589, + "loss": 1.718, + "theoretical_loss": 4.307371336502254, + "tokens_seen": 230752256 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046972918756268804, + "loss": 1.8123, + "theoretical_loss": 4.307222655010031, + "tokens_seen": 230817792 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004697191574724173, + "loss": 1.6552, + "theoretical_loss": 4.307074027543287, + "tokens_seen": 230883328 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046970912738214646, + "loss": 1.6397, + "theoretical_loss": 4.3069254540670645, + "tokens_seen": 230948864 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046969909729187564, + "loss": 1.6334, + "theoretical_loss": 4.306776934546436, + "tokens_seen": 231014400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004696890672016048, + "loss": 1.6815, + "theoretical_loss": 4.306628468946512, + "tokens_seen": 231079936 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046967903711133405, + "loss": 1.7504, + "theoretical_loss": 4.306480057232431, + "tokens_seen": 231145472 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004696690070210632, + "loss": 1.6507, + "theoretical_loss": 4.306331699369366, + "tokens_seen": 231211008 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004696589769307924, + "loss": 1.6597, + "theoretical_loss": 4.30618339532252, + "tokens_seen": 231276544 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046964894684052154, + "loss": 1.6114, + "theoretical_loss": 4.306035145057132, + "tokens_seen": 231342080 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004696389167502508, + "loss": 1.6222, + "theoretical_loss": 4.30588694853847, + "tokens_seen": 231407616 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046962888665997996, + "loss": 1.5838, + "theoretical_loss": 4.305738805731836, + "tokens_seen": 231473152 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046961885656970914, + "loss": 1.6619, + "theoretical_loss": 4.305590716602563, + "tokens_seen": 231538688 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004696088264794383, + "loss": 1.6522, + "theoretical_loss": 4.305442681116018, + "tokens_seen": 231604224 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004695987963891675, + "loss": 1.5406, + "theoretical_loss": 4.305294699237598, + "tokens_seen": 231669760 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004695887662988967, + "loss": 1.6923, + "theoretical_loss": 4.305146770932734, + "tokens_seen": 231735296 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004695787362086259, + "loss": 1.5915, + "theoretical_loss": 4.304998896166888, + "tokens_seen": 231800832 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046956870611835505, + "loss": 1.5754, + "theoretical_loss": 4.304851074905554, + "tokens_seen": 231866368 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004695586760280843, + "loss": 1.757, + "theoretical_loss": 4.304703307114258, + "tokens_seen": 231931904 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046954864593781346, + "loss": 1.7393, + "theoretical_loss": 4.304555592758559, + "tokens_seen": 231997440 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046953861584754264, + "loss": 1.6788, + "theoretical_loss": 4.304407931804047, + "tokens_seen": 232062976 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004695285857572718, + "loss": 1.7941, + "theoretical_loss": 4.304260324216344, + "tokens_seen": 232128512 + }, + { + "epoch": 0.07, + "learning_rate": 0.000469518555667001, + "loss": 1.6419, + "theoretical_loss": 4.304112769961102, + "tokens_seen": 232194048 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004695085255767302, + "loss": 1.5944, + "theoretical_loss": 4.303965269004008, + "tokens_seen": 232259584 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004694984954864594, + "loss": 1.7746, + "theoretical_loss": 4.30381782131078, + "tokens_seen": 232325120 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046948846539618855, + "loss": 1.6845, + "theoretical_loss": 4.303670426847166, + "tokens_seen": 232390656 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004694784353059178, + "loss": 1.8298, + "theoretical_loss": 4.303523085578946, + "tokens_seen": 232456192 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004694684052156469, + "loss": 1.6892, + "theoretical_loss": 4.303375797471935, + "tokens_seen": 232521728 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046945837512537615, + "loss": 1.6622, + "theoretical_loss": 4.303228562491974, + "tokens_seen": 232587264 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 83448, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6608941555023193, + "objective/train/theoretical_loss": 4.303081380604939, + "objective/train/tokens_used": 232652800, + "theoretical_loss": 4.303081380604939, + "tokens_seen": 232652800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004694483450351053, + "loss": 1.6913, + "theoretical_loss": 4.303081380604939, + "tokens_seen": 232652800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004694383149448345, + "loss": 1.6199, + "theoretical_loss": 4.302934251776739, + "tokens_seen": 232718336 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004694282848545637, + "loss": 1.5485, + "theoretical_loss": 4.302787175973311, + "tokens_seen": 232783872 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046941825476429287, + "loss": 1.7357, + "theoretical_loss": 4.302640153160625, + "tokens_seen": 232849408 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046940822467402205, + "loss": 1.6423, + "theoretical_loss": 4.302493183304683, + "tokens_seen": 232914944 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004693981945837513, + "loss": 1.6331, + "theoretical_loss": 4.302346266371517, + "tokens_seen": 232980480 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004693881644934804, + "loss": 1.6039, + "theoretical_loss": 4.302199402327191, + "tokens_seen": 233046016 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046937813440320965, + "loss": 1.7319, + "theoretical_loss": 4.302052591137802, + "tokens_seen": 233111552 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046936810431293883, + "loss": 1.5419, + "theoretical_loss": 4.301905832769474, + "tokens_seen": 233177088 + }, + { + "epoch": 0.07, + "learning_rate": 0.000469358074222668, + "loss": 1.6534, + "theoretical_loss": 4.301759127188367, + "tokens_seen": 233242624 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004693480441323972, + "loss": 1.7644, + "theoretical_loss": 4.301612474360669, + "tokens_seen": 233308160 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004693380140421264, + "loss": 1.663, + "theoretical_loss": 4.301465874252601, + "tokens_seen": 233373696 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046932798395185555, + "loss": 1.7376, + "theoretical_loss": 4.301319326830413, + "tokens_seen": 233439232 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004693179538615848, + "loss": 1.4432, + "theoretical_loss": 4.301172832060389, + "tokens_seen": 233504768 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004693079237713139, + "loss": 1.6665, + "theoretical_loss": 4.3010263899088415, + "tokens_seen": 233570304 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046929789368104315, + "loss": 1.7423, + "theoretical_loss": 4.300880000342114, + "tokens_seen": 233635840 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004692878635907723, + "loss": 1.7007, + "theoretical_loss": 4.300733663326583, + "tokens_seen": 233701376 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004692778335005015, + "loss": 1.5417, + "theoretical_loss": 4.300587378828655, + "tokens_seen": 233766912 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004692678034102307, + "loss": 1.7276, + "theoretical_loss": 4.300441146814766, + "tokens_seen": 233832448 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004692577733199599, + "loss": 1.5629, + "theoretical_loss": 4.300294967251386, + "tokens_seen": 233897984 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046924774322968906, + "loss": 1.7467, + "theoretical_loss": 4.300148840105011, + "tokens_seen": 233963520 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046923771313941824, + "loss": 1.7391, + "theoretical_loss": 4.300002765342173, + "tokens_seen": 234029056 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004692276830491474, + "loss": 1.67, + "theoretical_loss": 4.299856742929431, + "tokens_seen": 234094592 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046921765295887666, + "loss": 1.6904, + "theoretical_loss": 4.299710772833377, + "tokens_seen": 234160128 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046920762286860584, + "loss": 1.6584, + "theoretical_loss": 4.299564855020631, + "tokens_seen": 234225664 + }, + { + "epoch": 0.07, + "learning_rate": 0.000469197592778335, + "loss": 1.6575, + "theoretical_loss": 4.299418989457848, + "tokens_seen": 234291200 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046918756268806425, + "loss": 1.6063, + "theoretical_loss": 4.299273176111709, + "tokens_seen": 234356736 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004691775325977934, + "loss": 1.5833, + "theoretical_loss": 4.299127414948927, + "tokens_seen": 234422272 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004691675025075226, + "loss": 1.7663, + "theoretical_loss": 4.298981705936248, + "tokens_seen": 234487808 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046915747241725174, + "loss": 1.5635, + "theoretical_loss": 4.298836049040444, + "tokens_seen": 234553344 + }, + { + "epoch": 0.07, + "learning_rate": 0.000469147442326981, + "loss": 1.6299, + "theoretical_loss": 4.298690444228322, + "tokens_seen": 234618880 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046913741223671016, + "loss": 1.6275, + "theoretical_loss": 4.298544891466717, + "tokens_seen": 234684416 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046912738214643934, + "loss": 1.5659, + "theoretical_loss": 4.298399390722493, + "tokens_seen": 234749952 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004691173520561685, + "loss": 1.6906, + "theoretical_loss": 4.298253941962548, + "tokens_seen": 234815488 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004691073219658977, + "loss": 1.6898, + "theoretical_loss": 4.2981085451538075, + "tokens_seen": 234881024 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004690972918756269, + "loss": 1.6481, + "theoretical_loss": 4.297963200263228, + "tokens_seen": 234946560 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004690872617853561, + "loss": 1.5906, + "theoretical_loss": 4.297817907257797, + "tokens_seen": 235012096 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046907723169508525, + "loss": 1.5946, + "theoretical_loss": 4.29767266610453, + "tokens_seen": 235077632 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004690672016048145, + "loss": 1.6788, + "theoretical_loss": 4.297527476770476, + "tokens_seen": 235143168 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046905717151454366, + "loss": 1.6652, + "theoretical_loss": 4.297382339222711, + "tokens_seen": 235208704 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046904714142427284, + "loss": 1.7384, + "theoretical_loss": 4.297237253428342, + "tokens_seen": 235274240 + }, + { + "epoch": 0.07, + "learning_rate": 0.000469037111334002, + "loss": 1.6552, + "theoretical_loss": 4.297092219354509, + "tokens_seen": 235339776 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004690270812437312, + "loss": 1.5475, + "theoretical_loss": 4.296947236968376, + "tokens_seen": 235405312 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004690170511534604, + "loss": 1.7266, + "theoretical_loss": 4.296802306237143, + "tokens_seen": 235470848 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004690070210631896, + "loss": 1.7223, + "theoretical_loss": 4.296657427128036, + "tokens_seen": 235536384 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046899699097291875, + "loss": 1.6685, + "theoretical_loss": 4.296512599608311, + "tokens_seen": 235601920 + }, + { + "epoch": 0.07, + "learning_rate": 0.000468986960882648, + "loss": 1.7221, + "theoretical_loss": 4.296367823645259, + "tokens_seen": 235667456 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004689769307923771, + "loss": 1.6821, + "theoretical_loss": 4.296223099206193, + "tokens_seen": 235732992 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046896690070210635, + "loss": 1.6372, + "theoretical_loss": 4.296078426258462, + "tokens_seen": 235798528 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046895687061183553, + "loss": 1.625, + "theoretical_loss": 4.295933804769442, + "tokens_seen": 235864064 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 84344, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.699980616569519, + "objective/train/theoretical_loss": 4.295789234706538, + "objective/train/tokens_used": 235929600, + "theoretical_loss": 4.295789234706538, + "tokens_seen": 235929600 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004689468405215647, + "loss": 1.7665, + "theoretical_loss": 4.295789234706538, + "tokens_seen": 235929600 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004689368104312939, + "loss": 1.581, + "theoretical_loss": 4.295644716037188, + "tokens_seen": 235995136 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046892678034102307, + "loss": 1.7402, + "theoretical_loss": 4.295500248728856, + "tokens_seen": 236060672 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046891675025075225, + "loss": 1.6283, + "theoretical_loss": 4.295355832749038, + "tokens_seen": 236126208 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004689067201604815, + "loss": 1.6022, + "theoretical_loss": 4.295211468065258, + "tokens_seen": 236191744 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004688966900702106, + "loss": 1.6847, + "theoretical_loss": 4.295067154645071, + "tokens_seen": 236257280 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046888665997993985, + "loss": 1.6854, + "theoretical_loss": 4.294922892456061, + "tokens_seen": 236322816 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046887662988966903, + "loss": 1.5987, + "theoretical_loss": 4.294778681465841, + "tokens_seen": 236388352 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004688665997993982, + "loss": 1.5439, + "theoretical_loss": 4.294634521642053, + "tokens_seen": 236453888 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004688565697091274, + "loss": 1.7106, + "theoretical_loss": 4.294490412952371, + "tokens_seen": 236519424 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004688465396188566, + "loss": 1.6759, + "theoretical_loss": 4.294346355364494, + "tokens_seen": 236584960 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046883650952858575, + "loss": 1.6277, + "theoretical_loss": 4.294202348846156, + "tokens_seen": 236650496 + }, + { + "epoch": 0.07, + "learning_rate": 0.000468826479438315, + "loss": 1.6895, + "theoretical_loss": 4.294058393365114, + "tokens_seen": 236716032 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004688164493480441, + "loss": 1.7024, + "theoretical_loss": 4.293914488889161, + "tokens_seen": 236781568 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046880641925777335, + "loss": 1.6975, + "theoretical_loss": 4.293770635386112, + "tokens_seen": 236847104 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004687963891675025, + "loss": 1.648, + "theoretical_loss": 4.293626832823818, + "tokens_seen": 236912640 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004687863590772317, + "loss": 1.6535, + "theoretical_loss": 4.293483081170153, + "tokens_seen": 236978176 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004687763289869609, + "loss": 1.5852, + "theoretical_loss": 4.293339380393027, + "tokens_seen": 237043712 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004687662988966901, + "loss": 1.5586, + "theoretical_loss": 4.293195730460372, + "tokens_seen": 237109248 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046875626880641926, + "loss": 1.6472, + "theoretical_loss": 4.293052131340154, + "tokens_seen": 237174784 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046874623871614844, + "loss": 1.6611, + "theoretical_loss": 4.292908583000365, + "tokens_seen": 237240320 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004687362086258776, + "loss": 1.7502, + "theoretical_loss": 4.29276508540903, + "tokens_seen": 237305856 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046872617853560686, + "loss": 1.6797, + "theoretical_loss": 4.292621638534198, + "tokens_seen": 237371392 + }, + { + "epoch": 0.07, + "learning_rate": 0.000468716148445336, + "loss": 1.5708, + "theoretical_loss": 4.29247824234395, + "tokens_seen": 237436928 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004687061183550652, + "loss": 1.6078, + "theoretical_loss": 4.292334896806397, + "tokens_seen": 237502464 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004686960882647944, + "loss": 1.679, + "theoretical_loss": 4.292191601889673, + "tokens_seen": 237568000 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004686860581745236, + "loss": 1.8274, + "theoretical_loss": 4.2920483575619475, + "tokens_seen": 237633536 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046867602808425276, + "loss": 1.6495, + "theoretical_loss": 4.291905163791417, + "tokens_seen": 237699072 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046866599799398194, + "loss": 1.7211, + "theoretical_loss": 4.291762020546304, + "tokens_seen": 237764608 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004686559679037111, + "loss": 1.7126, + "theoretical_loss": 4.291618927794862, + "tokens_seen": 237830144 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046864593781344036, + "loss": 1.5886, + "theoretical_loss": 4.291475885505374, + "tokens_seen": 237895680 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004686359077231695, + "loss": 1.718, + "theoretical_loss": 4.291332893646149, + "tokens_seen": 237961216 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004686258776328987, + "loss": 1.6042, + "theoretical_loss": 4.291189952185527, + "tokens_seen": 238026752 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046861584754262785, + "loss": 1.5453, + "theoretical_loss": 4.291047061091875, + "tokens_seen": 238092288 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004686058174523571, + "loss": 1.6464, + "theoretical_loss": 4.290904220333591, + "tokens_seen": 238157824 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046859578736208626, + "loss": 1.6792, + "theoretical_loss": 4.2907614298790975, + "tokens_seen": 238223360 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046858575727181545, + "loss": 1.5258, + "theoretical_loss": 4.290618689696848, + "tokens_seen": 238288896 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004685757271815446, + "loss": 1.5529, + "theoretical_loss": 4.290475999755326, + "tokens_seen": 238354432 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004685656970912738, + "loss": 1.6414, + "theoretical_loss": 4.29033336002304, + "tokens_seen": 238419968 + }, + { + "epoch": 0.07, + "learning_rate": 0.000468555667001003, + "loss": 1.5429, + "theoretical_loss": 4.290190770468529, + "tokens_seen": 238485504 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004685456369107322, + "loss": 1.7384, + "theoretical_loss": 4.290048231060361, + "tokens_seen": 238551040 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046853560682046135, + "loss": 1.7423, + "theoretical_loss": 4.289905741767129, + "tokens_seen": 238616576 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004685255767301906, + "loss": 1.7811, + "theoretical_loss": 4.289763302557457, + "tokens_seen": 238682112 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046851554663991977, + "loss": 1.5965, + "theoretical_loss": 4.289620913399998, + "tokens_seen": 238747648 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046850551654964895, + "loss": 1.6381, + "theoretical_loss": 4.2894785742634305, + "tokens_seen": 238813184 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046849548645937813, + "loss": 1.6086, + "theoretical_loss": 4.289336285116463, + "tokens_seen": 238878720 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004684854563691073, + "loss": 1.6726, + "theoretical_loss": 4.289194045927832, + "tokens_seen": 238944256 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004684754262788365, + "loss": 1.6441, + "theoretical_loss": 4.289051856666303, + "tokens_seen": 239009792 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046846539618856573, + "loss": 1.6408, + "theoretical_loss": 4.288909717300667, + "tokens_seen": 239075328 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004684553660982949, + "loss": 1.5141, + "theoretical_loss": 4.288767627799745, + "tokens_seen": 239140864 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 85497, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.569667100906372, + "objective/train/theoretical_loss": 4.288625588132385, + "objective/train/tokens_used": 239206400, + "theoretical_loss": 4.288625588132385, + "tokens_seen": 239206400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004684453360080241, + "loss": 1.6194, + "theoretical_loss": 4.288625588132385, + "tokens_seen": 239206400 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046843530591775327, + "loss": 1.6322, + "theoretical_loss": 4.288483598267465, + "tokens_seen": 239271936 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046842527582748245, + "loss": 1.6645, + "theoretical_loss": 4.288341658173888, + "tokens_seen": 239337472 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004684152457372117, + "loss": 1.741, + "theoretical_loss": 4.288199767820587, + "tokens_seen": 239403008 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004684052156469408, + "loss": 1.602, + "theoretical_loss": 4.2880579271765225, + "tokens_seen": 239468544 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046839518555667005, + "loss": 1.7638, + "theoretical_loss": 4.287916136210682, + "tokens_seen": 239534080 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046838515546639923, + "loss": 1.6272, + "theoretical_loss": 4.287774394892082, + "tokens_seen": 239599616 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004683751253761284, + "loss": 1.5579, + "theoretical_loss": 4.287632703189766, + "tokens_seen": 239665152 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004683650952858576, + "loss": 1.5479, + "theoretical_loss": 4.2874910610728065, + "tokens_seen": 239730688 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004683550651955868, + "loss": 1.6172, + "theoretical_loss": 4.287349468510302, + "tokens_seen": 239796224 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046834503510531595, + "loss": 1.5951, + "theoretical_loss": 4.28720792547138, + "tokens_seen": 239861760 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004683350050150452, + "loss": 1.5643, + "theoretical_loss": 4.287066431925194, + "tokens_seen": 239927296 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004683249749247743, + "loss": 1.6577, + "theoretical_loss": 4.286924987840926, + "tokens_seen": 239992832 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046831494483450355, + "loss": 1.7997, + "theoretical_loss": 4.2867835931877885, + "tokens_seen": 240058368 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004683049147442327, + "loss": 1.6525, + "theoretical_loss": 4.286642247935017, + "tokens_seen": 240123904 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682948846539619, + "loss": 1.6644, + "theoretical_loss": 4.286500952051876, + "tokens_seen": 240189440 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682848545636911, + "loss": 1.6356, + "theoretical_loss": 4.286359705507659, + "tokens_seen": 240254976 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682748244734203, + "loss": 1.6543, + "theoretical_loss": 4.286218508271686, + "tokens_seen": 240320512 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046826479438314946, + "loss": 1.677, + "theoretical_loss": 4.286077360313303, + "tokens_seen": 240386048 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046825476429287864, + "loss": 1.6995, + "theoretical_loss": 4.285936261601887, + "tokens_seen": 240451584 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682447342026078, + "loss": 1.6893, + "theoretical_loss": 4.285795212106839, + "tokens_seen": 240517120 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046823470411233706, + "loss": 1.5582, + "theoretical_loss": 4.2856542117975875, + "tokens_seen": 240582656 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682246740220662, + "loss": 1.5866, + "theoretical_loss": 4.285513260643591, + "tokens_seen": 240648192 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682146439317954, + "loss": 1.6295, + "theoretical_loss": 4.285372358614332, + "tokens_seen": 240713728 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682046138415246, + "loss": 1.6114, + "theoretical_loss": 4.285231505679324, + "tokens_seen": 240779264 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004681945837512538, + "loss": 1.5701, + "theoretical_loss": 4.285090701808103, + "tokens_seen": 240844800 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046818455366098296, + "loss": 1.5872, + "theoretical_loss": 4.284949946970236, + "tokens_seen": 240910336 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046817452357071214, + "loss": 1.669, + "theoretical_loss": 4.284809241135315, + "tokens_seen": 240975872 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004681644934804413, + "loss": 1.6107, + "theoretical_loss": 4.284668584272962, + "tokens_seen": 241041408 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046815446339017056, + "loss": 1.6882, + "theoretical_loss": 4.284527976352823, + "tokens_seen": 241106944 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004681444332998997, + "loss": 1.6319, + "theoretical_loss": 4.284387417344572, + "tokens_seen": 241172480 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004681344032096289, + "loss": 1.6881, + "theoretical_loss": 4.28424690721791, + "tokens_seen": 241238016 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046812437311935805, + "loss": 1.5942, + "theoretical_loss": 4.284106445942565, + "tokens_seen": 241303552 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004681143430290873, + "loss": 1.6503, + "theoretical_loss": 4.2839660334882925, + "tokens_seen": 241369088 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046810431293881646, + "loss": 1.679, + "theoretical_loss": 4.283825669824875, + "tokens_seen": 241434624 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046809428284854565, + "loss": 1.6229, + "theoretical_loss": 4.28368535492212, + "tokens_seen": 241500160 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004680842527582748, + "loss": 1.5372, + "theoretical_loss": 4.283545088749865, + "tokens_seen": 241565696 + }, + { + "epoch": 0.07, + "learning_rate": 0.000468074222668004, + "loss": 1.6346, + "theoretical_loss": 4.2834048712779715, + "tokens_seen": 241631232 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004680641925777332, + "loss": 1.5192, + "theoretical_loss": 4.28326470247633, + "tokens_seen": 241696768 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004680541624874624, + "loss": 1.6334, + "theoretical_loss": 4.283124582314856, + "tokens_seen": 241762304 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046804413239719155, + "loss": 1.6775, + "theoretical_loss": 4.282984510763493, + "tokens_seen": 241827840 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004680341023069208, + "loss": 1.5433, + "theoretical_loss": 4.282844487792209, + "tokens_seen": 241893376 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046802407221664997, + "loss": 1.6274, + "theoretical_loss": 4.282704513371002, + "tokens_seen": 241958912 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046801404212637915, + "loss": 1.6204, + "theoretical_loss": 4.282564587469896, + "tokens_seen": 242024448 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046800401203610833, + "loss": 1.7125, + "theoretical_loss": 4.282424710058938, + "tokens_seen": 242089984 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004679939819458375, + "loss": 1.5752, + "theoretical_loss": 4.282284881108206, + "tokens_seen": 242155520 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004679839518555667, + "loss": 1.6093, + "theoretical_loss": 4.282145100587802, + "tokens_seen": 242221056 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046797392176529593, + "loss": 1.4998, + "theoretical_loss": 4.282005368467856, + "tokens_seen": 242286592 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046796389167502505, + "loss": 1.6406, + "theoretical_loss": 4.2818656847185235, + "tokens_seen": 242352128 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004679538615847543, + "loss": 1.6062, + "theoretical_loss": 4.281726049309987, + "tokens_seen": 242417664 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 86450, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5613939762115479, + "objective/train/theoretical_loss": 4.281586462212455, + "objective/train/tokens_used": 242483200, + "theoretical_loss": 4.281586462212455, + "tokens_seen": 242483200 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004679438314944834, + "loss": 1.542, + "theoretical_loss": 4.281586462212455, + "tokens_seen": 242483200 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046793380140421265, + "loss": 1.7244, + "theoretical_loss": 4.281446923396162, + "tokens_seen": 242548736 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046792377131394183, + "loss": 1.6168, + "theoretical_loss": 4.281307432831371, + "tokens_seen": 242614272 + }, + { + "epoch": 0.07, + "learning_rate": 0.000467913741223671, + "loss": 1.6771, + "theoretical_loss": 4.281167990488369, + "tokens_seen": 242679808 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004679037111334002, + "loss": 1.6302, + "theoretical_loss": 4.281028596337469, + "tokens_seen": 242745344 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046789368104312943, + "loss": 1.6059, + "theoretical_loss": 4.280889250349014, + "tokens_seen": 242810880 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046788365095285856, + "loss": 1.6477, + "theoretical_loss": 4.280749952493368, + "tokens_seen": 242876416 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004678736208625878, + "loss": 1.6348, + "theoretical_loss": 4.280610702740926, + "tokens_seen": 242941952 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004678635907723169, + "loss": 1.5674, + "theoretical_loss": 4.280471501062106, + "tokens_seen": 243007488 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046785356068204616, + "loss": 1.7449, + "theoretical_loss": 4.280332347427353, + "tokens_seen": 243073024 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046784353059177534, + "loss": 1.699, + "theoretical_loss": 4.280193241807139, + "tokens_seen": 243138560 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004678335005015045, + "loss": 1.6097, + "theoretical_loss": 4.280054184171963, + "tokens_seen": 243204096 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004678234704112337, + "loss": 1.502, + "theoretical_loss": 4.279915174492345, + "tokens_seen": 243269632 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004678134403209629, + "loss": 1.5845, + "theoretical_loss": 4.279776212738838, + "tokens_seen": 243335168 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046780341023069206, + "loss": 1.7526, + "theoretical_loss": 4.279637298882016, + "tokens_seen": 243400704 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004677933801404213, + "loss": 1.5974, + "theoretical_loss": 4.27949843289248, + "tokens_seen": 243466240 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004677833500501504, + "loss": 1.6544, + "theoretical_loss": 4.2793596147408595, + "tokens_seen": 243531776 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046777331995987966, + "loss": 1.6347, + "theoretical_loss": 4.279220844397806, + "tokens_seen": 243597312 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004677632898696088, + "loss": 1.6285, + "theoretical_loss": 4.279082121834001, + "tokens_seen": 243662848 + }, + { + "epoch": 0.07, + "learning_rate": 0.000467753259779338, + "loss": 1.6975, + "theoretical_loss": 4.278943447020148, + "tokens_seen": 243728384 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004677432296890672, + "loss": 1.6261, + "theoretical_loss": 4.278804819926978, + "tokens_seen": 243793920 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004677331995987964, + "loss": 1.5963, + "theoretical_loss": 4.278666240525249, + "tokens_seen": 243859456 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046772316950852556, + "loss": 1.7078, + "theoretical_loss": 4.278527708785743, + "tokens_seen": 243924992 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004677131394182548, + "loss": 1.6526, + "theoretical_loss": 4.278389224679268, + "tokens_seen": 243990528 + }, + { + "epoch": 0.07, + "learning_rate": 0.000467703109327984, + "loss": 1.6208, + "theoretical_loss": 4.27825078817666, + "tokens_seen": 244056064 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046769307923771316, + "loss": 1.698, + "theoretical_loss": 4.2781123992487755, + "tokens_seen": 244121600 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046768304914744234, + "loss": 1.5633, + "theoretical_loss": 4.277974057866503, + "tokens_seen": 244187136 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004676730190571715, + "loss": 1.6827, + "theoretical_loss": 4.277835764000752, + "tokens_seen": 244252672 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046766298896690076, + "loss": 1.6631, + "theoretical_loss": 4.277697517622459, + "tokens_seen": 244318208 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004676529588766299, + "loss": 1.5434, + "theoretical_loss": 4.277559318702588, + "tokens_seen": 244383744 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004676429287863591, + "loss": 1.7252, + "theoretical_loss": 4.277421167212125, + "tokens_seen": 244449280 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046763289869608825, + "loss": 1.6623, + "theoretical_loss": 4.277283063122083, + "tokens_seen": 244514816 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004676228686058175, + "loss": 1.6772, + "theoretical_loss": 4.277145006403503, + "tokens_seen": 244580352 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046761283851554666, + "loss": 1.6365, + "theoretical_loss": 4.2770069970274465, + "tokens_seen": 244645888 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046760280842527585, + "loss": 1.605, + "theoretical_loss": 4.276869034965005, + "tokens_seen": 244711424 + }, + { + "epoch": 0.07, + "learning_rate": 0.000467592778335005, + "loss": 1.7232, + "theoretical_loss": 4.276731120187293, + "tokens_seen": 244776960 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004675827482447342, + "loss": 1.5971, + "theoretical_loss": 4.2765932526654495, + "tokens_seen": 244842496 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004675727181544634, + "loss": 1.6008, + "theoretical_loss": 4.276455432370643, + "tokens_seen": 244908032 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004675626880641926, + "loss": 1.7228, + "theoretical_loss": 4.276317659274062, + "tokens_seen": 244973568 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046755265797392175, + "loss": 1.6757, + "theoretical_loss": 4.276179933346924, + "tokens_seen": 245039104 + }, + { + "epoch": 0.07, + "learning_rate": 0.000467542627883651, + "loss": 1.5684, + "theoretical_loss": 4.27604225456047, + "tokens_seen": 245104640 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046753259779338017, + "loss": 1.5975, + "theoretical_loss": 4.275904622885967, + "tokens_seen": 245170176 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046752256770310935, + "loss": 1.6542, + "theoretical_loss": 4.275767038294708, + "tokens_seen": 245235712 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046751253761283853, + "loss": 1.5141, + "theoretical_loss": 4.275629500758008, + "tokens_seen": 245301248 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004675025075225677, + "loss": 1.6943, + "theoretical_loss": 4.27549201024721, + "tokens_seen": 245366784 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004674924774322969, + "loss": 1.6643, + "theoretical_loss": 4.275354566733682, + "tokens_seen": 245432320 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046748244734202613, + "loss": 1.689, + "theoretical_loss": 4.2752171701888155, + "tokens_seen": 245497856 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046747241725175525, + "loss": 1.636, + "theoretical_loss": 4.275079820584029, + "tokens_seen": 245563392 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004674623871614845, + "loss": 1.6687, + "theoretical_loss": 4.274942517890764, + "tokens_seen": 245628928 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004674523570712136, + "loss": 1.5541, + "theoretical_loss": 4.2748052620804895, + "tokens_seen": 245694464 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 87678, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4288511276245117, + "objective/train/theoretical_loss": 4.274668053124696, + "objective/train/tokens_used": 245760000, + "theoretical_loss": 4.274668053124696, + "tokens_seen": 245760000 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046744232698094285, + "loss": 1.612, + "theoretical_loss": 4.274668053124696, + "tokens_seen": 245760000 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046743229689067203, + "loss": 1.7091, + "theoretical_loss": 4.2745308909949005, + "tokens_seen": 245825536 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004674222668004012, + "loss": 1.8182, + "theoretical_loss": 4.274393775662648, + "tokens_seen": 245891072 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004674122367101304, + "loss": 1.7077, + "theoretical_loss": 4.274256707099502, + "tokens_seen": 245956608 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046740220661985963, + "loss": 1.5645, + "theoretical_loss": 4.274119685277058, + "tokens_seen": 246022144 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046739217652958876, + "loss": 1.6915, + "theoretical_loss": 4.273982710166931, + "tokens_seen": 246087680 + }, + { + "epoch": 0.07, + "learning_rate": 0.000467382146439318, + "loss": 1.588, + "theoretical_loss": 4.273845781740762, + "tokens_seen": 246153216 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004673721163490471, + "loss": 1.7162, + "theoretical_loss": 4.273708899970218, + "tokens_seen": 246218752 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046736208625877636, + "loss": 1.583, + "theoretical_loss": 4.273572064826991, + "tokens_seen": 246284288 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046735205616850554, + "loss": 1.6282, + "theoretical_loss": 4.273435276282795, + "tokens_seen": 246349824 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004673420260782347, + "loss": 1.6174, + "theoretical_loss": 4.2732985343093715, + "tokens_seen": 246415360 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004673319959879639, + "loss": 1.6416, + "theoretical_loss": 4.273161838878485, + "tokens_seen": 246480896 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004673219658976931, + "loss": 1.5778, + "theoretical_loss": 4.273025189961926, + "tokens_seen": 246546432 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046731193580742226, + "loss": 1.4937, + "theoretical_loss": 4.272888587531508, + "tokens_seen": 246611968 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004673019057171515, + "loss": 1.639, + "theoretical_loss": 4.272752031559071, + "tokens_seen": 246677504 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004672918756268806, + "loss": 1.6889, + "theoretical_loss": 4.272615522016476, + "tokens_seen": 246743040 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046728184553660986, + "loss": 1.6441, + "theoretical_loss": 4.272479058875614, + "tokens_seen": 246808576 + }, + { + "epoch": 0.07, + "learning_rate": 0.000467271815446339, + "loss": 1.6685, + "theoretical_loss": 4.272342642108396, + "tokens_seen": 246874112 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004672617853560682, + "loss": 1.6446, + "theoretical_loss": 4.272206271686758, + "tokens_seen": 246939648 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004672517552657974, + "loss": 1.572, + "theoretical_loss": 4.272069947582662, + "tokens_seen": 247005184 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004672417251755266, + "loss": 1.7286, + "theoretical_loss": 4.271933669768094, + "tokens_seen": 247070720 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046723169508525576, + "loss": 1.669, + "theoretical_loss": 4.271797438215064, + "tokens_seen": 247136256 + }, + { + "epoch": 0.07, + "learning_rate": 0.000467221664994985, + "loss": 1.7019, + "theoretical_loss": 4.271661252895605, + "tokens_seen": 247201792 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004672116349047141, + "loss": 1.6459, + "theoretical_loss": 4.271525113781777, + "tokens_seen": 247267328 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046720160481444336, + "loss": 1.679, + "theoretical_loss": 4.271389020845662, + "tokens_seen": 247332864 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004671915747241725, + "loss": 1.5859, + "theoretical_loss": 4.27125297405937, + "tokens_seen": 247398400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004671815446339017, + "loss": 1.7032, + "theoretical_loss": 4.271116973395028, + "tokens_seen": 247463936 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004671715145436309, + "loss": 1.6762, + "theoretical_loss": 4.270981018824795, + "tokens_seen": 247529472 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004671614844533601, + "loss": 1.591, + "theoretical_loss": 4.270845110320851, + "tokens_seen": 247595008 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046715145436308927, + "loss": 1.5698, + "theoretical_loss": 4.270709247855397, + "tokens_seen": 247660544 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046714142427281845, + "loss": 1.5961, + "theoretical_loss": 4.270573431400663, + "tokens_seen": 247726080 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046713139418254763, + "loss": 1.6795, + "theoretical_loss": 4.270437660928902, + "tokens_seen": 247791616 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046712136409227686, + "loss": 1.5702, + "theoretical_loss": 4.270301936412389, + "tokens_seen": 247857152 + }, + { + "epoch": 0.08, + "learning_rate": 0.000467111334002006, + "loss": 1.6429, + "theoretical_loss": 4.2701662578234245, + "tokens_seen": 247922688 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004671013039117352, + "loss": 1.6113, + "theoretical_loss": 4.270030625134334, + "tokens_seen": 247988224 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046709127382146435, + "loss": 1.6636, + "theoretical_loss": 4.269895038317464, + "tokens_seen": 248053760 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004670812437311936, + "loss": 1.7513, + "theoretical_loss": 4.269759497345188, + "tokens_seen": 248119296 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046707121364092277, + "loss": 1.5923, + "theoretical_loss": 4.269624002189901, + "tokens_seen": 248184832 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046706118355065195, + "loss": 1.7359, + "theoretical_loss": 4.269488552824024, + "tokens_seen": 248250368 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046705115346038113, + "loss": 1.8144, + "theoretical_loss": 4.269353149220001, + "tokens_seen": 248315904 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046704112337011037, + "loss": 1.5604, + "theoretical_loss": 4.2692177913503, + "tokens_seen": 248381440 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004670310932798395, + "loss": 1.6212, + "theoretical_loss": 4.2690824791874125, + "tokens_seen": 248446976 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046702106318956873, + "loss": 1.6048, + "theoretical_loss": 4.2689472127038535, + "tokens_seen": 248512512 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046701103309929786, + "loss": 1.6474, + "theoretical_loss": 4.2688119918721625, + "tokens_seen": 248578048 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004670010030090271, + "loss": 1.7735, + "theoretical_loss": 4.268676816664902, + "tokens_seen": 248643584 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004669909729187563, + "loss": 1.6336, + "theoretical_loss": 4.26854168705466, + "tokens_seen": 248709120 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046698094282848545, + "loss": 1.648, + "theoretical_loss": 4.2684066030140455, + "tokens_seen": 248774656 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046697091273821464, + "loss": 1.6652, + "theoretical_loss": 4.268271564515693, + "tokens_seen": 248840192 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004669608826479438, + "loss": 1.5842, + "theoretical_loss": 4.26813657153226, + "tokens_seen": 248905728 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046695085255767305, + "loss": 1.7014, + "theoretical_loss": 4.268001624036429, + "tokens_seen": 248971264 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 88571, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.563905119895935, + "objective/train/theoretical_loss": 4.267866722000902, + "objective/train/tokens_used": 249036800, + "theoretical_loss": 4.267866722000902, + "tokens_seen": 249036800 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046694082246740223, + "loss": 1.647, + "theoretical_loss": 4.267866722000902, + "tokens_seen": 249036800 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004669307923771314, + "loss": 1.5445, + "theoretical_loss": 4.26773186539841, + "tokens_seen": 249102336 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004669207622868606, + "loss": 1.5405, + "theoretical_loss": 4.267597054201705, + "tokens_seen": 249167872 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046691073219658983, + "loss": 1.717, + "theoretical_loss": 4.267462288383561, + "tokens_seen": 249233408 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046690070210631896, + "loss": 1.7213, + "theoretical_loss": 4.267327567916777, + "tokens_seen": 249298944 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004668906720160482, + "loss": 1.6949, + "theoretical_loss": 4.267192892774176, + "tokens_seen": 249364480 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004668806419257773, + "loss": 1.6688, + "theoretical_loss": 4.267058262928604, + "tokens_seen": 249430016 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046687061183550656, + "loss": 1.7047, + "theoretical_loss": 4.26692367835293, + "tokens_seen": 249495552 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046686058174523574, + "loss": 1.694, + "theoretical_loss": 4.266789139020045, + "tokens_seen": 249561088 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004668505516549649, + "loss": 1.6634, + "theoretical_loss": 4.266654644902868, + "tokens_seen": 249626624 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004668405215646941, + "loss": 1.5956, + "theoretical_loss": 4.266520195974336, + "tokens_seen": 249692160 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004668304914744233, + "loss": 1.613, + "theoretical_loss": 4.266385792207412, + "tokens_seen": 249757696 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046682046138415246, + "loss": 1.7208, + "theoretical_loss": 4.266251433575082, + "tokens_seen": 249823232 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004668104312938817, + "loss": 1.7512, + "theoretical_loss": 4.266117120050355, + "tokens_seen": 249888768 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004668004012036108, + "loss": 1.6535, + "theoretical_loss": 4.265982851606263, + "tokens_seen": 249954304 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046679037111334006, + "loss": 1.5248, + "theoretical_loss": 4.265848628215862, + "tokens_seen": 250019840 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004667803410230692, + "loss": 1.6291, + "theoretical_loss": 4.265714449852229, + "tokens_seen": 250085376 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004667703109327984, + "loss": 1.557, + "theoretical_loss": 4.265580316488467, + "tokens_seen": 250150912 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004667602808425276, + "loss": 1.6308, + "theoretical_loss": 4.2654462280977015, + "tokens_seen": 250216448 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004667502507522568, + "loss": 1.5144, + "theoretical_loss": 4.265312184653078, + "tokens_seen": 250281984 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046674022066198596, + "loss": 1.6669, + "theoretical_loss": 4.2651781861277716, + "tokens_seen": 250347520 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004667301905717152, + "loss": 1.7018, + "theoretical_loss": 4.265044232494972, + "tokens_seen": 250413056 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004667201604814443, + "loss": 1.6775, + "theoretical_loss": 4.264910323727898, + "tokens_seen": 250478592 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046671013039117356, + "loss": 1.7088, + "theoretical_loss": 4.264776459799791, + "tokens_seen": 250544128 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004667001003009027, + "loss": 1.5905, + "theoretical_loss": 4.264642640683911, + "tokens_seen": 250609664 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004666900702106319, + "loss": 1.5784, + "theoretical_loss": 4.264508866353546, + "tokens_seen": 250675200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004666800401203611, + "loss": 1.566, + "theoretical_loss": 4.264375136782004, + "tokens_seen": 250740736 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004666700100300903, + "loss": 1.6166, + "theoretical_loss": 4.264241451942617, + "tokens_seen": 250806272 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046665997993981947, + "loss": 1.6368, + "theoretical_loss": 4.264107811808739, + "tokens_seen": 250871808 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046664994984954865, + "loss": 1.5863, + "theoretical_loss": 4.263974216353749, + "tokens_seen": 250937344 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046663991975927783, + "loss": 1.6177, + "theoretical_loss": 4.2638406655510455, + "tokens_seen": 251002880 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046662988966900706, + "loss": 1.6121, + "theoretical_loss": 4.263707159374051, + "tokens_seen": 251068416 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004666198595787362, + "loss": 1.6268, + "theoretical_loss": 4.263573697796213, + "tokens_seen": 251133952 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046660982948846543, + "loss": 1.5674, + "theoretical_loss": 4.263440280790999, + "tokens_seen": 251199488 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046659979939819455, + "loss": 1.6507, + "theoretical_loss": 4.263306908331899, + "tokens_seen": 251265024 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004665897693079238, + "loss": 1.6105, + "theoretical_loss": 4.2631735803924276, + "tokens_seen": 251330560 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046657973921765297, + "loss": 1.658, + "theoretical_loss": 4.263040296946122, + "tokens_seen": 251396096 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046656970912738215, + "loss": 1.5648, + "theoretical_loss": 4.26290705796654, + "tokens_seen": 251461632 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046655967903711133, + "loss": 1.6752, + "theoretical_loss": 4.262773863427265, + "tokens_seen": 251527168 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046654964894684057, + "loss": 1.6734, + "theoretical_loss": 4.262640713301899, + "tokens_seen": 251592704 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004665396188565697, + "loss": 1.8053, + "theoretical_loss": 4.26250760756407, + "tokens_seen": 251658240 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046652958876629893, + "loss": 1.553, + "theoretical_loss": 4.262374546187428, + "tokens_seen": 251723776 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046651955867602806, + "loss": 1.6336, + "theoretical_loss": 4.262241529145643, + "tokens_seen": 251789312 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004665095285857573, + "loss": 1.6393, + "theoretical_loss": 4.262108556412411, + "tokens_seen": 251854848 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004664994984954865, + "loss": 1.7332, + "theoretical_loss": 4.261975627961448, + "tokens_seen": 251920384 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046648946840521565, + "loss": 1.553, + "theoretical_loss": 4.2618427437664925, + "tokens_seen": 251985920 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046647943831494484, + "loss": 1.542, + "theoretical_loss": 4.261709903801307, + "tokens_seen": 252051456 + }, + { + "epoch": 0.08, + "learning_rate": 0.000466469408224674, + "loss": 1.5811, + "theoretical_loss": 4.261577108039674, + "tokens_seen": 252116992 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004664593781344032, + "loss": 1.629, + "theoretical_loss": 4.261444356455402, + "tokens_seen": 252182528 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046644934804413243, + "loss": 1.7648, + "theoretical_loss": 4.261311649022318, + "tokens_seen": 252248064 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 89851, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.8168458938598633, + "objective/train/theoretical_loss": 4.261178985714273, + "objective/train/tokens_used": 252313600, + "theoretical_loss": 4.261178985714273, + "tokens_seen": 252313600 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046643931795386156, + "loss": 1.7208, + "theoretical_loss": 4.261178985714273, + "tokens_seen": 252313600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004664292878635908, + "loss": 1.5687, + "theoretical_loss": 4.26104636650514, + "tokens_seen": 252379136 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004664192577733199, + "loss": 1.5287, + "theoretical_loss": 4.260913791368815, + "tokens_seen": 252444672 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046640922768304916, + "loss": 1.7731, + "theoretical_loss": 4.260781260279216, + "tokens_seen": 252510208 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046639919759277834, + "loss": 1.7087, + "theoretical_loss": 4.260648773210281, + "tokens_seen": 252575744 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004663891675025075, + "loss": 1.5346, + "theoretical_loss": 4.2605163301359745, + "tokens_seen": 252641280 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004663791374122367, + "loss": 1.6912, + "theoretical_loss": 4.260383931030279, + "tokens_seen": 252706816 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046636910732196594, + "loss": 1.7034, + "theoretical_loss": 4.2602515758672, + "tokens_seen": 252772352 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046635907723169506, + "loss": 1.6574, + "theoretical_loss": 4.260119264620768, + "tokens_seen": 252837888 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004663490471414243, + "loss": 1.6626, + "theoretical_loss": 4.259986997265033, + "tokens_seen": 252903424 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004663390170511534, + "loss": 1.6603, + "theoretical_loss": 4.259854773774066, + "tokens_seen": 252968960 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046632898696088266, + "loss": 1.6013, + "theoretical_loss": 4.259722594121963, + "tokens_seen": 253034496 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046631895687061184, + "loss": 1.6849, + "theoretical_loss": 4.259590458282839, + "tokens_seen": 253100032 + }, + { + "epoch": 0.08, + "learning_rate": 0.000466308926780341, + "loss": 1.635, + "theoretical_loss": 4.259458366230835, + "tokens_seen": 253165568 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004662988966900702, + "loss": 1.6575, + "theoretical_loss": 4.2593263179401095, + "tokens_seen": 253231104 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004662888665997994, + "loss": 1.5773, + "theoretical_loss": 4.259194313384846, + "tokens_seen": 253296640 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046627883650952857, + "loss": 1.5534, + "theoretical_loss": 4.259062352539247, + "tokens_seen": 253362176 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004662688064192578, + "loss": 1.6416, + "theoretical_loss": 4.258930435377541, + "tokens_seen": 253427712 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046625877632898693, + "loss": 1.4958, + "theoretical_loss": 4.258798561873974, + "tokens_seen": 253493248 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046624874623871616, + "loss": 1.5656, + "theoretical_loss": 4.258666732002817, + "tokens_seen": 253558784 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046623871614844535, + "loss": 1.7065, + "theoretical_loss": 4.258534945738362, + "tokens_seen": 253624320 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004662286860581745, + "loss": 1.6302, + "theoretical_loss": 4.258403203054923, + "tokens_seen": 253689856 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004662186559679037, + "loss": 1.56, + "theoretical_loss": 4.258271503926833, + "tokens_seen": 253755392 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004662086258776329, + "loss": 1.6517, + "theoretical_loss": 4.258139848328451, + "tokens_seen": 253820928 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004661985957873621, + "loss": 1.4246, + "theoretical_loss": 4.258008236234154, + "tokens_seen": 253886464 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004661885656970913, + "loss": 1.6164, + "theoretical_loss": 4.257876667618344, + "tokens_seen": 253952000 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004661785356068205, + "loss": 1.5958, + "theoretical_loss": 4.257745142455442, + "tokens_seen": 254017536 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046616850551654967, + "loss": 1.6427, + "theoretical_loss": 4.257613660719892, + "tokens_seen": 254083072 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046615847542627885, + "loss": 1.5977, + "theoretical_loss": 4.25748222238616, + "tokens_seen": 254148608 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046614844533600803, + "loss": 1.6377, + "theoretical_loss": 4.2573508274287315, + "tokens_seen": 254214144 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046613841524573727, + "loss": 1.7826, + "theoretical_loss": 4.257219475822116, + "tokens_seen": 254279680 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004661283851554664, + "loss": 1.645, + "theoretical_loss": 4.257088167540843, + "tokens_seen": 254345216 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046611835506519563, + "loss": 1.6635, + "theoretical_loss": 4.256956902559465, + "tokens_seen": 254410752 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046610832497492475, + "loss": 1.5823, + "theoretical_loss": 4.256825680852554, + "tokens_seen": 254476288 + }, + { + "epoch": 0.08, + "learning_rate": 0.000466098294884654, + "loss": 1.4914, + "theoretical_loss": 4.256694502394704, + "tokens_seen": 254541824 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046608826479438317, + "loss": 1.5829, + "theoretical_loss": 4.256563367160533, + "tokens_seen": 254607360 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046607823470411235, + "loss": 1.5817, + "theoretical_loss": 4.256432275124676, + "tokens_seen": 254672896 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046606820461384153, + "loss": 1.6677, + "theoretical_loss": 4.256301226261794, + "tokens_seen": 254738432 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046605817452357077, + "loss": 1.709, + "theoretical_loss": 4.256170220546565, + "tokens_seen": 254803968 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004660481444332999, + "loss": 1.6199, + "theoretical_loss": 4.2560392579536925, + "tokens_seen": 254869504 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046603811434302913, + "loss": 1.5098, + "theoretical_loss": 4.255908338457898, + "tokens_seen": 254935040 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046602808425275826, + "loss": 1.5339, + "theoretical_loss": 4.255777462033926, + "tokens_seen": 255000576 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004660180541624875, + "loss": 1.676, + "theoretical_loss": 4.255646628656542, + "tokens_seen": 255066112 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004660080240722167, + "loss": 1.5911, + "theoretical_loss": 4.255515838300534, + "tokens_seen": 255131648 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046599799398194586, + "loss": 1.4846, + "theoretical_loss": 4.255385090940708, + "tokens_seen": 255197184 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046598796389167504, + "loss": 1.6087, + "theoretical_loss": 4.255254386551894, + "tokens_seen": 255262720 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004659779338014042, + "loss": 1.5941, + "theoretical_loss": 4.255123725108943, + "tokens_seen": 255328256 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004659679037111334, + "loss": 1.5404, + "theoretical_loss": 4.254993106586725, + "tokens_seen": 255393792 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046595787362086263, + "loss": 1.7079, + "theoretical_loss": 4.254862530960134, + "tokens_seen": 255459328 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046594784353059176, + "loss": 1.6902, + "theoretical_loss": 4.254731998204083, + "tokens_seen": 255524864 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 91036, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6914513111114502, + "objective/train/theoretical_loss": 4.254601508293508, + "objective/train/tokens_used": 255590400, + "theoretical_loss": 4.254601508293508, + "tokens_seen": 255590400 + }, + { + "epoch": 0.08, + "learning_rate": 0.000465937813440321, + "loss": 1.6595, + "theoretical_loss": 4.254601508293508, + "tokens_seen": 255590400 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004659277833500501, + "loss": 1.672, + "theoretical_loss": 4.254471061203364, + "tokens_seen": 255655936 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046591775325977936, + "loss": 1.6537, + "theoretical_loss": 4.254340656908628, + "tokens_seen": 255721472 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046590772316950854, + "loss": 1.6263, + "theoretical_loss": 4.254210295384298, + "tokens_seen": 255787008 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004658976930792377, + "loss": 1.6508, + "theoretical_loss": 4.254079976605394, + "tokens_seen": 255852544 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004658876629889669, + "loss": 1.5684, + "theoretical_loss": 4.253949700546956, + "tokens_seen": 255918080 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046587763289869614, + "loss": 1.5845, + "theoretical_loss": 4.2538194671840435, + "tokens_seen": 255983616 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046586760280842526, + "loss": 1.5644, + "theoretical_loss": 4.25368927649174, + "tokens_seen": 256049152 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004658575727181545, + "loss": 1.6043, + "theoretical_loss": 4.253559128445148, + "tokens_seen": 256114688 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004658475426278836, + "loss": 1.6058, + "theoretical_loss": 4.253429023019391, + "tokens_seen": 256180224 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046583751253761286, + "loss": 1.6137, + "theoretical_loss": 4.253298960189614, + "tokens_seen": 256245760 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046582748244734204, + "loss": 1.7489, + "theoretical_loss": 4.253168939930982, + "tokens_seen": 256311296 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004658174523570712, + "loss": 1.479, + "theoretical_loss": 4.253038962218682, + "tokens_seen": 256376832 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004658074222668004, + "loss": 1.6524, + "theoretical_loss": 4.25290902702792, + "tokens_seen": 256442368 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004657973921765296, + "loss": 1.5726, + "theoretical_loss": 4.252779134333926, + "tokens_seen": 256507904 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046578736208625877, + "loss": 1.5982, + "theoretical_loss": 4.252649284111946, + "tokens_seen": 256573440 + }, + { + "epoch": 0.08, + "learning_rate": 0.000465777331995988, + "loss": 1.701, + "theoretical_loss": 4.252519476337251, + "tokens_seen": 256638976 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046576730190571713, + "loss": 1.5078, + "theoretical_loss": 4.252389710985131, + "tokens_seen": 256704512 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046575727181544636, + "loss": 1.5708, + "theoretical_loss": 4.252259988030896, + "tokens_seen": 256770048 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046574724172517555, + "loss": 1.6994, + "theoretical_loss": 4.252130307449878, + "tokens_seen": 256835584 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004657372116349047, + "loss": 1.7283, + "theoretical_loss": 4.252000669217429, + "tokens_seen": 256901120 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004657271815446339, + "loss": 1.7279, + "theoretical_loss": 4.251871073308923, + "tokens_seen": 256966656 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004657171514543631, + "loss": 1.6408, + "theoretical_loss": 4.251741519699751, + "tokens_seen": 257032192 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046570712136409227, + "loss": 1.6027, + "theoretical_loss": 4.251612008365328, + "tokens_seen": 257097728 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004656970912738215, + "loss": 1.7105, + "theoretical_loss": 4.2514825392810875, + "tokens_seen": 257163264 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046568706118355063, + "loss": 1.5488, + "theoretical_loss": 4.251353112422486, + "tokens_seen": 257228800 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046567703109327987, + "loss": 1.7074, + "theoretical_loss": 4.251223727764999, + "tokens_seen": 257294336 + }, + { + "epoch": 0.08, + "learning_rate": 0.000465667001003009, + "loss": 1.5545, + "theoretical_loss": 4.251094385284121, + "tokens_seen": 257359872 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046565697091273823, + "loss": 1.6369, + "theoretical_loss": 4.25096508495537, + "tokens_seen": 257425408 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004656469408224674, + "loss": 1.5991, + "theoretical_loss": 4.250835826754281, + "tokens_seen": 257490944 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004656369107321966, + "loss": 1.6493, + "theoretical_loss": 4.2507066106564135, + "tokens_seen": 257556480 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004656268806419258, + "loss": 1.6636, + "theoretical_loss": 4.250577436637344, + "tokens_seen": 257622016 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046561685055165495, + "loss": 1.589, + "theoretical_loss": 4.25044830467267, + "tokens_seen": 257687552 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046560682046138414, + "loss": 1.6297, + "theoretical_loss": 4.250319214738011, + "tokens_seen": 257753088 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046559679037111337, + "loss": 1.5587, + "theoretical_loss": 4.250190166809005, + "tokens_seen": 257818624 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004655867602808425, + "loss": 1.6044, + "theoretical_loss": 4.250061160861311, + "tokens_seen": 257884160 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046557673019057173, + "loss": 1.7002, + "theoretical_loss": 4.249932196870608, + "tokens_seen": 257949696 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004655667001003009, + "loss": 1.4958, + "theoretical_loss": 4.249803274812598, + "tokens_seen": 258015232 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004655566700100301, + "loss": 1.5588, + "theoretical_loss": 4.249674394662998, + "tokens_seen": 258080768 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004655466399197593, + "loss": 1.5301, + "theoretical_loss": 4.249545556397549, + "tokens_seen": 258146304 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046553660982948846, + "loss": 1.5687, + "theoretical_loss": 4.249416759992011, + "tokens_seen": 258211840 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046552657973921764, + "loss": 1.6515, + "theoretical_loss": 4.249288005422166, + "tokens_seen": 258277376 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004655165496489469, + "loss": 1.5408, + "theoretical_loss": 4.249159292663812, + "tokens_seen": 258342912 + }, + { + "epoch": 0.08, + "learning_rate": 0.000465506519558676, + "loss": 1.608, + "theoretical_loss": 4.249030621692772, + "tokens_seen": 258408448 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046549648946840524, + "loss": 1.6242, + "theoretical_loss": 4.248901992484885, + "tokens_seen": 258473984 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046548645937813436, + "loss": 1.616, + "theoretical_loss": 4.248773405016014, + "tokens_seen": 258539520 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004654764292878636, + "loss": 1.6199, + "theoretical_loss": 4.248644859262039, + "tokens_seen": 258605056 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004654663991975928, + "loss": 1.5444, + "theoretical_loss": 4.248516355198861, + "tokens_seen": 258670592 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046545636910732196, + "loss": 1.6937, + "theoretical_loss": 4.2483878928024, + "tokens_seen": 258736128 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004654463390170512, + "loss": 1.5393, + "theoretical_loss": 4.248259472048598, + "tokens_seen": 258801664 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 91949, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5083632469177246, + "objective/train/theoretical_loss": 4.248131092913416, + "objective/train/tokens_used": 258867200, + "theoretical_loss": 4.248131092913416, + "tokens_seen": 258867200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004654363089267803, + "loss": 1.545, + "theoretical_loss": 4.248131092913416, + "tokens_seen": 258867200 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046542627883650956, + "loss": 1.5495, + "theoretical_loss": 4.248002755372836, + "tokens_seen": 258932736 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046541624874623874, + "loss": 1.5893, + "theoretical_loss": 4.247874459402857, + "tokens_seen": 258998272 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004654062186559679, + "loss": 1.478, + "theoretical_loss": 4.2477462049795, + "tokens_seen": 259063808 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004653961885656971, + "loss": 1.753, + "theoretical_loss": 4.2476179920788075, + "tokens_seen": 259129344 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046538615847542634, + "loss": 1.5947, + "theoretical_loss": 4.247489820676839, + "tokens_seen": 259194880 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046537612838515546, + "loss": 1.6061, + "theoretical_loss": 4.247361690749676, + "tokens_seen": 259260416 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004653660982948847, + "loss": 1.6323, + "theoretical_loss": 4.2472336022734165, + "tokens_seen": 259325952 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004653560682046138, + "loss": 1.5686, + "theoretical_loss": 4.2471055552241825, + "tokens_seen": 259391488 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046534603811434306, + "loss": 1.6379, + "theoretical_loss": 4.246977549578113, + "tokens_seen": 259457024 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046533600802407224, + "loss": 1.6022, + "theoretical_loss": 4.246849585311368, + "tokens_seen": 259522560 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004653259779338014, + "loss": 1.6099, + "theoretical_loss": 4.246721662400128, + "tokens_seen": 259588096 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004653159478435306, + "loss": 1.6079, + "theoretical_loss": 4.246593780820589, + "tokens_seen": 259653632 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004653059177532598, + "loss": 1.6119, + "theoretical_loss": 4.2464659405489735, + "tokens_seen": 259719168 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046529588766298897, + "loss": 1.7519, + "theoretical_loss": 4.246338141561518, + "tokens_seen": 259784704 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004652858575727182, + "loss": 1.655, + "theoretical_loss": 4.24621038383448, + "tokens_seen": 259850240 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046527582748244733, + "loss": 1.681, + "theoretical_loss": 4.246082667344139, + "tokens_seen": 259915776 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046526579739217656, + "loss": 1.6527, + "theoretical_loss": 4.245954992066792, + "tokens_seen": 259981312 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046525576730190575, + "loss": 1.651, + "theoretical_loss": 4.245827357978754, + "tokens_seen": 260046848 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004652457372116349, + "loss": 1.6981, + "theoretical_loss": 4.245699765056365, + "tokens_seen": 260112384 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004652357071213641, + "loss": 1.6741, + "theoretical_loss": 4.245572213275978, + "tokens_seen": 260177920 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004652256770310933, + "loss": 1.6647, + "theoretical_loss": 4.245444702613971, + "tokens_seen": 260243456 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046521564694082247, + "loss": 1.6058, + "theoretical_loss": 4.245317233046738, + "tokens_seen": 260308992 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004652056168505517, + "loss": 1.5595, + "theoretical_loss": 4.2451898045506935, + "tokens_seen": 260374528 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046519558676028083, + "loss": 1.6067, + "theoretical_loss": 4.245062417102272, + "tokens_seen": 260440064 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046518555667001007, + "loss": 1.7596, + "theoretical_loss": 4.244935070677927, + "tokens_seen": 260505600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004651755265797392, + "loss": 1.5821, + "theoretical_loss": 4.244807765254132, + "tokens_seen": 260571136 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046516549648946843, + "loss": 1.5512, + "theoretical_loss": 4.244680500807378, + "tokens_seen": 260636672 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004651554663991976, + "loss": 1.5675, + "theoretical_loss": 4.244553277314178, + "tokens_seen": 260702208 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004651454363089268, + "loss": 1.6992, + "theoretical_loss": 4.244426094751063, + "tokens_seen": 260767744 + }, + { + "epoch": 0.08, + "learning_rate": 0.000465135406218656, + "loss": 1.6333, + "theoretical_loss": 4.244298953094583, + "tokens_seen": 260833280 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046512537612838515, + "loss": 1.5968, + "theoretical_loss": 4.244171852321308, + "tokens_seen": 260898816 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046511534603811434, + "loss": 1.562, + "theoretical_loss": 4.244044792407828, + "tokens_seen": 260964352 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046510531594784357, + "loss": 1.5413, + "theoretical_loss": 4.24391777333075, + "tokens_seen": 261029888 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004650952858575727, + "loss": 1.6202, + "theoretical_loss": 4.243790795066703, + "tokens_seen": 261095424 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046508525576730193, + "loss": 1.5675, + "theoretical_loss": 4.243663857592332, + "tokens_seen": 261160960 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004650752256770311, + "loss": 1.6729, + "theoretical_loss": 4.243536960884305, + "tokens_seen": 261226496 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004650651955867603, + "loss": 1.533, + "theoretical_loss": 4.243410104919307, + "tokens_seen": 261292032 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004650551654964895, + "loss": 1.7077, + "theoretical_loss": 4.243283289674042, + "tokens_seen": 261357568 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046504513540621866, + "loss": 1.6598, + "theoretical_loss": 4.243156515125234, + "tokens_seen": 261423104 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046503510531594784, + "loss": 1.4703, + "theoretical_loss": 4.243029781249625, + "tokens_seen": 261488640 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004650250752256771, + "loss": 1.5252, + "theoretical_loss": 4.2429030880239775, + "tokens_seen": 261554176 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004650150451354062, + "loss": 1.5817, + "theoretical_loss": 4.242776435425072, + "tokens_seen": 261619712 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046500501504513544, + "loss": 1.5428, + "theoretical_loss": 4.24264982342971, + "tokens_seen": 261685248 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046499498495486456, + "loss": 1.525, + "theoretical_loss": 4.242523252014709, + "tokens_seen": 261750784 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004649849548645938, + "loss": 1.6554, + "theoretical_loss": 4.242396721156908, + "tokens_seen": 261816320 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464974924774323, + "loss": 1.5997, + "theoretical_loss": 4.242270230833164, + "tokens_seen": 261881856 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046496489468405216, + "loss": 1.6701, + "theoretical_loss": 4.242143781020353, + "tokens_seen": 261947392 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046495486459378134, + "loss": 1.5416, + "theoretical_loss": 4.24201737169537, + "tokens_seen": 262012928 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004649448345035105, + "loss": 1.5223, + "theoretical_loss": 4.24189100283513, + "tokens_seen": 262078464 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 93156, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5985779762268066, + "objective/train/theoretical_loss": 4.241764674416565, + "objective/train/tokens_used": 262144000, + "theoretical_loss": 4.241764674416565, + "tokens_seen": 262144000 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004649348044132397, + "loss": 1.6017, + "theoretical_loss": 4.241764674416565, + "tokens_seen": 262144000 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046492477432296894, + "loss": 1.7298, + "theoretical_loss": 4.241638386416628, + "tokens_seen": 262209536 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046491474423269807, + "loss": 1.5923, + "theoretical_loss": 4.241512138812288, + "tokens_seen": 262275072 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004649047141424273, + "loss": 1.6647, + "theoretical_loss": 4.2413859315805365, + "tokens_seen": 262340608 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004648946840521565, + "loss": 1.6266, + "theoretical_loss": 4.241259764698381, + "tokens_seen": 262406144 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046488465396188566, + "loss": 1.6278, + "theoretical_loss": 4.24113363814285, + "tokens_seen": 262471680 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046487462387161485, + "loss": 1.5767, + "theoretical_loss": 4.241007551890988, + "tokens_seen": 262537216 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464864593781344, + "loss": 1.5253, + "theoretical_loss": 4.2408815059198615, + "tokens_seen": 262602752 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004648545636910732, + "loss": 1.5912, + "theoretical_loss": 4.240755500206554, + "tokens_seen": 262668288 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046484453360080244, + "loss": 1.6079, + "theoretical_loss": 4.240629534728167, + "tokens_seen": 262733824 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046483450351053157, + "loss": 1.485, + "theoretical_loss": 4.240503609461822, + "tokens_seen": 262799360 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004648244734202608, + "loss": 1.5489, + "theoretical_loss": 4.24037772438466, + "tokens_seen": 262864896 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046481444332998993, + "loss": 1.665, + "theoretical_loss": 4.240251879473839, + "tokens_seen": 262930432 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046480441323971917, + "loss": 1.5011, + "theoretical_loss": 4.240126074706536, + "tokens_seen": 262995968 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046479438314944835, + "loss": 1.6576, + "theoretical_loss": 4.240000310059948, + "tokens_seen": 263061504 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046478435305917753, + "loss": 1.5072, + "theoretical_loss": 4.239874585511288, + "tokens_seen": 263127040 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004647743229689067, + "loss": 1.675, + "theoretical_loss": 4.239748901037791, + "tokens_seen": 263192576 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046476429287863595, + "loss": 1.6926, + "theoretical_loss": 4.2396232566167065, + "tokens_seen": 263258112 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004647542627883651, + "loss": 1.5671, + "theoretical_loss": 4.239497652225307, + "tokens_seen": 263323648 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004647442326980943, + "loss": 1.7781, + "theoretical_loss": 4.23937208784088, + "tokens_seen": 263389184 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046473420260782344, + "loss": 1.6672, + "theoretical_loss": 4.239246563440734, + "tokens_seen": 263454720 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046472417251755267, + "loss": 1.611, + "theoretical_loss": 4.239121079002194, + "tokens_seen": 263520256 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004647141424272819, + "loss": 1.6738, + "theoretical_loss": 4.238995634502606, + "tokens_seen": 263585792 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046470411233701103, + "loss": 1.614, + "theoretical_loss": 4.238870229919331, + "tokens_seen": 263651328 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046469408224674027, + "loss": 1.5421, + "theoretical_loss": 4.2387448652297515, + "tokens_seen": 263716864 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004646840521564694, + "loss": 1.5328, + "theoretical_loss": 4.238619540411268, + "tokens_seen": 263782400 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046467402206619863, + "loss": 1.5723, + "theoretical_loss": 4.238494255441296, + "tokens_seen": 263847936 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004646639919759278, + "loss": 1.4756, + "theoretical_loss": 4.238369010297275, + "tokens_seen": 263913472 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464653961885657, + "loss": 1.5863, + "theoretical_loss": 4.238243804956658, + "tokens_seen": 263979008 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004646439317953862, + "loss": 1.6201, + "theoretical_loss": 4.23811863939692, + "tokens_seen": 264044544 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046463390170511535, + "loss": 1.5955, + "theoretical_loss": 4.237993513595552, + "tokens_seen": 264110080 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046462387161484454, + "loss": 1.603, + "theoretical_loss": 4.2378684275300635, + "tokens_seen": 264175616 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046461384152457377, + "loss": 1.6832, + "theoretical_loss": 4.237743381177983, + "tokens_seen": 264241152 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004646038114343029, + "loss": 1.6539, + "theoretical_loss": 4.237618374516858, + "tokens_seen": 264306688 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046459378134403213, + "loss": 1.587, + "theoretical_loss": 4.237493407524253, + "tokens_seen": 264372224 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004645837512537613, + "loss": 1.5526, + "theoretical_loss": 4.23736848017775, + "tokens_seen": 264437760 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004645737211634905, + "loss": 1.667, + "theoretical_loss": 4.237243592454951, + "tokens_seen": 264503296 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004645636910732197, + "loss": 1.6807, + "theoretical_loss": 4.2371187443334755, + "tokens_seen": 264568832 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046455366098294886, + "loss": 1.6248, + "theoretical_loss": 4.236993935790962, + "tokens_seen": 264634368 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046454363089267804, + "loss": 1.6088, + "theoretical_loss": 4.236869166805064, + "tokens_seen": 264699904 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004645336008024073, + "loss": 1.6581, + "theoretical_loss": 4.236744437353458, + "tokens_seen": 264765440 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004645235707121364, + "loss": 1.5788, + "theoretical_loss": 4.236619747413836, + "tokens_seen": 264830976 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046451354062186564, + "loss": 1.6827, + "theoretical_loss": 4.236495096963906, + "tokens_seen": 264896512 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046450351053159476, + "loss": 1.4878, + "theoretical_loss": 4.236370485981398, + "tokens_seen": 264962048 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464493480441324, + "loss": 1.4903, + "theoretical_loss": 4.236245914444058, + "tokens_seen": 265027584 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004644834503510532, + "loss": 1.5727, + "theoretical_loss": 4.2361213823296495, + "tokens_seen": 265093120 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046447342026078236, + "loss": 1.4561, + "theoretical_loss": 4.235996889615957, + "tokens_seen": 265158656 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046446339017051154, + "loss": 1.5525, + "theoretical_loss": 4.235872436280779, + "tokens_seen": 265224192 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004644533600802407, + "loss": 1.5875, + "theoretical_loss": 4.235748022301935, + "tokens_seen": 265289728 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004644433299899699, + "loss": 1.5998, + "theoretical_loss": 4.235623647657261, + "tokens_seen": 265355264 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 94058, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5976884365081787, + "objective/train/theoretical_loss": 4.235499312324611, + "objective/train/tokens_used": 265420800, + "theoretical_loss": 4.235499312324611, + "tokens_seen": 265420800 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046443329989969914, + "loss": 1.5414, + "theoretical_loss": 4.235499312324611, + "tokens_seen": 265420800 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046442326980942827, + "loss": 1.6241, + "theoretical_loss": 4.235375016281858, + "tokens_seen": 265486336 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004644132397191575, + "loss": 1.5487, + "theoretical_loss": 4.235250759506892, + "tokens_seen": 265551872 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004644032096288867, + "loss": 1.6273, + "theoretical_loss": 4.235126541977621, + "tokens_seen": 265617408 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046439317953861586, + "loss": 1.6816, + "theoretical_loss": 4.2350023636719705, + "tokens_seen": 265682944 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046438314944834505, + "loss": 1.6372, + "theoretical_loss": 4.234878224567885, + "tokens_seen": 265748480 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004643731193580742, + "loss": 1.6893, + "theoretical_loss": 4.2347541246433265, + "tokens_seen": 265814016 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004643630892678034, + "loss": 1.5826, + "theoretical_loss": 4.234630063876273, + "tokens_seen": 265879552 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046435305917753264, + "loss": 1.5223, + "theoretical_loss": 4.234506042244723, + "tokens_seen": 265945088 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046434302908726177, + "loss": 1.6225, + "theoretical_loss": 4.234382059726691, + "tokens_seen": 266010624 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464332998996991, + "loss": 1.5946, + "theoretical_loss": 4.234258116300211, + "tokens_seen": 266076160 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046432296890672013, + "loss": 1.5082, + "theoretical_loss": 4.234134211943332, + "tokens_seen": 266141696 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046431293881644937, + "loss": 1.6919, + "theoretical_loss": 4.234010346634123, + "tokens_seen": 266207232 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046430290872617855, + "loss": 1.7256, + "theoretical_loss": 4.23388652035067, + "tokens_seen": 266272768 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046429287863590773, + "loss": 1.7799, + "theoretical_loss": 4.233762733071076, + "tokens_seen": 266338304 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004642828485456369, + "loss": 1.5433, + "theoretical_loss": 4.233638984773464, + "tokens_seen": 266403840 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046427281845536615, + "loss": 1.4677, + "theoretical_loss": 4.2335152754359715, + "tokens_seen": 266469376 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004642627883650953, + "loss": 1.5718, + "theoretical_loss": 4.233391605036756, + "tokens_seen": 266534912 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004642527582748245, + "loss": 1.5611, + "theoretical_loss": 4.233267973553991, + "tokens_seen": 266600448 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046424272818455364, + "loss": 1.6579, + "theoretical_loss": 4.233144380965869, + "tokens_seen": 266665984 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046423269809428287, + "loss": 1.5922, + "theoretical_loss": 4.2330208272505985, + "tokens_seen": 266731520 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046422266800401205, + "loss": 1.5657, + "theoretical_loss": 4.232897312386408, + "tokens_seen": 266797056 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046421263791374123, + "loss": 1.5515, + "theoretical_loss": 4.232773836351541, + "tokens_seen": 266862592 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004642026078234704, + "loss": 1.5735, + "theoretical_loss": 4.2326503991242586, + "tokens_seen": 266928128 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004641925777331996, + "loss": 1.6258, + "theoretical_loss": 4.232527000682843, + "tokens_seen": 266993664 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004641825476429288, + "loss": 1.5692, + "theoretical_loss": 4.232403641005589, + "tokens_seen": 267059200 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464172517552658, + "loss": 1.6257, + "theoretical_loss": 4.232280320070812, + "tokens_seen": 267124736 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046416248746238714, + "loss": 1.7284, + "theoretical_loss": 4.232157037856844, + "tokens_seen": 267190272 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004641524573721164, + "loss": 1.6436, + "theoretical_loss": 4.232033794342033, + "tokens_seen": 267255808 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004641424272818455, + "loss": 1.6627, + "theoretical_loss": 4.231910589504748, + "tokens_seen": 267321344 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046413239719157474, + "loss": 1.5349, + "theoretical_loss": 4.2317874233233725, + "tokens_seen": 267386880 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004641223671013039, + "loss": 1.605, + "theoretical_loss": 4.231664295776307, + "tokens_seen": 267452416 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004641123370110331, + "loss": 1.5992, + "theoretical_loss": 4.231541206841971, + "tokens_seen": 267517952 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004641023069207623, + "loss": 1.6434, + "theoretical_loss": 4.2314181564988, + "tokens_seen": 267583488 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004640922768304915, + "loss": 1.668, + "theoretical_loss": 4.23129514472525, + "tokens_seen": 267649024 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046408224674022064, + "loss": 1.6932, + "theoretical_loss": 4.231172171499789, + "tokens_seen": 267714560 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004640722166499499, + "loss": 1.6867, + "theoretical_loss": 4.231049236800907, + "tokens_seen": 267780096 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464062186559679, + "loss": 1.5659, + "theoretical_loss": 4.230926340607109, + "tokens_seen": 267845632 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046405215646940824, + "loss": 1.5654, + "theoretical_loss": 4.230803482896918, + "tokens_seen": 267911168 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004640421263791374, + "loss": 1.5907, + "theoretical_loss": 4.230680663648872, + "tokens_seen": 267976704 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004640320962888666, + "loss": 1.5946, + "theoretical_loss": 4.230557882841531, + "tokens_seen": 268042240 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004640220661985958, + "loss": 1.5577, + "theoretical_loss": 4.2304351404534675, + "tokens_seen": 268107776 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046401203610832496, + "loss": 1.6454, + "theoretical_loss": 4.230312436463272, + "tokens_seen": 268173312 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046400200601805414, + "loss": 1.7226, + "theoretical_loss": 4.230189770849556, + "tokens_seen": 268238848 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004639919759277834, + "loss": 1.4881, + "theoretical_loss": 4.230067143590943, + "tokens_seen": 268304384 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004639819458375125, + "loss": 1.5675, + "theoretical_loss": 4.229944554666076, + "tokens_seen": 268369920 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046397191574724174, + "loss": 1.5476, + "theoretical_loss": 4.229822004053617, + "tokens_seen": 268435456 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004639618856569709, + "loss": 1.6675, + "theoretical_loss": 4.229699491732242, + "tokens_seen": 268500992 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004639518555667001, + "loss": 1.5432, + "theoretical_loss": 4.2295770176806435, + "tokens_seen": 268566528 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046394182547642934, + "loss": 1.5473, + "theoretical_loss": 4.229454581877535, + "tokens_seen": 268632064 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 95370, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6092371940612793, + "objective/train/theoretical_loss": 4.229332184301644, + "objective/train/tokens_used": 268697600, + "theoretical_loss": 4.229332184301644, + "tokens_seen": 268697600 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046393179538615847, + "loss": 1.6801, + "theoretical_loss": 4.229332184301644, + "tokens_seen": 268697600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004639217652958877, + "loss": 1.5884, + "theoretical_loss": 4.229209824931714, + "tokens_seen": 268763136 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004639117352056169, + "loss": 1.5562, + "theoretical_loss": 4.22908750374651, + "tokens_seen": 268828672 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046390170511534606, + "loss": 1.6454, + "theoretical_loss": 4.22896522072481, + "tokens_seen": 268894208 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046389167502507525, + "loss": 1.6468, + "theoretical_loss": 4.22884297584541, + "tokens_seen": 268959744 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004638816449348044, + "loss": 1.6314, + "theoretical_loss": 4.228720769087124, + "tokens_seen": 269025280 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004638716148445336, + "loss": 1.6108, + "theoretical_loss": 4.22859860042878, + "tokens_seen": 269090816 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046386158475426284, + "loss": 1.667, + "theoretical_loss": 4.2284764698492285, + "tokens_seen": 269156352 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046385155466399197, + "loss": 1.6896, + "theoretical_loss": 4.22835437732733, + "tokens_seen": 269221888 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004638415245737212, + "loss": 1.6397, + "theoretical_loss": 4.228232322841967, + "tokens_seen": 269287424 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046383149448345033, + "loss": 1.6365, + "theoretical_loss": 4.228110306372036, + "tokens_seen": 269352960 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046382146439317957, + "loss": 1.6334, + "theoretical_loss": 4.227988327896453, + "tokens_seen": 269418496 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046381143430290875, + "loss": 1.5388, + "theoretical_loss": 4.2278663873941476, + "tokens_seen": 269484032 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046380140421263793, + "loss": 1.7087, + "theoretical_loss": 4.227744484844068, + "tokens_seen": 269549568 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004637913741223671, + "loss": 1.6254, + "theoretical_loss": 4.227622620225182, + "tokens_seen": 269615104 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046378134403209635, + "loss": 1.6379, + "theoretical_loss": 4.227500793516466, + "tokens_seen": 269680640 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004637713139418255, + "loss": 1.4566, + "theoretical_loss": 4.227379004696923, + "tokens_seen": 269746176 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004637612838515547, + "loss": 1.6134, + "theoretical_loss": 4.227257253745566, + "tokens_seen": 269811712 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046375125376128384, + "loss": 1.7325, + "theoretical_loss": 4.227135540641426, + "tokens_seen": 269877248 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046374122367101307, + "loss": 1.6657, + "theoretical_loss": 4.227013865363553, + "tokens_seen": 269942784 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046373119358074225, + "loss": 1.546, + "theoretical_loss": 4.226892227891012, + "tokens_seen": 270008320 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046372116349047143, + "loss": 1.6361, + "theoretical_loss": 4.2267706282028845, + "tokens_seen": 270073856 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004637111334002006, + "loss": 1.5426, + "theoretical_loss": 4.226649066278269, + "tokens_seen": 270139392 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004637011033099298, + "loss": 1.5312, + "theoretical_loss": 4.226527542096282, + "tokens_seen": 270204928 + }, + { + "epoch": 0.08, + "learning_rate": 0.000463691073219659, + "loss": 1.4836, + "theoretical_loss": 4.226406055636053, + "tokens_seen": 270270464 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004636810431293882, + "loss": 1.6545, + "theoretical_loss": 4.226284606876731, + "tokens_seen": 270336000 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046367101303911734, + "loss": 1.5028, + "theoretical_loss": 4.226163195797481, + "tokens_seen": 270401536 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004636609829488466, + "loss": 1.6033, + "theoretical_loss": 4.226041822377486, + "tokens_seen": 270467072 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004636509528585757, + "loss": 1.6703, + "theoretical_loss": 4.225920486595941, + "tokens_seen": 270532608 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046364092276830494, + "loss": 1.541, + "theoretical_loss": 4.225799188432063, + "tokens_seen": 270598144 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004636308926780341, + "loss": 1.5935, + "theoretical_loss": 4.225677927865083, + "tokens_seen": 270663680 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004636208625877633, + "loss": 1.7244, + "theoretical_loss": 4.225556704874247, + "tokens_seen": 270729216 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004636108324974925, + "loss": 1.6142, + "theoretical_loss": 4.22543551943882, + "tokens_seen": 270794752 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004636008024072217, + "loss": 1.6516, + "theoretical_loss": 4.225314371538083, + "tokens_seen": 270860288 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046359077231695084, + "loss": 1.6748, + "theoretical_loss": 4.225193261151332, + "tokens_seen": 270925824 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004635807422266801, + "loss": 1.3851, + "theoretical_loss": 4.22507218825788, + "tokens_seen": 270991360 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004635707121364092, + "loss": 1.5873, + "theoretical_loss": 4.224951152837058, + "tokens_seen": 271056896 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046356068204613844, + "loss": 1.5331, + "theoretical_loss": 4.2248301548682115, + "tokens_seen": 271122432 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004635506519558676, + "loss": 1.577, + "theoretical_loss": 4.2247091943307025, + "tokens_seen": 271187968 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004635406218655968, + "loss": 1.5814, + "theoretical_loss": 4.224588271203911, + "tokens_seen": 271253504 + }, + { + "epoch": 0.08, + "learning_rate": 0.000463530591775326, + "loss": 1.4946, + "theoretical_loss": 4.22446738546723, + "tokens_seen": 271319040 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046352056168505516, + "loss": 1.5406, + "theoretical_loss": 4.224346537100074, + "tokens_seen": 271384576 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046351053159478434, + "loss": 1.5716, + "theoretical_loss": 4.22422572608187, + "tokens_seen": 271450112 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004635005015045136, + "loss": 1.721, + "theoretical_loss": 4.224104952392061, + "tokens_seen": 271515648 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004634904714142427, + "loss": 1.5638, + "theoretical_loss": 4.223984216010107, + "tokens_seen": 271581184 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046348044132397194, + "loss": 1.6077, + "theoretical_loss": 4.223863516915486, + "tokens_seen": 271646720 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046347041123370107, + "loss": 1.6248, + "theoretical_loss": 4.223742855087691, + "tokens_seen": 271712256 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004634603811434303, + "loss": 1.634, + "theoretical_loss": 4.223622230506231, + "tokens_seen": 271777792 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004634503510531595, + "loss": 1.6695, + "theoretical_loss": 4.22350164315063, + "tokens_seen": 271843328 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046344032096288867, + "loss": 1.5591, + "theoretical_loss": 4.22338109300043, + "tokens_seen": 271908864 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 96286, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6222212314605713, + "objective/train/theoretical_loss": 4.22326058003519, + "objective/train/tokens_used": 271974400, + "theoretical_loss": 4.22326058003519, + "tokens_seen": 271974400 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046343029087261785, + "loss": 1.6575, + "theoretical_loss": 4.22326058003519, + "tokens_seen": 271974400 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004634202607823471, + "loss": 1.5368, + "theoretical_loss": 4.223140104234482, + "tokens_seen": 272039936 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004634102306920762, + "loss": 1.6042, + "theoretical_loss": 4.2230196655778975, + "tokens_seen": 272105472 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046340020060180545, + "loss": 1.5208, + "theoretical_loss": 4.222899264045042, + "tokens_seen": 272171008 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046339017051153457, + "loss": 1.598, + "theoretical_loss": 4.2227788996155375, + "tokens_seen": 272236544 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004633801404212638, + "loss": 1.5276, + "theoretical_loss": 4.222658572269022, + "tokens_seen": 272302080 + }, + { + "epoch": 0.08, + "learning_rate": 0.000463370110330993, + "loss": 1.6172, + "theoretical_loss": 4.222538281985151, + "tokens_seen": 272367616 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046336008024072217, + "loss": 1.5701, + "theoretical_loss": 4.2224180287435935, + "tokens_seen": 272433152 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046335005015045135, + "loss": 1.8116, + "theoretical_loss": 4.222297812524037, + "tokens_seen": 272498688 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046334002006018053, + "loss": 1.5677, + "theoretical_loss": 4.222177633306183, + "tokens_seen": 272564224 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004633299899699097, + "loss": 1.5685, + "theoretical_loss": 4.222057491069751, + "tokens_seen": 272629760 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046331995987963895, + "loss": 1.6798, + "theoretical_loss": 4.221937385794474, + "tokens_seen": 272695296 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004633099297893681, + "loss": 1.6531, + "theoretical_loss": 4.221817317460104, + "tokens_seen": 272760832 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004632998996990973, + "loss": 1.5471, + "theoretical_loss": 4.221697286046407, + "tokens_seen": 272826368 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046328986960882644, + "loss": 1.5546, + "theoretical_loss": 4.221577291533165, + "tokens_seen": 272891904 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004632798395185557, + "loss": 1.5931, + "theoretical_loss": 4.221457333900176, + "tokens_seen": 272957440 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046326980942828485, + "loss": 1.5115, + "theoretical_loss": 4.221337413127254, + "tokens_seen": 273022976 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046325977933801404, + "loss": 1.6184, + "theoretical_loss": 4.221217529194231, + "tokens_seen": 273088512 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004632497492477432, + "loss": 1.504, + "theoretical_loss": 4.22109768208095, + "tokens_seen": 273154048 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046323971915747245, + "loss": 1.5123, + "theoretical_loss": 4.220977871767275, + "tokens_seen": 273219584 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004632296890672016, + "loss": 1.4982, + "theoretical_loss": 4.220858098233082, + "tokens_seen": 273285120 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004632196589769308, + "loss": 1.5516, + "theoretical_loss": 4.220738361458266, + "tokens_seen": 273350656 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046320962888666, + "loss": 1.7078, + "theoretical_loss": 4.220618661422735, + "tokens_seen": 273416192 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631995987963892, + "loss": 1.6062, + "theoretical_loss": 4.2204989981064145, + "tokens_seen": 273481728 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631895687061184, + "loss": 1.5411, + "theoretical_loss": 4.220379371489246, + "tokens_seen": 273547264 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046317953861584754, + "loss": 1.6323, + "theoretical_loss": 4.220259781551185, + "tokens_seen": 273612800 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631695085255768, + "loss": 1.5672, + "theoretical_loss": 4.2201402282722045, + "tokens_seen": 273678336 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631594784353059, + "loss": 1.6982, + "theoretical_loss": 4.2200207116322925, + "tokens_seen": 273743872 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046314944834503514, + "loss": 1.6306, + "theoretical_loss": 4.219901231611452, + "tokens_seen": 273809408 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631394182547643, + "loss": 1.5325, + "theoretical_loss": 4.219781788189703, + "tokens_seen": 273874944 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631293881644935, + "loss": 1.6252, + "theoretical_loss": 4.2196623813470815, + "tokens_seen": 273940480 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631193580742227, + "loss": 1.4999, + "theoretical_loss": 4.219543011063637, + "tokens_seen": 274006016 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631093279839519, + "loss": 1.5827, + "theoretical_loss": 4.219423677319437, + "tokens_seen": 274071552 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046309929789368104, + "loss": 1.6188, + "theoretical_loss": 4.219304380094562, + "tokens_seen": 274137088 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004630892678034103, + "loss": 1.5474, + "theoretical_loss": 4.2191851193691114, + "tokens_seen": 274202624 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004630792377131394, + "loss": 1.6785, + "theoretical_loss": 4.219065895123197, + "tokens_seen": 274268160 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046306920762286864, + "loss": 1.6867, + "theoretical_loss": 4.218946707336949, + "tokens_seen": 274333696 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004630591775325978, + "loss": 1.5943, + "theoretical_loss": 4.2188275559905115, + "tokens_seen": 274399232 + }, + { + "epoch": 0.08, + "learning_rate": 0.000463049147442327, + "loss": 1.5315, + "theoretical_loss": 4.218708441064044, + "tokens_seen": 274464768 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004630391173520562, + "loss": 1.6187, + "theoretical_loss": 4.218589362537723, + "tokens_seen": 274530304 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046302908726178536, + "loss": 1.6021, + "theoretical_loss": 4.218470320391739, + "tokens_seen": 274595840 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046301905717151455, + "loss": 1.7499, + "theoretical_loss": 4.218351314606298, + "tokens_seen": 274661376 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004630090270812438, + "loss": 1.6049, + "theoretical_loss": 4.218232345161622, + "tokens_seen": 274726912 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004629989969909729, + "loss": 1.5793, + "theoretical_loss": 4.21811341203795, + "tokens_seen": 274792448 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046298896690070214, + "loss": 1.6332, + "theoretical_loss": 4.217994515215534, + "tokens_seen": 274857984 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046297893681043127, + "loss": 1.5841, + "theoretical_loss": 4.217875654674642, + "tokens_seen": 274923520 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004629689067201605, + "loss": 1.6071, + "theoretical_loss": 4.217756830395559, + "tokens_seen": 274989056 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004629588766298897, + "loss": 1.5641, + "theoretical_loss": 4.217638042358583, + "tokens_seen": 275054592 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046294884653961887, + "loss": 1.6144, + "theoretical_loss": 4.21751929054403, + "tokens_seen": 275120128 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046293881644934805, + "loss": 1.5566, + "theoretical_loss": 4.2174005749322285, + "tokens_seen": 275185664 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 97409, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6986513137817383, + "objective/train/theoretical_loss": 4.2172818955035245, + "objective/train/tokens_used": 275251200, + "theoretical_loss": 4.2172818955035245, + "tokens_seen": 275251200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004629287863590773, + "loss": 1.6159, + "theoretical_loss": 4.2172818955035245, + "tokens_seen": 275251200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004629187562688064, + "loss": 1.5422, + "theoretical_loss": 4.217163252238279, + "tokens_seen": 275316736 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046290872617853565, + "loss": 1.5968, + "theoretical_loss": 4.217044645116869, + "tokens_seen": 275382272 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046289869608826477, + "loss": 1.6072, + "theoretical_loss": 4.216926074119684, + "tokens_seen": 275447808 + }, + { + "epoch": 0.08, + "learning_rate": 0.000462888665997994, + "loss": 1.6223, + "theoretical_loss": 4.216807539227132, + "tokens_seen": 275513344 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004628786359077232, + "loss": 1.5193, + "theoretical_loss": 4.2166890404196335, + "tokens_seen": 275578880 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046286860581745237, + "loss": 1.616, + "theoretical_loss": 4.216570577677627, + "tokens_seen": 275644416 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046285857572718155, + "loss": 1.6968, + "theoretical_loss": 4.216452150981566, + "tokens_seen": 275709952 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046284854563691073, + "loss": 1.5773, + "theoretical_loss": 4.216333760311916, + "tokens_seen": 275775488 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004628385155466399, + "loss": 1.6909, + "theoretical_loss": 4.216215405649161, + "tokens_seen": 275841024 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046282848545636915, + "loss": 1.7341, + "theoretical_loss": 4.2160970869738, + "tokens_seen": 275906560 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004628184553660983, + "loss": 1.6621, + "theoretical_loss": 4.215978804266345, + "tokens_seen": 275972096 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004628084252758275, + "loss": 1.716, + "theoretical_loss": 4.215860557507324, + "tokens_seen": 276037632 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046279839518555664, + "loss": 1.6486, + "theoretical_loss": 4.215742346677283, + "tokens_seen": 276103168 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004627883650952859, + "loss": 1.4982, + "theoretical_loss": 4.215624171756779, + "tokens_seen": 276168704 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046277833500501505, + "loss": 1.6398, + "theoretical_loss": 4.215506032726386, + "tokens_seen": 276234240 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046276830491474424, + "loss": 1.5794, + "theoretical_loss": 4.215387929566694, + "tokens_seen": 276299776 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004627582748244734, + "loss": 1.5633, + "theoretical_loss": 4.215269862258307, + "tokens_seen": 276365312 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046274824473420265, + "loss": 1.5973, + "theoretical_loss": 4.215151830781843, + "tokens_seen": 276430848 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004627382146439318, + "loss": 1.5256, + "theoretical_loss": 4.2150338351179375, + "tokens_seen": 276496384 + }, + { + "epoch": 0.08, + "learning_rate": 0.000462728184553661, + "loss": 1.6973, + "theoretical_loss": 4.21491587524724, + "tokens_seen": 276561920 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046271815446339014, + "loss": 1.6027, + "theoretical_loss": 4.214797951150415, + "tokens_seen": 276627456 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004627081243731194, + "loss": 1.6309, + "theoretical_loss": 4.214680062808142, + "tokens_seen": 276692992 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046269809428284856, + "loss": 1.6439, + "theoretical_loss": 4.214562210201114, + "tokens_seen": 276758528 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046268806419257774, + "loss": 1.605, + "theoretical_loss": 4.214444393310042, + "tokens_seen": 276824064 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004626780341023069, + "loss": 1.5234, + "theoretical_loss": 4.2143266121156495, + "tokens_seen": 276889600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004626680040120361, + "loss": 1.668, + "theoretical_loss": 4.2142088665986766, + "tokens_seen": 276955136 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004626579739217653, + "loss": 1.7572, + "theoretical_loss": 4.214091156739878, + "tokens_seen": 277020672 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004626479438314945, + "loss": 1.7135, + "theoretical_loss": 4.213973482520021, + "tokens_seen": 277086208 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046263791374122364, + "loss": 1.5754, + "theoretical_loss": 4.213855843919891, + "tokens_seen": 277151744 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004626278836509529, + "loss": 1.6766, + "theoretical_loss": 4.2137382409202875, + "tokens_seen": 277217280 + }, + { + "epoch": 0.08, + "learning_rate": 0.000462617853560682, + "loss": 1.68, + "theoretical_loss": 4.2136206735020245, + "tokens_seen": 277282816 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046260782347041124, + "loss": 1.616, + "theoretical_loss": 4.21350314164593, + "tokens_seen": 277348352 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004625977933801404, + "loss": 1.5601, + "theoretical_loss": 4.213385645332848, + "tokens_seen": 277413888 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004625877632898696, + "loss": 1.6423, + "theoretical_loss": 4.213268184543638, + "tokens_seen": 277479424 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004625777331995988, + "loss": 1.6551, + "theoretical_loss": 4.213150759259172, + "tokens_seen": 277544960 + }, + { + "epoch": 0.08, + "learning_rate": 0.000462567703109328, + "loss": 1.6077, + "theoretical_loss": 4.213033369460339, + "tokens_seen": 277610496 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046255767301905715, + "loss": 1.5806, + "theoretical_loss": 4.212916015128041, + "tokens_seen": 277676032 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004625476429287864, + "loss": 1.6066, + "theoretical_loss": 4.212798696243197, + "tokens_seen": 277741568 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004625376128385155, + "loss": 1.5255, + "theoretical_loss": 4.212681412786738, + "tokens_seen": 277807104 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046252758274824475, + "loss": 1.6518, + "theoretical_loss": 4.212564164739613, + "tokens_seen": 277872640 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004625175526579739, + "loss": 1.6388, + "theoretical_loss": 4.212446952082783, + "tokens_seen": 277938176 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004625075225677031, + "loss": 1.6311, + "theoretical_loss": 4.212329774797225, + "tokens_seen": 278003712 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004624974924774323, + "loss": 1.7109, + "theoretical_loss": 4.212212632863931, + "tokens_seen": 278069248 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046248746238716147, + "loss": 1.6115, + "theoretical_loss": 4.212095526263905, + "tokens_seen": 278134784 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046247743229689065, + "loss": 1.5879, + "theoretical_loss": 4.211978454978171, + "tokens_seen": 278200320 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004624674022066199, + "loss": 1.5839, + "theoretical_loss": 4.211861418987762, + "tokens_seen": 278265856 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046245737211634907, + "loss": 1.5825, + "theoretical_loss": 4.211744418273728, + "tokens_seen": 278331392 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046244734202607825, + "loss": 1.62, + "theoretical_loss": 4.211627452817136, + "tokens_seen": 278396928 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004624373119358075, + "loss": 1.5885, + "theoretical_loss": 4.211510522599063, + "tokens_seen": 278462464 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 98545, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.574640154838562, + "objective/train/theoretical_loss": 4.2113936276006045, + "objective/train/tokens_used": 278528000, + "theoretical_loss": 4.2113936276006045, + "tokens_seen": 278528000 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004624272818455366, + "loss": 1.569, + "theoretical_loss": 4.2113936276006045, + "tokens_seen": 278528000 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046241725175526585, + "loss": 1.634, + "theoretical_loss": 4.211276767802868, + "tokens_seen": 278593536 + }, + { + "epoch": 0.08, + "learning_rate": 0.000462407221664995, + "loss": 1.6379, + "theoretical_loss": 4.211159943186978, + "tokens_seen": 278659072 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004623971915747242, + "loss": 1.6896, + "theoretical_loss": 4.211043153734071, + "tokens_seen": 278724608 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004623871614844534, + "loss": 1.6614, + "theoretical_loss": 4.2109263994253, + "tokens_seen": 278790144 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046237713139418257, + "loss": 1.5757, + "theoretical_loss": 4.21080968024183, + "tokens_seen": 278855680 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046236710130391175, + "loss": 1.5915, + "theoretical_loss": 4.210692996164845, + "tokens_seen": 278921216 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046235707121364093, + "loss": 1.5888, + "theoretical_loss": 4.210576347175539, + "tokens_seen": 278986752 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004623470411233701, + "loss": 1.513, + "theoretical_loss": 4.210459733255123, + "tokens_seen": 279052288 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046233701103309935, + "loss": 1.5449, + "theoretical_loss": 4.2103431543848195, + "tokens_seen": 279117824 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004623269809428285, + "loss": 1.5543, + "theoretical_loss": 4.210226610545872, + "tokens_seen": 279183360 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004623169508525577, + "loss": 1.6503, + "theoretical_loss": 4.21011010171953, + "tokens_seen": 279248896 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046230692076228684, + "loss": 1.6741, + "theoretical_loss": 4.209993627887064, + "tokens_seen": 279314432 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004622968906720161, + "loss": 1.6137, + "theoretical_loss": 4.209877189029756, + "tokens_seen": 279379968 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046228686058174525, + "loss": 1.6526, + "theoretical_loss": 4.209760785128903, + "tokens_seen": 279445504 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046227683049147444, + "loss": 1.6733, + "theoretical_loss": 4.2096444161658155, + "tokens_seen": 279511040 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004622668004012036, + "loss": 1.5603, + "theoretical_loss": 4.209528082121819, + "tokens_seen": 279576576 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046225677031093285, + "loss": 1.6788, + "theoretical_loss": 4.209411782978256, + "tokens_seen": 279642112 + }, + { + "epoch": 0.08, + "learning_rate": 0.000462246740220662, + "loss": 1.5835, + "theoretical_loss": 4.209295518716477, + "tokens_seen": 279707648 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004622367101303912, + "loss": 1.4769, + "theoretical_loss": 4.209179289317854, + "tokens_seen": 279773184 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046222668004012034, + "loss": 1.7319, + "theoretical_loss": 4.209063094763767, + "tokens_seen": 279838720 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004622166499498496, + "loss": 1.6983, + "theoretical_loss": 4.208946935035616, + "tokens_seen": 279904256 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046220661985957876, + "loss": 1.5698, + "theoretical_loss": 4.2088308101148115, + "tokens_seen": 279969792 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046219658976930794, + "loss": 1.5526, + "theoretical_loss": 4.208714719982778, + "tokens_seen": 280035328 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004621865596790371, + "loss": 1.5406, + "theoretical_loss": 4.208598664620958, + "tokens_seen": 280100864 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004621765295887663, + "loss": 1.7357, + "theoretical_loss": 4.208482644010804, + "tokens_seen": 280166400 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004621664994984955, + "loss": 1.6597, + "theoretical_loss": 4.2083666581337855, + "tokens_seen": 280231936 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004621564694082247, + "loss": 1.5877, + "theoretical_loss": 4.208250706971385, + "tokens_seen": 280297472 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046214643931795384, + "loss": 1.5392, + "theoretical_loss": 4.208134790505099, + "tokens_seen": 280363008 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004621364092276831, + "loss": 1.4883, + "theoretical_loss": 4.20801890871644, + "tokens_seen": 280428544 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004621263791374122, + "loss": 1.5786, + "theoretical_loss": 4.207903061586931, + "tokens_seen": 280494080 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046211634904714144, + "loss": 1.5564, + "theoretical_loss": 4.207787249098113, + "tokens_seen": 280559616 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004621063189568706, + "loss": 1.601, + "theoretical_loss": 4.20767147123154, + "tokens_seen": 280625152 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004620962888665998, + "loss": 1.658, + "theoretical_loss": 4.20755572796878, + "tokens_seen": 280690688 + }, + { + "epoch": 0.09, + "learning_rate": 0.000462086258776329, + "loss": 1.6292, + "theoretical_loss": 4.207440019291413, + "tokens_seen": 280756224 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004620762286860582, + "loss": 1.5908, + "theoretical_loss": 4.2073243451810365, + "tokens_seen": 280821760 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046206619859578735, + "loss": 1.6219, + "theoretical_loss": 4.20720870561926, + "tokens_seen": 280887296 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004620561685055166, + "loss": 1.5333, + "theoretical_loss": 4.2070931005877075, + "tokens_seen": 280952832 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004620461384152457, + "loss": 1.7092, + "theoretical_loss": 4.206977530068017, + "tokens_seen": 281018368 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046203610832497495, + "loss": 1.531, + "theoretical_loss": 4.206861994041842, + "tokens_seen": 281083904 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004620260782347041, + "loss": 1.6349, + "theoretical_loss": 4.206746492490847, + "tokens_seen": 281149440 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004620160481444333, + "loss": 1.6054, + "theoretical_loss": 4.206631025396714, + "tokens_seen": 281214976 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004620060180541625, + "loss": 1.5845, + "theoretical_loss": 4.206515592741135, + "tokens_seen": 281280512 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046199598796389167, + "loss": 1.5651, + "theoretical_loss": 4.20640019450582, + "tokens_seen": 281346048 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046198595787362085, + "loss": 1.6894, + "theoretical_loss": 4.206284830672491, + "tokens_seen": 281411584 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004619759277833501, + "loss": 1.609, + "theoretical_loss": 4.206169501222885, + "tokens_seen": 281477120 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004619658976930792, + "loss": 1.4427, + "theoretical_loss": 4.20605420613875, + "tokens_seen": 281542656 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046195586760280845, + "loss": 1.5745, + "theoretical_loss": 4.205938945401851, + "tokens_seen": 281608192 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046194583751253763, + "loss": 1.4875, + "theoretical_loss": 4.205823718993966, + "tokens_seen": 281673728 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004619358074222668, + "loss": 1.533, + "theoretical_loss": 4.2057085268968875, + "tokens_seen": 281739264 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 99475, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6061879396438599, + "objective/train/theoretical_loss": 4.205593369092421, + "objective/train/tokens_used": 281804800, + "theoretical_loss": 4.205593369092421, + "tokens_seen": 281804800 + }, + { + "epoch": 0.09, + "learning_rate": 0.000461925777331996, + "loss": 1.5836, + "theoretical_loss": 4.205593369092421, + "tokens_seen": 281804800 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004619157472417252, + "loss": 1.6773, + "theoretical_loss": 4.205478245562386, + "tokens_seen": 281870336 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046190571715145435, + "loss": 1.5803, + "theoretical_loss": 4.205363156288615, + "tokens_seen": 281935872 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618956870611836, + "loss": 1.6053, + "theoretical_loss": 4.205248101252957, + "tokens_seen": 282001408 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618856569709127, + "loss": 1.5145, + "theoretical_loss": 4.205133080437273, + "tokens_seen": 282066944 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046187562688064195, + "loss": 1.6978, + "theoretical_loss": 4.205018093823437, + "tokens_seen": 282132480 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618655967903711, + "loss": 1.5611, + "theoretical_loss": 4.204903141393338, + "tokens_seen": 282198016 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618555667001003, + "loss": 1.5037, + "theoretical_loss": 4.204788223128879, + "tokens_seen": 282263552 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618455366098295, + "loss": 1.5888, + "theoretical_loss": 4.204673339011976, + "tokens_seen": 282329088 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618355065195587, + "loss": 1.6625, + "theoretical_loss": 4.204558489024561, + "tokens_seen": 282394624 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046182547642928786, + "loss": 1.5943, + "theoretical_loss": 4.204443673148575, + "tokens_seen": 282460160 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046181544633901704, + "loss": 1.486, + "theoretical_loss": 4.204328891365977, + "tokens_seen": 282525696 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618054162487462, + "loss": 1.5678, + "theoretical_loss": 4.2042141436587395, + "tokens_seen": 282591232 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046179538615847545, + "loss": 1.5174, + "theoretical_loss": 4.204099430008846, + "tokens_seen": 282656768 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004617853560682046, + "loss": 1.62, + "theoretical_loss": 4.203984750398296, + "tokens_seen": 282722304 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004617753259779338, + "loss": 1.6092, + "theoretical_loss": 4.203870104809101, + "tokens_seen": 282787840 + }, + { + "epoch": 0.09, + "learning_rate": 0.000461765295887663, + "loss": 1.5154, + "theoretical_loss": 4.203755493223289, + "tokens_seen": 282853376 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004617552657973922, + "loss": 1.625, + "theoretical_loss": 4.203640915622898, + "tokens_seen": 282918912 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046174523570712136, + "loss": 1.5811, + "theoretical_loss": 4.203526371989983, + "tokens_seen": 282984448 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046173520561685054, + "loss": 1.6317, + "theoretical_loss": 4.203411862306609, + "tokens_seen": 283049984 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004617251755265797, + "loss": 1.611, + "theoretical_loss": 4.203297386554859, + "tokens_seen": 283115520 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046171514543630896, + "loss": 1.5395, + "theoretical_loss": 4.203182944716826, + "tokens_seen": 283181056 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046170511534603814, + "loss": 1.6287, + "theoretical_loss": 4.203068536774619, + "tokens_seen": 283246592 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004616950852557673, + "loss": 1.6767, + "theoretical_loss": 4.202954162710357, + "tokens_seen": 283312128 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004616850551654965, + "loss": 1.5672, + "theoretical_loss": 4.202839822506178, + "tokens_seen": 283377664 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004616750250752257, + "loss": 1.6017, + "theoretical_loss": 4.202725516144228, + "tokens_seen": 283443200 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004616649949849549, + "loss": 1.6326, + "theoretical_loss": 4.202611243606671, + "tokens_seen": 283508736 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046165496489468404, + "loss": 1.6517, + "theoretical_loss": 4.202497004875681, + "tokens_seen": 283574272 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004616449348044133, + "loss": 1.5927, + "theoretical_loss": 4.202382799933449, + "tokens_seen": 283639808 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004616349047141424, + "loss": 1.5759, + "theoretical_loss": 4.2022686287621775, + "tokens_seen": 283705344 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046162487462387164, + "loss": 1.6293, + "theoretical_loss": 4.20215449134408, + "tokens_seen": 283770880 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004616148445336008, + "loss": 1.5713, + "theoretical_loss": 4.202040387661389, + "tokens_seen": 283836416 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046160481444333, + "loss": 1.6009, + "theoretical_loss": 4.201926317696347, + "tokens_seen": 283901952 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004615947843530592, + "loss": 1.5766, + "theoretical_loss": 4.201812281431209, + "tokens_seen": 283967488 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004615847542627884, + "loss": 1.7301, + "theoretical_loss": 4.201698278848246, + "tokens_seen": 284033024 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046157472417251755, + "loss": 1.6651, + "theoretical_loss": 4.201584309929743, + "tokens_seen": 284098560 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004615646940822468, + "loss": 1.5775, + "theoretical_loss": 4.201470374657993, + "tokens_seen": 284164096 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004615546639919759, + "loss": 1.6636, + "theoretical_loss": 4.201356473015308, + "tokens_seen": 284229632 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046154463390170515, + "loss": 1.5359, + "theoretical_loss": 4.201242604984013, + "tokens_seen": 284295168 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004615346038114343, + "loss": 1.616, + "theoretical_loss": 4.201128770546442, + "tokens_seen": 284360704 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004615245737211635, + "loss": 1.5085, + "theoretical_loss": 4.201014969684948, + "tokens_seen": 284426240 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004615145436308927, + "loss": 1.5816, + "theoretical_loss": 4.200901202381893, + "tokens_seen": 284491776 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046150451354062187, + "loss": 1.6152, + "theoretical_loss": 4.200787468619653, + "tokens_seen": 284557312 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046149448345035105, + "loss": 1.5926, + "theoretical_loss": 4.200673768380621, + "tokens_seen": 284622848 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004614844533600803, + "loss": 1.618, + "theoretical_loss": 4.200560101647198, + "tokens_seen": 284688384 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004614744232698094, + "loss": 1.6783, + "theoretical_loss": 4.2004464684018, + "tokens_seen": 284753920 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046146439317953865, + "loss": 1.5982, + "theoretical_loss": 4.200332868626861, + "tokens_seen": 284819456 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046145436308926783, + "loss": 1.5831, + "theoretical_loss": 4.200219302304821, + "tokens_seen": 284884992 + }, + { + "epoch": 0.09, + "learning_rate": 0.000461444332998997, + "loss": 1.6036, + "theoretical_loss": 4.200105769418138, + "tokens_seen": 284950528 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004614343029087262, + "loss": 1.6844, + "theoretical_loss": 4.199992269949281, + "tokens_seen": 285016064 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 100822, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3841652870178223, + "objective/train/theoretical_loss": 4.199878803880733, + "objective/train/tokens_used": 285081600, + "theoretical_loss": 4.199878803880733, + "tokens_seen": 285081600 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004614242728184554, + "loss": 1.487, + "theoretical_loss": 4.199878803880733, + "tokens_seen": 285081600 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046141424272818455, + "loss": 1.6636, + "theoretical_loss": 4.1997653711949905, + "tokens_seen": 285147136 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004614042126379138, + "loss": 1.61, + "theoretical_loss": 4.199651971874562, + "tokens_seen": 285212672 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004613941825476429, + "loss": 1.5296, + "theoretical_loss": 4.199538605901973, + "tokens_seen": 285278208 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046138415245737215, + "loss": 1.5888, + "theoretical_loss": 4.199425273259756, + "tokens_seen": 285343744 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004613741223671013, + "loss": 1.6043, + "theoretical_loss": 4.199311973930461, + "tokens_seen": 285409280 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004613640922768305, + "loss": 1.6074, + "theoretical_loss": 4.199198707896651, + "tokens_seen": 285474816 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004613540621865597, + "loss": 1.5576, + "theoretical_loss": 4.199085475140899, + "tokens_seen": 285540352 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004613440320962889, + "loss": 1.609, + "theoretical_loss": 4.1989722756457954, + "tokens_seen": 285605888 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046133400200601806, + "loss": 1.6493, + "theoretical_loss": 4.198859109393941, + "tokens_seen": 285671424 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046132397191574724, + "loss": 1.5456, + "theoretical_loss": 4.198745976367949, + "tokens_seen": 285736960 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004613139418254764, + "loss": 1.7346, + "theoretical_loss": 4.198632876550449, + "tokens_seen": 285802496 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046130391173520566, + "loss": 1.459, + "theoretical_loss": 4.19851980992408, + "tokens_seen": 285868032 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004612938816449348, + "loss": 1.6108, + "theoretical_loss": 4.198406776471497, + "tokens_seen": 285933568 + }, + { + "epoch": 0.09, + "learning_rate": 0.000461283851554664, + "loss": 1.7597, + "theoretical_loss": 4.198293776175365, + "tokens_seen": 285999104 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004612738214643932, + "loss": 1.5149, + "theoretical_loss": 4.198180809018366, + "tokens_seen": 286064640 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004612637913741224, + "loss": 1.5863, + "theoretical_loss": 4.198067874983192, + "tokens_seen": 286130176 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046125376128385156, + "loss": 1.625, + "theoretical_loss": 4.197954974052546, + "tokens_seen": 286195712 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046124373119358074, + "loss": 1.7222, + "theoretical_loss": 4.19784210620915, + "tokens_seen": 286261248 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004612337011033099, + "loss": 1.6523, + "theoretical_loss": 4.197729271435736, + "tokens_seen": 286326784 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046122367101303916, + "loss": 1.6684, + "theoretical_loss": 4.197616469715046, + "tokens_seen": 286392320 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004612136409227683, + "loss": 1.6476, + "theoretical_loss": 4.19750370102984, + "tokens_seen": 286457856 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004612036108324975, + "loss": 1.6093, + "theoretical_loss": 4.197390965362887, + "tokens_seen": 286523392 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046119358074222665, + "loss": 1.5758, + "theoretical_loss": 4.197278262696971, + "tokens_seen": 286588928 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004611835506519559, + "loss": 1.5782, + "theoretical_loss": 4.197165593014889, + "tokens_seen": 286654464 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046117352056168506, + "loss": 1.6051, + "theoretical_loss": 4.197052956299449, + "tokens_seen": 286720000 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046116349047141425, + "loss": 1.5403, + "theoretical_loss": 4.196940352533474, + "tokens_seen": 286785536 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004611534603811434, + "loss": 1.5922, + "theoretical_loss": 4.196827781699799, + "tokens_seen": 286851072 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004611434302908726, + "loss": 1.6661, + "theoretical_loss": 4.196715243781273, + "tokens_seen": 286916608 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004611334002006018, + "loss": 1.6312, + "theoretical_loss": 4.196602738760754, + "tokens_seen": 286982144 + }, + { + "epoch": 0.09, + "learning_rate": 0.000461123370110331, + "loss": 1.6225, + "theoretical_loss": 4.196490266621119, + "tokens_seen": 287047680 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046111334002006015, + "loss": 1.4926, + "theoretical_loss": 4.196377827345252, + "tokens_seen": 287113216 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004611033099297894, + "loss": 1.6104, + "theoretical_loss": 4.196265420916053, + "tokens_seen": 287178752 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046109327983951857, + "loss": 1.6789, + "theoretical_loss": 4.196153047316433, + "tokens_seen": 287244288 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046108324974924775, + "loss": 1.6032, + "theoretical_loss": 4.196040706529319, + "tokens_seen": 287309824 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046107321965897693, + "loss": 1.6091, + "theoretical_loss": 4.195928398537647, + "tokens_seen": 287375360 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004610631895687061, + "loss": 1.5571, + "theoretical_loss": 4.195816123324367, + "tokens_seen": 287440896 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004610531594784353, + "loss": 1.6503, + "theoretical_loss": 4.195703880872443, + "tokens_seen": 287506432 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004610431293881645, + "loss": 1.5722, + "theoretical_loss": 4.195591671164852, + "tokens_seen": 287571968 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046103309929789365, + "loss": 1.5621, + "theoretical_loss": 4.195479494184581, + "tokens_seen": 287637504 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004610230692076229, + "loss": 1.6086, + "theoretical_loss": 4.195367349914631, + "tokens_seen": 287703040 + }, + { + "epoch": 0.09, + "learning_rate": 0.000461013039117352, + "loss": 1.6349, + "theoretical_loss": 4.195255238338016, + "tokens_seen": 287768576 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046100300902708125, + "loss": 1.6375, + "theoretical_loss": 4.195143159437764, + "tokens_seen": 287834112 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046099297893681043, + "loss": 1.5937, + "theoretical_loss": 4.195031113196913, + "tokens_seen": 287899648 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004609829488465396, + "loss": 1.5758, + "theoretical_loss": 4.194919099598516, + "tokens_seen": 287965184 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004609729187562688, + "loss": 1.4192, + "theoretical_loss": 4.194807118625636, + "tokens_seen": 288030720 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046096288866599803, + "loss": 1.595, + "theoretical_loss": 4.194695170261353, + "tokens_seen": 288096256 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004609528585757272, + "loss": 1.6054, + "theoretical_loss": 4.194583254488754, + "tokens_seen": 288161792 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004609428284854564, + "loss": 1.697, + "theoretical_loss": 4.194471371290944, + "tokens_seen": 288227328 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004609327983951856, + "loss": 1.5103, + "theoretical_loss": 4.194359520651036, + "tokens_seen": 288292864 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 101706, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6587169170379639, + "objective/train/theoretical_loss": 4.1942477025521585, + "objective/train/tokens_used": 288358400, + "theoretical_loss": 4.1942477025521585, + "tokens_seen": 288358400 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046092276830491475, + "loss": 1.6245, + "theoretical_loss": 4.1942477025521585, + "tokens_seen": 288358400 + }, + { + "epoch": 0.09, + "learning_rate": 0.000460912738214644, + "loss": 1.6083, + "theoretical_loss": 4.194135916977452, + "tokens_seen": 288423936 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004609027081243731, + "loss": 1.6458, + "theoretical_loss": 4.19402416391007, + "tokens_seen": 288489472 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046089267803410235, + "loss": 1.6983, + "theoretical_loss": 4.193912443333177, + "tokens_seen": 288555008 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004608826479438315, + "loss": 1.7264, + "theoretical_loss": 4.193800755229951, + "tokens_seen": 288620544 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004608726178535607, + "loss": 1.6839, + "theoretical_loss": 4.193689099583582, + "tokens_seen": 288686080 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004608625877632899, + "loss": 1.7651, + "theoretical_loss": 4.193577476377275, + "tokens_seen": 288751616 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004608525576730191, + "loss": 1.5823, + "theoretical_loss": 4.193465885594242, + "tokens_seen": 288817152 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046084252758274826, + "loss": 1.6227, + "theoretical_loss": 4.1933543272177145, + "tokens_seen": 288882688 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046083249749247744, + "loss": 1.7178, + "theoretical_loss": 4.193242801230931, + "tokens_seen": 288948224 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004608224674022066, + "loss": 1.6188, + "theoretical_loss": 4.193131307617145, + "tokens_seen": 289013760 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046081243731193586, + "loss": 1.601, + "theoretical_loss": 4.193019846359622, + "tokens_seen": 289079296 + }, + { + "epoch": 0.09, + "learning_rate": 0.000460802407221665, + "loss": 1.5714, + "theoretical_loss": 4.192908417441639, + "tokens_seen": 289144832 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004607923771313942, + "loss": 1.5523, + "theoretical_loss": 4.192797020846487, + "tokens_seen": 289210368 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004607823470411234, + "loss": 1.6224, + "theoretical_loss": 4.192685656557468, + "tokens_seen": 289275904 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004607723169508526, + "loss": 1.6504, + "theoretical_loss": 4.1925743245578975, + "tokens_seen": 289341440 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046076228686058176, + "loss": 1.5781, + "theoretical_loss": 4.1924630248311034, + "tokens_seen": 289406976 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046075225677031094, + "loss": 1.5869, + "theoretical_loss": 4.192351757360425, + "tokens_seen": 289472512 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004607422266800401, + "loss": 1.6047, + "theoretical_loss": 4.1922405221292145, + "tokens_seen": 289538048 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046073219658976936, + "loss": 1.6603, + "theoretical_loss": 4.192129319120838, + "tokens_seen": 289603584 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004607221664994985, + "loss": 1.6344, + "theoretical_loss": 4.1920181483186685, + "tokens_seen": 289669120 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004607121364092277, + "loss": 1.6179, + "theoretical_loss": 4.1919070097060995, + "tokens_seen": 289734656 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046070210631895685, + "loss": 1.5886, + "theoretical_loss": 4.19179590326653, + "tokens_seen": 289800192 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004606920762286861, + "loss": 1.5338, + "theoretical_loss": 4.191684828983375, + "tokens_seen": 289865728 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046068204613841526, + "loss": 1.5962, + "theoretical_loss": 4.191573786840061, + "tokens_seen": 289931264 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046067201604814445, + "loss": 1.6627, + "theoretical_loss": 4.191462776820025, + "tokens_seen": 289996800 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004606619859578736, + "loss": 1.5902, + "theoretical_loss": 4.191351798906719, + "tokens_seen": 290062336 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004606519558676028, + "loss": 1.5823, + "theoretical_loss": 4.191240853083604, + "tokens_seen": 290127872 + }, + { + "epoch": 0.09, + "learning_rate": 0.000460641925777332, + "loss": 1.5785, + "theoretical_loss": 4.191129939334159, + "tokens_seen": 290193408 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004606318956870612, + "loss": 1.7811, + "theoretical_loss": 4.191019057641868, + "tokens_seen": 290258944 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046062186559679035, + "loss": 1.5877, + "theoretical_loss": 4.1909082079902324, + "tokens_seen": 290324480 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004606118355065196, + "loss": 1.6721, + "theoretical_loss": 4.190797390362764, + "tokens_seen": 290390016 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046060180541624877, + "loss": 1.5822, + "theoretical_loss": 4.190686604742986, + "tokens_seen": 290455552 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046059177532597795, + "loss": 1.5483, + "theoretical_loss": 4.190575851114437, + "tokens_seen": 290521088 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046058174523570713, + "loss": 1.6812, + "theoretical_loss": 4.190465129460662, + "tokens_seen": 290586624 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004605717151454363, + "loss": 1.5963, + "theoretical_loss": 4.190354439765224, + "tokens_seen": 290652160 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004605616850551655, + "loss": 1.599, + "theoretical_loss": 4.190243782011696, + "tokens_seen": 290717696 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004605516549648947, + "loss": 1.6094, + "theoretical_loss": 4.190133156183663, + "tokens_seen": 290783232 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046054162487462385, + "loss": 1.5143, + "theoretical_loss": 4.190022562264721, + "tokens_seen": 290848768 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004605315947843531, + "loss": 1.6056, + "theoretical_loss": 4.189912000238479, + "tokens_seen": 290914304 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004605215646940822, + "loss": 1.659, + "theoretical_loss": 4.189801470088559, + "tokens_seen": 290979840 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046051153460381145, + "loss": 1.6982, + "theoretical_loss": 4.189690971798596, + "tokens_seen": 291045376 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046050150451354063, + "loss": 1.6523, + "theoretical_loss": 4.189580505352232, + "tokens_seen": 291110912 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004604914744232698, + "loss": 1.6078, + "theoretical_loss": 4.189470070733129, + "tokens_seen": 291176448 + }, + { + "epoch": 0.09, + "learning_rate": 0.000460481444332999, + "loss": 1.5821, + "theoretical_loss": 4.189359667924952, + "tokens_seen": 291241984 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046047141424272823, + "loss": 1.641, + "theoretical_loss": 4.189249296911386, + "tokens_seen": 291307520 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046046138415245736, + "loss": 1.6439, + "theoretical_loss": 4.189138957676125, + "tokens_seen": 291373056 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004604513540621866, + "loss": 1.6086, + "theoretical_loss": 4.1890286502028715, + "tokens_seen": 291438592 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004604413239719157, + "loss": 1.7489, + "theoretical_loss": 4.188918374475347, + "tokens_seen": 291504128 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046043129388164495, + "loss": 1.6095, + "theoretical_loss": 4.188808130477279, + "tokens_seen": 291569664 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 102857, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4298292398452759, + "objective/train/theoretical_loss": 4.1886979181924096, + "objective/train/tokens_used": 291635200, + "theoretical_loss": 4.1886979181924096, + "tokens_seen": 291635200 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046042126379137414, + "loss": 1.5218, + "theoretical_loss": 4.1886979181924096, + "tokens_seen": 291635200 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004604112337011033, + "loss": 1.6599, + "theoretical_loss": 4.188587737604493, + "tokens_seen": 291700736 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004604012036108325, + "loss": 1.6004, + "theoretical_loss": 4.188477588697295, + "tokens_seen": 291766272 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004603911735205617, + "loss": 1.6032, + "theoretical_loss": 4.188367471454593, + "tokens_seen": 291831808 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046038114343029086, + "loss": 1.6519, + "theoretical_loss": 4.188257385860177, + "tokens_seen": 291897344 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004603711133400201, + "loss": 1.723, + "theoretical_loss": 4.188147331897849, + "tokens_seen": 291962880 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004603610832497492, + "loss": 1.5905, + "theoretical_loss": 4.1880373095514205, + "tokens_seen": 292028416 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046035105315947846, + "loss": 1.5589, + "theoretical_loss": 4.18792731880472, + "tokens_seen": 292093952 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004603410230692076, + "loss": 1.5839, + "theoretical_loss": 4.187817359641581, + "tokens_seen": 292159488 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004603309929789368, + "loss": 1.6503, + "theoretical_loss": 4.1877074320458565, + "tokens_seen": 292225024 + }, + { + "epoch": 0.09, + "learning_rate": 0.000460320962888666, + "loss": 1.5882, + "theoretical_loss": 4.187597536001406, + "tokens_seen": 292290560 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004603109327983952, + "loss": 1.6418, + "theoretical_loss": 4.187487671492102, + "tokens_seen": 292356096 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046030090270812436, + "loss": 1.6219, + "theoretical_loss": 4.18737783850183, + "tokens_seen": 292421632 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004602908726178536, + "loss": 1.5863, + "theoretical_loss": 4.187268037014486, + "tokens_seen": 292487168 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004602808425275827, + "loss": 1.5333, + "theoretical_loss": 4.187158267013979, + "tokens_seen": 292552704 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046027081243731196, + "loss": 1.5943, + "theoretical_loss": 4.18704852848423, + "tokens_seen": 292618240 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004602607823470411, + "loss": 1.6109, + "theoretical_loss": 4.18693882140917, + "tokens_seen": 292683776 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004602507522567703, + "loss": 1.6599, + "theoretical_loss": 4.186829145772743, + "tokens_seen": 292749312 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004602407221664995, + "loss": 1.5455, + "theoretical_loss": 4.186719501558905, + "tokens_seen": 292814848 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004602306920762287, + "loss": 1.5837, + "theoretical_loss": 4.186609888751623, + "tokens_seen": 292880384 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046022066198595787, + "loss": 1.7117, + "theoretical_loss": 4.186500307334878, + "tokens_seen": 292945920 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046021063189568705, + "loss": 1.6191, + "theoretical_loss": 4.186390757292659, + "tokens_seen": 293011456 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004602006018054163, + "loss": 1.5649, + "theoretical_loss": 4.18628123860897, + "tokens_seen": 293076992 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046019057171514546, + "loss": 1.5196, + "theoretical_loss": 4.186171751267825, + "tokens_seen": 293142528 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046018054162487465, + "loss": 1.5476, + "theoretical_loss": 4.186062295253249, + "tokens_seen": 293208064 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004601705115346038, + "loss": 1.6621, + "theoretical_loss": 4.185952870549283, + "tokens_seen": 293273600 + }, + { + "epoch": 0.09, + "learning_rate": 0.000460160481444333, + "loss": 1.6178, + "theoretical_loss": 4.185843477139974, + "tokens_seen": 293339136 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004601504513540622, + "loss": 1.6957, + "theoretical_loss": 4.185734115009383, + "tokens_seen": 293404672 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004601404212637914, + "loss": 1.4654, + "theoretical_loss": 4.185624784141585, + "tokens_seen": 293470208 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046013039117352055, + "loss": 1.6774, + "theoretical_loss": 4.185515484520664, + "tokens_seen": 293535744 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004601203610832498, + "loss": 1.6964, + "theoretical_loss": 4.1854062161307155, + "tokens_seen": 293601280 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046011033099297897, + "loss": 1.5609, + "theoretical_loss": 4.185296978955848, + "tokens_seen": 293666816 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046010030090270815, + "loss": 1.6209, + "theoretical_loss": 4.185187772980181, + "tokens_seen": 293732352 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046009027081243733, + "loss": 1.7449, + "theoretical_loss": 4.185078598187846, + "tokens_seen": 293797888 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004600802407221665, + "loss": 1.4763, + "theoretical_loss": 4.184969454562985, + "tokens_seen": 293863424 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004600702106318957, + "loss": 1.6177, + "theoretical_loss": 4.1848603420897525, + "tokens_seen": 293928960 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046006018054162493, + "loss": 1.6278, + "theoretical_loss": 4.184751260752316, + "tokens_seen": 293994496 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046005015045135405, + "loss": 1.7176, + "theoretical_loss": 4.18464221053485, + "tokens_seen": 294060032 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004600401203610833, + "loss": 1.483, + "theoretical_loss": 4.184533191421547, + "tokens_seen": 294125568 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004600300902708124, + "loss": 1.5372, + "theoretical_loss": 4.184424203396606, + "tokens_seen": 294191104 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046002006018054165, + "loss": 1.6386, + "theoretical_loss": 4.184315246444239, + "tokens_seen": 294256640 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046001003009027083, + "loss": 1.5098, + "theoretical_loss": 4.18420632054867, + "tokens_seen": 294322176 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046, + "loss": 1.63, + "theoretical_loss": 4.1840974256941355, + "tokens_seen": 294387712 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004599899699097292, + "loss": 1.5182, + "theoretical_loss": 4.183988561864879, + "tokens_seen": 294453248 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045997993981945843, + "loss": 1.5543, + "theoretical_loss": 4.183879729045163, + "tokens_seen": 294518784 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045996990972918756, + "loss": 1.687, + "theoretical_loss": 4.183770927219255, + "tokens_seen": 294584320 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004599598796389168, + "loss": 1.6337, + "theoretical_loss": 4.183662156371436, + "tokens_seen": 294649856 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004599498495486459, + "loss": 1.6558, + "theoretical_loss": 4.183553416485999, + "tokens_seen": 294715392 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045993981945837515, + "loss": 1.5003, + "theoretical_loss": 4.183444707547249, + "tokens_seen": 294780928 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045992978936810434, + "loss": 1.5689, + "theoretical_loss": 4.1833360295395, + "tokens_seen": 294846464 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 103707, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5481871366500854, + "objective/train/theoretical_loss": 4.183227382447081, + "objective/train/tokens_used": 294912000, + "theoretical_loss": 4.183227382447081, + "tokens_seen": 294912000 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004599197592778335, + "loss": 1.6119, + "theoretical_loss": 4.183227382447081, + "tokens_seen": 294912000 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004599097291875627, + "loss": 1.6385, + "theoretical_loss": 4.183118766254328, + "tokens_seen": 294977536 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004598996990972919, + "loss": 1.5895, + "theoretical_loss": 4.183010180945593, + "tokens_seen": 295043072 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045988966900702106, + "loss": 1.7058, + "theoretical_loss": 4.182901626505237, + "tokens_seen": 295108608 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004598796389167503, + "loss": 1.5637, + "theoretical_loss": 4.182793102917632, + "tokens_seen": 295174144 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004598696088264794, + "loss": 1.6573, + "theoretical_loss": 4.182684610167162, + "tokens_seen": 295239680 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045985957873620866, + "loss": 1.6564, + "theoretical_loss": 4.1825761482382235, + "tokens_seen": 295305216 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004598495486459378, + "loss": 1.5605, + "theoretical_loss": 4.182467717115221, + "tokens_seen": 295370752 + }, + { + "epoch": 0.09, + "learning_rate": 0.000459839518555667, + "loss": 1.6656, + "theoretical_loss": 4.182359316782576, + "tokens_seen": 295436288 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004598294884653962, + "loss": 1.5882, + "theoretical_loss": 4.1822509472247145, + "tokens_seen": 295501824 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004598194583751254, + "loss": 1.5595, + "theoretical_loss": 4.18214260842608, + "tokens_seen": 295567360 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045980942828485456, + "loss": 1.5589, + "theoretical_loss": 4.182034300371122, + "tokens_seen": 295632896 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004597993981945838, + "loss": 1.622, + "theoretical_loss": 4.181926023044306, + "tokens_seen": 295698432 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004597893681043129, + "loss": 1.6169, + "theoretical_loss": 4.181817776430106, + "tokens_seen": 295763968 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045977933801404216, + "loss": 1.6115, + "theoretical_loss": 4.181709560513008, + "tokens_seen": 295829504 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004597693079237713, + "loss": 1.6455, + "theoretical_loss": 4.181601375277509, + "tokens_seen": 295895040 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004597592778335005, + "loss": 1.6229, + "theoretical_loss": 4.181493220708117, + "tokens_seen": 295960576 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004597492477432297, + "loss": 1.579, + "theoretical_loss": 4.181385096789353, + "tokens_seen": 296026112 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004597392176529589, + "loss": 1.642, + "theoretical_loss": 4.181277003505747, + "tokens_seen": 296091648 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045972918756268807, + "loss": 1.6379, + "theoretical_loss": 4.181168940841843, + "tokens_seen": 296157184 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045971915747241725, + "loss": 1.4943, + "theoretical_loss": 4.181060908782191, + "tokens_seen": 296222720 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045970912738214643, + "loss": 1.567, + "theoretical_loss": 4.18095290731136, + "tokens_seen": 296288256 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045969909729187566, + "loss": 1.7021, + "theoretical_loss": 4.180844936413922, + "tokens_seen": 296353792 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004596890672016048, + "loss": 1.6724, + "theoretical_loss": 4.180736996074465, + "tokens_seen": 296419328 + }, + { + "epoch": 0.09, + "learning_rate": 0.000459679037111334, + "loss": 1.644, + "theoretical_loss": 4.180629086277589, + "tokens_seen": 296484864 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045966900702106315, + "loss": 1.5635, + "theoretical_loss": 4.180521207007902, + "tokens_seen": 296550400 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004596589769307924, + "loss": 1.5265, + "theoretical_loss": 4.180413358250024, + "tokens_seen": 296615936 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045964894684052157, + "loss": 1.6248, + "theoretical_loss": 4.180305539988588, + "tokens_seen": 296681472 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045963891675025075, + "loss": 1.5089, + "theoretical_loss": 4.180197752208235, + "tokens_seen": 296747008 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045962888665997993, + "loss": 1.6032, + "theoretical_loss": 4.1800899948936205, + "tokens_seen": 296812544 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045961885656970917, + "loss": 1.6443, + "theoretical_loss": 4.179982268029409, + "tokens_seen": 296878080 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004596088264794383, + "loss": 1.6703, + "theoretical_loss": 4.179874571600277, + "tokens_seen": 296943616 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045959879638916753, + "loss": 1.5093, + "theoretical_loss": 4.1797669055909115, + "tokens_seen": 297009152 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045958876629889666, + "loss": 1.5974, + "theoretical_loss": 4.1796592699860105, + "tokens_seen": 297074688 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004595787362086259, + "loss": 1.5621, + "theoretical_loss": 4.179551664770283, + "tokens_seen": 297140224 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004595687061183551, + "loss": 1.6053, + "theoretical_loss": 4.1794440899284515, + "tokens_seen": 297205760 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045955867602808425, + "loss": 1.5362, + "theoretical_loss": 4.179336545445245, + "tokens_seen": 297271296 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045954864593781344, + "loss": 1.5299, + "theoretical_loss": 4.1792290313054075, + "tokens_seen": 297336832 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004595386158475426, + "loss": 1.5963, + "theoretical_loss": 4.179121547493692, + "tokens_seen": 297402368 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004595285857572718, + "loss": 1.6351, + "theoretical_loss": 4.179014093994862, + "tokens_seen": 297467904 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045951855566700103, + "loss": 1.6087, + "theoretical_loss": 4.178906670793695, + "tokens_seen": 297533440 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045950852557673016, + "loss": 1.5341, + "theoretical_loss": 4.178799277874977, + "tokens_seen": 297598976 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004594984954864594, + "loss": 1.5372, + "theoretical_loss": 4.178691915223505, + "tokens_seen": 297664512 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004594884653961885, + "loss": 1.5154, + "theoretical_loss": 4.178584582824088, + "tokens_seen": 297730048 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045947843530591776, + "loss": 1.5373, + "theoretical_loss": 4.1784772806615464, + "tokens_seen": 297795584 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045946840521564694, + "loss": 1.5205, + "theoretical_loss": 4.178370008720709, + "tokens_seen": 297861120 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004594583751253761, + "loss": 1.6103, + "theoretical_loss": 4.178262766986418, + "tokens_seen": 297926656 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045944834503510536, + "loss": 1.4865, + "theoretical_loss": 4.178155555443525, + "tokens_seen": 297992192 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045943831494483454, + "loss": 1.6157, + "theoretical_loss": 4.178048374076894, + "tokens_seen": 298057728 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004594282848545637, + "loss": 1.5685, + "theoretical_loss": 4.1779412228714, + "tokens_seen": 298123264 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 105034, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4982225894927979, + "objective/train/theoretical_loss": 4.177834101811927, + "objective/train/tokens_used": 298188800, + "theoretical_loss": 4.177834101811927, + "tokens_seen": 298188800 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004594182547642929, + "loss": 1.6447, + "theoretical_loss": 4.177834101811927, + "tokens_seen": 298188800 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004594082246740221, + "loss": 1.6281, + "theoretical_loss": 4.1777270108833715, + "tokens_seen": 298254336 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045939819458375126, + "loss": 1.5271, + "theoretical_loss": 4.177619950070639, + "tokens_seen": 298319872 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004593881644934805, + "loss": 1.4994, + "theoretical_loss": 4.177512919358649, + "tokens_seen": 298385408 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004593781344032096, + "loss": 1.6354, + "theoretical_loss": 4.17740591873233, + "tokens_seen": 298450944 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045936810431293886, + "loss": 1.6065, + "theoretical_loss": 4.177298948176619, + "tokens_seen": 298516480 + }, + { + "epoch": 0.09, + "learning_rate": 0.000459358074222668, + "loss": 1.6189, + "theoretical_loss": 4.17719200767647, + "tokens_seen": 298582016 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004593480441323972, + "loss": 1.6078, + "theoretical_loss": 4.177085097216842, + "tokens_seen": 298647552 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004593380140421264, + "loss": 1.6331, + "theoretical_loss": 4.1769782167827065, + "tokens_seen": 298713088 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004593279839518556, + "loss": 1.635, + "theoretical_loss": 4.176871366359047, + "tokens_seen": 298778624 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045931795386158476, + "loss": 1.5498, + "theoretical_loss": 4.176764545930858, + "tokens_seen": 298844160 + }, + { + "epoch": 0.09, + "learning_rate": 0.000459307923771314, + "loss": 1.658, + "theoretical_loss": 4.176657755483143, + "tokens_seen": 298909696 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004592978936810431, + "loss": 1.5902, + "theoretical_loss": 4.176550995000917, + "tokens_seen": 298975232 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045928786359077236, + "loss": 1.4861, + "theoretical_loss": 4.176444264469206, + "tokens_seen": 299040768 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004592778335005015, + "loss": 1.5962, + "theoretical_loss": 4.176337563873046, + "tokens_seen": 299106304 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004592678034102307, + "loss": 1.558, + "theoretical_loss": 4.176230893197486, + "tokens_seen": 299171840 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004592577733199599, + "loss": 1.5454, + "theoretical_loss": 4.1761242524275834, + "tokens_seen": 299237376 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004592477432296891, + "loss": 1.6242, + "theoretical_loss": 4.176017641548407, + "tokens_seen": 299302912 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045923771313941827, + "loss": 1.6442, + "theoretical_loss": 4.175911060545037, + "tokens_seen": 299368448 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045922768304914745, + "loss": 1.4502, + "theoretical_loss": 4.175804509402562, + "tokens_seen": 299433984 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045921765295887663, + "loss": 1.5458, + "theoretical_loss": 4.175697988106085, + "tokens_seen": 299499520 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045920762286860586, + "loss": 1.6018, + "theoretical_loss": 4.175591496640718, + "tokens_seen": 299565056 + }, + { + "epoch": 0.09, + "learning_rate": 0.000459197592778335, + "loss": 1.657, + "theoretical_loss": 4.175485034991581, + "tokens_seen": 299630592 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004591875626880642, + "loss": 1.692, + "theoretical_loss": 4.17537860314381, + "tokens_seen": 299696128 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045917753259779335, + "loss": 1.5191, + "theoretical_loss": 4.175272201082547, + "tokens_seen": 299761664 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004591675025075226, + "loss": 1.6463, + "theoretical_loss": 4.175165828792946, + "tokens_seen": 299827200 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045915747241725177, + "loss": 1.5826, + "theoretical_loss": 4.1750594862601735, + "tokens_seen": 299892736 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045914744232698095, + "loss": 1.5051, + "theoretical_loss": 4.1749531734694045, + "tokens_seen": 299958272 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045913741223671013, + "loss": 1.5589, + "theoretical_loss": 4.174846890405825, + "tokens_seen": 300023808 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045912738214643937, + "loss": 1.6746, + "theoretical_loss": 4.174740637054632, + "tokens_seen": 300089344 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004591173520561685, + "loss": 1.646, + "theoretical_loss": 4.174634413401034, + "tokens_seen": 300154880 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045910732196589773, + "loss": 1.4478, + "theoretical_loss": 4.174528219430247, + "tokens_seen": 300220416 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045909729187562686, + "loss": 1.5256, + "theoretical_loss": 4.174422055127502, + "tokens_seen": 300285952 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004590872617853561, + "loss": 1.6543, + "theoretical_loss": 4.174315920478037, + "tokens_seen": 300351488 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004590772316950853, + "loss": 1.5638, + "theoretical_loss": 4.174209815467102, + "tokens_seen": 300417024 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045906720160481445, + "loss": 1.5821, + "theoretical_loss": 4.174103740079958, + "tokens_seen": 300482560 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045905717151454364, + "loss": 1.5525, + "theoretical_loss": 4.1739976943018755, + "tokens_seen": 300548096 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004590471414242728, + "loss": 1.7127, + "theoretical_loss": 4.173891678118135, + "tokens_seen": 300613632 + }, + { + "epoch": 0.09, + "learning_rate": 0.000459037111334002, + "loss": 1.5185, + "theoretical_loss": 4.173785691514031, + "tokens_seen": 300679168 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045902708124373123, + "loss": 1.8115, + "theoretical_loss": 4.173679734474862, + "tokens_seen": 300744704 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045901705115346036, + "loss": 1.6229, + "theoretical_loss": 4.173573806985945, + "tokens_seen": 300810240 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004590070210631896, + "loss": 1.6001, + "theoretical_loss": 4.173467909032602, + "tokens_seen": 300875776 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004589969909729187, + "loss": 1.4785, + "theoretical_loss": 4.173362040600166, + "tokens_seen": 300941312 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045898696088264796, + "loss": 1.6896, + "theoretical_loss": 4.173256201673983, + "tokens_seen": 301006848 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045897693079237714, + "loss": 1.6234, + "theoretical_loss": 4.173150392239406, + "tokens_seen": 301072384 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004589669007021063, + "loss": 1.6233, + "theoretical_loss": 4.173044612281802, + "tokens_seen": 301137920 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004589568706118355, + "loss": 1.578, + "theoretical_loss": 4.172938861786546, + "tokens_seen": 301203456 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045894684052156474, + "loss": 1.6248, + "theoretical_loss": 4.1728331407390264, + "tokens_seen": 301268992 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045893681043129386, + "loss": 1.6472, + "theoretical_loss": 4.172727449124636, + "tokens_seen": 301334528 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004589267803410231, + "loss": 1.6347, + "theoretical_loss": 4.172621786928786, + "tokens_seen": 301400064 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 106236, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4996633529663086, + "objective/train/theoretical_loss": 4.17251615413689, + "objective/train/tokens_used": 301465600, + "theoretical_loss": 4.17251615413689, + "tokens_seen": 301465600 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004589167502507522, + "loss": 1.5822, + "theoretical_loss": 4.17251615413689, + "tokens_seen": 301465600 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045890672016048146, + "loss": 1.5137, + "theoretical_loss": 4.172410550734378, + "tokens_seen": 301531136 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045889669007021064, + "loss": 1.5982, + "theoretical_loss": 4.172304976706689, + "tokens_seen": 301596672 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004588866599799398, + "loss": 1.6554, + "theoretical_loss": 4.17219943203927, + "tokens_seen": 301662208 + }, + { + "epoch": 0.09, + "learning_rate": 0.000458876629889669, + "loss": 1.6047, + "theoretical_loss": 4.172093916717581, + "tokens_seen": 301727744 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004588665997993982, + "loss": 1.6032, + "theoretical_loss": 4.171988430727091, + "tokens_seen": 301793280 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045885656970912737, + "loss": 1.5375, + "theoretical_loss": 4.17188297405328, + "tokens_seen": 301858816 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004588465396188566, + "loss": 1.5812, + "theoretical_loss": 4.171777546681638, + "tokens_seen": 301924352 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045883650952858573, + "loss": 1.5445, + "theoretical_loss": 4.171672148597665, + "tokens_seen": 301989888 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045882647943831496, + "loss": 1.6037, + "theoretical_loss": 4.171566779786872, + "tokens_seen": 302055424 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045881644934804415, + "loss": 1.7251, + "theoretical_loss": 4.17146144023478, + "tokens_seen": 302120960 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004588064192577733, + "loss": 1.6356, + "theoretical_loss": 4.171356129926921, + "tokens_seen": 302186496 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004587963891675025, + "loss": 1.5405, + "theoretical_loss": 4.171250848848835, + "tokens_seen": 302252032 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004587863590772317, + "loss": 1.6065, + "theoretical_loss": 4.171145596986076, + "tokens_seen": 302317568 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045877632898696087, + "loss": 1.4761, + "theoretical_loss": 4.171040374324204, + "tokens_seen": 302383104 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004587662988966901, + "loss": 1.5041, + "theoretical_loss": 4.170935180848793, + "tokens_seen": 302448640 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045875626880641923, + "loss": 1.5562, + "theoretical_loss": 4.170830016545425, + "tokens_seen": 302514176 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045874623871614847, + "loss": 1.5549, + "theoretical_loss": 4.1707248813996936, + "tokens_seen": 302579712 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004587362086258776, + "loss": 1.559, + "theoretical_loss": 4.170619775397201, + "tokens_seen": 302645248 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045872617853560683, + "loss": 1.6444, + "theoretical_loss": 4.1705146985235615, + "tokens_seen": 302710784 + }, + { + "epoch": 0.09, + "learning_rate": 0.000458716148445336, + "loss": 1.7365, + "theoretical_loss": 4.170409650764397, + "tokens_seen": 302776320 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004587061183550652, + "loss": 1.7017, + "theoretical_loss": 4.170304632105344, + "tokens_seen": 302841856 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004586960882647944, + "loss": 1.5708, + "theoretical_loss": 4.170199642532045, + "tokens_seen": 302907392 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045868605817452355, + "loss": 1.5768, + "theoretical_loss": 4.170094682030154, + "tokens_seen": 302972928 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004586760280842528, + "loss": 1.5431, + "theoretical_loss": 4.169989750585335, + "tokens_seen": 303038464 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045866599799398197, + "loss": 1.6449, + "theoretical_loss": 4.169884848183264, + "tokens_seen": 303104000 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045865596790371115, + "loss": 1.6794, + "theoretical_loss": 4.169779974809624, + "tokens_seen": 303169536 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045864593781344033, + "loss": 1.6438, + "theoretical_loss": 4.169675130450111, + "tokens_seen": 303235072 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045863590772316957, + "loss": 1.6029, + "theoretical_loss": 4.16957031509043, + "tokens_seen": 303300608 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004586258776328987, + "loss": 1.6815, + "theoretical_loss": 4.169465528716295, + "tokens_seen": 303366144 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045861584754262793, + "loss": 1.6559, + "theoretical_loss": 4.169360771313434, + "tokens_seen": 303431680 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045860581745235706, + "loss": 1.5605, + "theoretical_loss": 4.169256042867579, + "tokens_seen": 303497216 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004585957873620863, + "loss": 1.6774, + "theoretical_loss": 4.169151343364476, + "tokens_seen": 303562752 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004585857572718155, + "loss": 1.5153, + "theoretical_loss": 4.169046672789882, + "tokens_seen": 303628288 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045857572718154465, + "loss": 1.6786, + "theoretical_loss": 4.168942031129562, + "tokens_seen": 303693824 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045856569709127384, + "loss": 1.5553, + "theoretical_loss": 4.168837418369292, + "tokens_seen": 303759360 + }, + { + "epoch": 0.09, + "learning_rate": 0.000458555667001003, + "loss": 1.5343, + "theoretical_loss": 4.168732834494857, + "tokens_seen": 303824896 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004585456369107322, + "loss": 1.6106, + "theoretical_loss": 4.1686282794920535, + "tokens_seen": 303890432 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045853560682046143, + "loss": 1.6411, + "theoretical_loss": 4.168523753346687, + "tokens_seen": 303955968 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045852557673019056, + "loss": 1.6116, + "theoretical_loss": 4.168419256044573, + "tokens_seen": 304021504 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004585155466399198, + "loss": 1.5987, + "theoretical_loss": 4.168314787571538, + "tokens_seen": 304087040 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004585055165496489, + "loss": 1.498, + "theoretical_loss": 4.168210347913418, + "tokens_seen": 304152576 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045849548645937816, + "loss": 1.632, + "theoretical_loss": 4.168105937056059, + "tokens_seen": 304218112 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045848545636910734, + "loss": 1.5802, + "theoretical_loss": 4.168001554985317, + "tokens_seen": 304283648 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004584754262788365, + "loss": 1.6754, + "theoretical_loss": 4.167897201687056, + "tokens_seen": 304349184 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004584653961885657, + "loss": 1.7443, + "theoretical_loss": 4.167792877147155, + "tokens_seen": 304414720 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045845536609829494, + "loss": 1.5971, + "theoretical_loss": 4.1676885813514986, + "tokens_seen": 304480256 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045844533600802406, + "loss": 1.5311, + "theoretical_loss": 4.167584314285982, + "tokens_seen": 304545792 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004584353059177533, + "loss": 1.6616, + "theoretical_loss": 4.167480075936512, + "tokens_seen": 304611328 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004584252758274824, + "loss": 1.5663, + "theoretical_loss": 4.167375866289004, + "tokens_seen": 304676864 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 107151, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6123368740081787, + "objective/train/theoretical_loss": 4.167271685329383, + "objective/train/tokens_used": 304742400, + "theoretical_loss": 4.167271685329383, + "tokens_seen": 304742400 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045841524573721166, + "loss": 1.7085, + "theoretical_loss": 4.167271685329383, + "tokens_seen": 304742400 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045840521564694084, + "loss": 1.7307, + "theoretical_loss": 4.167167533043585, + "tokens_seen": 304807936 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045839518555667, + "loss": 1.5368, + "theoretical_loss": 4.1670634094175565, + "tokens_seen": 304873472 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004583851554663992, + "loss": 1.4476, + "theoretical_loss": 4.166959314437252, + "tokens_seen": 304939008 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004583751253761284, + "loss": 1.5578, + "theoretical_loss": 4.166855248088637, + "tokens_seen": 305004544 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045836509528585757, + "loss": 1.519, + "theoretical_loss": 4.1667512103576865, + "tokens_seen": 305070080 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004583550651955868, + "loss": 1.625, + "theoretical_loss": 4.166647201230386, + "tokens_seen": 305135616 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045834503510531593, + "loss": 1.6336, + "theoretical_loss": 4.166543220692731, + "tokens_seen": 305201152 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045833500501504516, + "loss": 1.5645, + "theoretical_loss": 4.166439268730724, + "tokens_seen": 305266688 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045832497492477435, + "loss": 1.5445, + "theoretical_loss": 4.166335345330383, + "tokens_seen": 305332224 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004583149448345035, + "loss": 1.6446, + "theoretical_loss": 4.16623145047773, + "tokens_seen": 305397760 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004583049147442327, + "loss": 1.7235, + "theoretical_loss": 4.166127584158801, + "tokens_seen": 305463296 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004582948846539619, + "loss": 1.639, + "theoretical_loss": 4.166023746359639, + "tokens_seen": 305528832 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045828485456369107, + "loss": 1.6627, + "theoretical_loss": 4.165919937066299, + "tokens_seen": 305594368 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004582748244734203, + "loss": 1.6253, + "theoretical_loss": 4.165816156264844, + "tokens_seen": 305659904 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045826479438314943, + "loss": 1.6594, + "theoretical_loss": 4.165712403941348, + "tokens_seen": 305725440 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045825476429287867, + "loss": 1.6366, + "theoretical_loss": 4.1656086800818946, + "tokens_seen": 305790976 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004582447342026078, + "loss": 1.5074, + "theoretical_loss": 4.165504984672577, + "tokens_seen": 305856512 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045823470411233703, + "loss": 1.6084, + "theoretical_loss": 4.165401317699498, + "tokens_seen": 305922048 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004582246740220662, + "loss": 1.4893, + "theoretical_loss": 4.16529767914877, + "tokens_seen": 305987584 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004582146439317954, + "loss": 1.5691, + "theoretical_loss": 4.165194069006516, + "tokens_seen": 306053120 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045820461384152457, + "loss": 1.5459, + "theoretical_loss": 4.165090487258867, + "tokens_seen": 306118656 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045819458375125375, + "loss": 1.6705, + "theoretical_loss": 4.164986933891968, + "tokens_seen": 306184192 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045818455366098294, + "loss": 1.6205, + "theoretical_loss": 4.164883408891968, + "tokens_seen": 306249728 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045817452357071217, + "loss": 1.5719, + "theoretical_loss": 4.164779912245029, + "tokens_seen": 306315264 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004581644934804413, + "loss": 1.558, + "theoretical_loss": 4.1646764439373225, + "tokens_seen": 306380800 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045815446339017053, + "loss": 1.522, + "theoretical_loss": 4.164573003955029, + "tokens_seen": 306446336 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004581444332998997, + "loss": 1.5859, + "theoretical_loss": 4.164469592284338, + "tokens_seen": 306511872 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004581344032096289, + "loss": 1.6152, + "theoretical_loss": 4.164366208911453, + "tokens_seen": 306577408 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004581243731193581, + "loss": 1.5544, + "theoretical_loss": 4.1642628538225805, + "tokens_seen": 306642944 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045811434302908726, + "loss": 1.6008, + "theoretical_loss": 4.1641595270039415, + "tokens_seen": 306708480 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045810431293881644, + "loss": 1.5329, + "theoretical_loss": 4.1640562284417655, + "tokens_seen": 306774016 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004580942828485457, + "loss": 1.6386, + "theoretical_loss": 4.16395295812229, + "tokens_seen": 306839552 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004580842527582748, + "loss": 1.5785, + "theoretical_loss": 4.1638497160317645, + "tokens_seen": 306905088 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045807422266800404, + "loss": 1.5413, + "theoretical_loss": 4.163746502156448, + "tokens_seen": 306970624 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045806419257773316, + "loss": 1.5834, + "theoretical_loss": 4.163643316482606, + "tokens_seen": 307036160 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004580541624874624, + "loss": 1.4695, + "theoretical_loss": 4.1635401589965175, + "tokens_seen": 307101696 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004580441323971916, + "loss": 1.5363, + "theoretical_loss": 4.163437029684469, + "tokens_seen": 307167232 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045803410230692076, + "loss": 1.5388, + "theoretical_loss": 4.163333928532758, + "tokens_seen": 307232768 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045802407221664994, + "loss": 1.5821, + "theoretical_loss": 4.163230855527688, + "tokens_seen": 307298304 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004580140421263791, + "loss": 1.4544, + "theoretical_loss": 4.163127810655578, + "tokens_seen": 307363840 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004580040120361083, + "loss": 1.6742, + "theoretical_loss": 4.163024793902752, + "tokens_seen": 307429376 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045799398194583754, + "loss": 1.5014, + "theoretical_loss": 4.162921805255543, + "tokens_seen": 307494912 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045798395185556667, + "loss": 1.6501, + "theoretical_loss": 4.162818844700298, + "tokens_seen": 307560448 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004579739217652959, + "loss": 1.6495, + "theoretical_loss": 4.162715912223369, + "tokens_seen": 307625984 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004579638916750251, + "loss": 1.5016, + "theoretical_loss": 4.162613007811122, + "tokens_seen": 307691520 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045795386158475426, + "loss": 1.5809, + "theoretical_loss": 4.1625101314499275, + "tokens_seen": 307757056 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004579438314944835, + "loss": 1.6127, + "theoretical_loss": 4.162407283126169, + "tokens_seen": 307822592 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004579338014042126, + "loss": 1.7351, + "theoretical_loss": 4.1623044628262384, + "tokens_seen": 307888128 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045792377131394186, + "loss": 1.639, + "theoretical_loss": 4.162201670536537, + "tokens_seen": 307953664 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 108409, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5380980968475342, + "objective/train/theoretical_loss": 4.162098906243477, + "objective/train/tokens_used": 308019200, + "theoretical_loss": 4.162098906243477, + "tokens_seen": 308019200 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045791374122367104, + "loss": 1.5746, + "theoretical_loss": 4.162098906243477, + "tokens_seen": 308019200 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004579037111334002, + "loss": 1.5021, + "theoretical_loss": 4.161996169933477, + "tokens_seen": 308084736 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004578936810431294, + "loss": 1.7819, + "theoretical_loss": 4.161893461592968, + "tokens_seen": 308150272 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004578836509528586, + "loss": 1.533, + "theoretical_loss": 4.16179078120839, + "tokens_seen": 308215808 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045787362086258777, + "loss": 1.588, + "theoretical_loss": 4.161688128766191, + "tokens_seen": 308281344 + }, + { + "epoch": 0.09, + "learning_rate": 0.000457863590772317, + "loss": 1.5713, + "theoretical_loss": 4.16158550425283, + "tokens_seen": 308346880 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045785356068204613, + "loss": 1.5907, + "theoretical_loss": 4.161482907654775, + "tokens_seen": 308412416 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045784353059177536, + "loss": 1.5552, + "theoretical_loss": 4.161380338958502, + "tokens_seen": 308477952 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045783350050150455, + "loss": 1.4718, + "theoretical_loss": 4.161277798150498, + "tokens_seen": 308543488 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004578234704112337, + "loss": 1.5997, + "theoretical_loss": 4.16117528521726, + "tokens_seen": 308609024 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004578134403209629, + "loss": 1.5803, + "theoretical_loss": 4.161072800145292, + "tokens_seen": 308674560 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004578034102306921, + "loss": 1.5899, + "theoretical_loss": 4.16097034292111, + "tokens_seen": 308740096 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045779338014042127, + "loss": 1.5637, + "theoretical_loss": 4.160867913531238, + "tokens_seen": 308805632 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004577833500501505, + "loss": 1.621, + "theoretical_loss": 4.160765511962209, + "tokens_seen": 308871168 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045777331995987963, + "loss": 1.5759, + "theoretical_loss": 4.160663138200567, + "tokens_seen": 308936704 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045776328986960887, + "loss": 1.705, + "theoretical_loss": 4.1605607922328645, + "tokens_seen": 309002240 + }, + { + "epoch": 0.09, + "learning_rate": 0.000457753259779338, + "loss": 1.6578, + "theoretical_loss": 4.160458474045662, + "tokens_seen": 309067776 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045774322968906723, + "loss": 1.5695, + "theoretical_loss": 4.16035618362553, + "tokens_seen": 309133312 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004577331995987964, + "loss": 1.7353, + "theoretical_loss": 4.16025392095905, + "tokens_seen": 309198848 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004577231695085256, + "loss": 1.744, + "theoretical_loss": 4.160151686032814, + "tokens_seen": 309264384 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004577131394182548, + "loss": 1.5005, + "theoretical_loss": 4.160049478833416, + "tokens_seen": 309329920 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045770310932798395, + "loss": 1.7399, + "theoretical_loss": 4.159947299347468, + "tokens_seen": 309395456 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045769307923771314, + "loss": 1.5958, + "theoretical_loss": 4.159845147561587, + "tokens_seen": 309460992 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045768304914744237, + "loss": 1.6055, + "theoretical_loss": 4.1597430234624, + "tokens_seen": 309526528 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004576730190571715, + "loss": 1.5653, + "theoretical_loss": 4.159640927036541, + "tokens_seen": 309592064 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045766298896690073, + "loss": 1.6591, + "theoretical_loss": 4.159538858270659, + "tokens_seen": 309657600 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004576529588766299, + "loss": 1.6972, + "theoretical_loss": 4.159436817151407, + "tokens_seen": 309723136 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004576429287863591, + "loss": 1.6833, + "theoretical_loss": 4.159334803665448, + "tokens_seen": 309788672 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004576328986960883, + "loss": 1.5823, + "theoretical_loss": 4.159232817799458, + "tokens_seen": 309854208 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045762286860581746, + "loss": 1.6655, + "theoretical_loss": 4.1591308595401175, + "tokens_seen": 309919744 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045761283851554664, + "loss": 1.6146, + "theoretical_loss": 4.159028928874118, + "tokens_seen": 309985280 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004576028084252759, + "loss": 1.7017, + "theoretical_loss": 4.158927025788162, + "tokens_seen": 310050816 + }, + { + "epoch": 0.09, + "learning_rate": 0.000457592778335005, + "loss": 1.6406, + "theoretical_loss": 4.158825150268958, + "tokens_seen": 310116352 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045758274824473424, + "loss": 1.6638, + "theoretical_loss": 4.158723302303227, + "tokens_seen": 310181888 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045757271815446336, + "loss": 1.4546, + "theoretical_loss": 4.158621481877697, + "tokens_seen": 310247424 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004575626880641926, + "loss": 1.5575, + "theoretical_loss": 4.158519688979106, + "tokens_seen": 310312960 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004575526579739218, + "loss": 1.5607, + "theoretical_loss": 4.1584179235942, + "tokens_seen": 310378496 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045754262788365096, + "loss": 1.5935, + "theoretical_loss": 4.158316185709737, + "tokens_seen": 310444032 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045753259779338014, + "loss": 1.6562, + "theoretical_loss": 4.158214475312481, + "tokens_seen": 310509568 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004575225677031093, + "loss": 1.5794, + "theoretical_loss": 4.158112792389206, + "tokens_seen": 310575104 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004575125376128385, + "loss": 1.5438, + "theoretical_loss": 4.158011136926698, + "tokens_seen": 310640640 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045750250752256774, + "loss": 1.5299, + "theoretical_loss": 4.157909508911747, + "tokens_seen": 310706176 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045749247743229687, + "loss": 1.5483, + "theoretical_loss": 4.157807908331157, + "tokens_seen": 310771712 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004574824473420261, + "loss": 1.7311, + "theoretical_loss": 4.157706335171738, + "tokens_seen": 310837248 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004574724172517553, + "loss": 1.6625, + "theoretical_loss": 4.15760478942031, + "tokens_seen": 310902784 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045746238716148446, + "loss": 1.6129, + "theoretical_loss": 4.157503271063703, + "tokens_seen": 310968320 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045745235707121364, + "loss": 1.5925, + "theoretical_loss": 4.157401780088756, + "tokens_seen": 311033856 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004574423269809428, + "loss": 1.5276, + "theoretical_loss": 4.157300316482315, + "tokens_seen": 311099392 + }, + { + "epoch": 0.09, + "learning_rate": 0.000457432296890672, + "loss": 1.5898, + "theoretical_loss": 4.157198880231238, + "tokens_seen": 311164928 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045742226680040124, + "loss": 1.562, + "theoretical_loss": 4.157097471322389, + "tokens_seen": 311230464 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 109428, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.718328833580017, + "objective/train/theoretical_loss": 4.156996089742645, + "objective/train/tokens_used": 311296000, + "theoretical_loss": 4.156996089742645, + "tokens_seen": 311296000 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045741223671013037, + "loss": 1.6672, + "theoretical_loss": 4.156996089742645, + "tokens_seen": 311296000 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004574022066198596, + "loss": 1.6956, + "theoretical_loss": 4.156894735478888, + "tokens_seen": 311361536 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045739217652958873, + "loss": 1.633, + "theoretical_loss": 4.156793408518011, + "tokens_seen": 311427072 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045738214643931797, + "loss": 1.5541, + "theoretical_loss": 4.156692108846916, + "tokens_seen": 311492608 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045737211634904715, + "loss": 1.5665, + "theoretical_loss": 4.156590836452514, + "tokens_seen": 311558144 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045736208625877633, + "loss": 1.642, + "theoretical_loss": 4.156489591321726, + "tokens_seen": 311623680 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004573520561685055, + "loss": 1.6292, + "theoretical_loss": 4.15638837344148, + "tokens_seen": 311689216 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045734202607823475, + "loss": 1.7193, + "theoretical_loss": 4.156287182798714, + "tokens_seen": 311754752 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045733199598796387, + "loss": 1.6637, + "theoretical_loss": 4.156186019380375, + "tokens_seen": 311820288 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004573219658976931, + "loss": 1.6057, + "theoretical_loss": 4.156084883173419, + "tokens_seen": 311885824 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045731193580742223, + "loss": 1.6329, + "theoretical_loss": 4.155983774164811, + "tokens_seen": 311951360 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045730190571715147, + "loss": 1.5144, + "theoretical_loss": 4.1558826923415255, + "tokens_seen": 312016896 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045729187562688065, + "loss": 1.5596, + "theoretical_loss": 4.155781637690545, + "tokens_seen": 312082432 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045728184553660983, + "loss": 1.6617, + "theoretical_loss": 4.1556806101988615, + "tokens_seen": 312147968 + }, + { + "epoch": 0.09, + "learning_rate": 0.000457271815446339, + "loss": 1.5815, + "theoretical_loss": 4.155579609853476, + "tokens_seen": 312213504 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004572617853560682, + "loss": 1.6225, + "theoretical_loss": 4.155478636641398, + "tokens_seen": 312279040 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004572517552657974, + "loss": 1.7392, + "theoretical_loss": 4.155377690549646, + "tokens_seen": 312344576 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004572417251755266, + "loss": 1.5631, + "theoretical_loss": 4.155276771565248, + "tokens_seen": 312410112 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045723169508525574, + "loss": 1.5772, + "theoretical_loss": 4.155175879675241, + "tokens_seen": 312475648 + }, + { + "epoch": 0.09, + "learning_rate": 0.000457221664994985, + "loss": 1.5251, + "theoretical_loss": 4.155075014866671, + "tokens_seen": 312541184 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004572116349047141, + "loss": 1.6629, + "theoretical_loss": 4.15497417712659, + "tokens_seen": 312606720 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045720160481444334, + "loss": 1.5268, + "theoretical_loss": 4.1548733664420645, + "tokens_seen": 312672256 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045719157472417257, + "loss": 1.5426, + "theoretical_loss": 4.154772582800165, + "tokens_seen": 312737792 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004571815446339017, + "loss": 1.5388, + "theoretical_loss": 4.154671826187972, + "tokens_seen": 312803328 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045717151454363093, + "loss": 1.5521, + "theoretical_loss": 4.154571096592576, + "tokens_seen": 312868864 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004571614844533601, + "loss": 1.6783, + "theoretical_loss": 4.154470394001077, + "tokens_seen": 312934400 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004571514543630893, + "loss": 1.6928, + "theoretical_loss": 4.154369718400583, + "tokens_seen": 312999936 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004571414242728185, + "loss": 1.61, + "theoretical_loss": 4.154269069778207, + "tokens_seen": 313065472 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045713139418254766, + "loss": 1.677, + "theoretical_loss": 4.154168448121078, + "tokens_seen": 313131008 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045712136409227684, + "loss": 1.675, + "theoretical_loss": 4.15406785341633, + "tokens_seen": 313196544 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004571113340020061, + "loss": 1.5521, + "theoretical_loss": 4.153967285651105, + "tokens_seen": 313262080 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004571013039117352, + "loss": 1.5706, + "theoretical_loss": 4.153866744812555, + "tokens_seen": 313327616 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045709127382146444, + "loss": 1.4949, + "theoretical_loss": 4.1537662308878405, + "tokens_seen": 313393152 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045708124373119356, + "loss": 1.6482, + "theoretical_loss": 4.153665743864131, + "tokens_seen": 313458688 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004570712136409228, + "loss": 1.5832, + "theoretical_loss": 4.153565283728606, + "tokens_seen": 313524224 + }, + { + "epoch": 0.1, + "learning_rate": 0.000457061183550652, + "loss": 1.6154, + "theoretical_loss": 4.153464850468452, + "tokens_seen": 313589760 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045705115346038116, + "loss": 1.6242, + "theoretical_loss": 4.1533644440708635, + "tokens_seen": 313655296 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045704112337011034, + "loss": 1.5647, + "theoretical_loss": 4.153264064523047, + "tokens_seen": 313720832 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004570310932798395, + "loss": 1.5444, + "theoretical_loss": 4.1531637118122156, + "tokens_seen": 313786368 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004570210631895687, + "loss": 1.5413, + "theoretical_loss": 4.153063385925591, + "tokens_seen": 313851904 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045701103309929794, + "loss": 1.5842, + "theoretical_loss": 4.152963086850405, + "tokens_seen": 313917440 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045700100300902707, + "loss": 1.4471, + "theoretical_loss": 4.152862814573895, + "tokens_seen": 313982976 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004569909729187563, + "loss": 1.6538, + "theoretical_loss": 4.152762569083313, + "tokens_seen": 314048512 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004569809428284855, + "loss": 1.5339, + "theoretical_loss": 4.1526623503659135, + "tokens_seen": 314114048 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045697091273821466, + "loss": 1.6019, + "theoretical_loss": 4.152562158408962, + "tokens_seen": 314179584 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045696088264794384, + "loss": 1.5978, + "theoretical_loss": 4.1524619931997355, + "tokens_seen": 314245120 + }, + { + "epoch": 0.1, + "learning_rate": 0.000456950852557673, + "loss": 1.5974, + "theoretical_loss": 4.152361854725515, + "tokens_seen": 314310656 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004569408224674022, + "loss": 1.6388, + "theoretical_loss": 4.152261742973594, + "tokens_seen": 314376192 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045693079237713144, + "loss": 1.6724, + "theoretical_loss": 4.152161657931273, + "tokens_seen": 314441728 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045692076228686057, + "loss": 1.5435, + "theoretical_loss": 4.15206159958586, + "tokens_seen": 314507264 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 110660, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6904749870300293, + "objective/train/theoretical_loss": 4.1519615679246735, + "objective/train/tokens_used": 314572800, + "theoretical_loss": 4.1519615679246735, + "tokens_seen": 314572800 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004569107321965898, + "loss": 1.5965, + "theoretical_loss": 4.1519615679246735, + "tokens_seen": 314572800 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045690070210631893, + "loss": 1.5865, + "theoretical_loss": 4.151861562935041, + "tokens_seen": 314638336 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045689067201604817, + "loss": 1.7151, + "theoretical_loss": 4.151761584604298, + "tokens_seen": 314703872 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045688064192577735, + "loss": 1.5765, + "theoretical_loss": 4.1516616329197875, + "tokens_seen": 314769408 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045687061183550653, + "loss": 1.6143, + "theoretical_loss": 4.151561707868863, + "tokens_seen": 314834944 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004568605817452357, + "loss": 1.4574, + "theoretical_loss": 4.151461809438885, + "tokens_seen": 314900480 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045685055165496495, + "loss": 1.591, + "theoretical_loss": 4.1513619376172235, + "tokens_seen": 314966016 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045684052156469407, + "loss": 1.6228, + "theoretical_loss": 4.151262092391257, + "tokens_seen": 315031552 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004568304914744233, + "loss": 1.6141, + "theoretical_loss": 4.1511622737483735, + "tokens_seen": 315097088 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045682046138415243, + "loss": 1.4752, + "theoretical_loss": 4.151062481675968, + "tokens_seen": 315162624 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045681043129388167, + "loss": 1.6372, + "theoretical_loss": 4.150962716161445, + "tokens_seen": 315228160 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045680040120361085, + "loss": 1.5283, + "theoretical_loss": 4.150862977192217, + "tokens_seen": 315293696 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045679037111334003, + "loss": 1.5174, + "theoretical_loss": 4.150763264755706, + "tokens_seen": 315359232 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004567803410230692, + "loss": 1.6067, + "theoretical_loss": 4.150663578839342, + "tokens_seen": 315424768 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004567703109327984, + "loss": 1.5966, + "theoretical_loss": 4.150563919430562, + "tokens_seen": 315490304 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004567602808425276, + "loss": 1.672, + "theoretical_loss": 4.150464286516816, + "tokens_seen": 315555840 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004567502507522568, + "loss": 1.6206, + "theoretical_loss": 4.150364680085558, + "tokens_seen": 315621376 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045674022066198594, + "loss": 1.5614, + "theoretical_loss": 4.150265100124253, + "tokens_seen": 315686912 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004567301905717152, + "loss": 1.7087, + "theoretical_loss": 4.150165546620373, + "tokens_seen": 315752448 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004567201604814443, + "loss": 1.5499, + "theoretical_loss": 4.1500660195614, + "tokens_seen": 315817984 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045671013039117354, + "loss": 1.6101, + "theoretical_loss": 4.149966518934824, + "tokens_seen": 315883520 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004567001003009027, + "loss": 1.8209, + "theoretical_loss": 4.149867044728143, + "tokens_seen": 315949056 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004566900702106319, + "loss": 1.5531, + "theoretical_loss": 4.149767596928863, + "tokens_seen": 316014592 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004566800401203611, + "loss": 1.5797, + "theoretical_loss": 4.149668175524502, + "tokens_seen": 316080128 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004566700100300903, + "loss": 1.6885, + "theoretical_loss": 4.14956878050258, + "tokens_seen": 316145664 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045665997993981944, + "loss": 1.6763, + "theoretical_loss": 4.1494694118506334, + "tokens_seen": 316211200 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004566499498495487, + "loss": 1.5787, + "theoretical_loss": 4.1493700695562, + "tokens_seen": 316276736 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004566399197592778, + "loss": 1.6065, + "theoretical_loss": 4.14927075360683, + "tokens_seen": 316342272 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045662988966900704, + "loss": 1.552, + "theoretical_loss": 4.149171463990082, + "tokens_seen": 316407808 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004566198595787362, + "loss": 1.5938, + "theoretical_loss": 4.149072200693521, + "tokens_seen": 316473344 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004566098294884654, + "loss": 1.7397, + "theoretical_loss": 4.148972963704722, + "tokens_seen": 316538880 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004565997993981946, + "loss": 1.498, + "theoretical_loss": 4.148873753011269, + "tokens_seen": 316604416 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045658976930792376, + "loss": 1.6235, + "theoretical_loss": 4.148774568600752, + "tokens_seen": 316669952 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045657973921765294, + "loss": 1.5424, + "theoretical_loss": 4.148675410460772, + "tokens_seen": 316735488 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004565697091273822, + "loss": 1.6009, + "theoretical_loss": 4.148576278578937, + "tokens_seen": 316801024 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004565596790371113, + "loss": 1.6267, + "theoretical_loss": 4.148477172942863, + "tokens_seen": 316866560 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045654964894684054, + "loss": 1.528, + "theoretical_loss": 4.1483780935401775, + "tokens_seen": 316932096 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045653961885656967, + "loss": 1.8455, + "theoretical_loss": 4.148279040358511, + "tokens_seen": 316997632 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004565295887662989, + "loss": 1.6213, + "theoretical_loss": 4.148180013385507, + "tokens_seen": 317063168 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004565195586760281, + "loss": 1.6746, + "theoretical_loss": 4.148081012608817, + "tokens_seen": 317128704 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045650952858575727, + "loss": 1.5926, + "theoretical_loss": 4.147982038016096, + "tokens_seen": 317194240 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045649949849548645, + "loss": 1.7918, + "theoretical_loss": 4.147883089595015, + "tokens_seen": 317259776 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004564894684052157, + "loss": 1.5086, + "theoretical_loss": 4.147784167333247, + "tokens_seen": 317325312 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004564794383149448, + "loss": 1.5699, + "theoretical_loss": 4.147685271218477, + "tokens_seen": 317390848 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045646940822467405, + "loss": 1.6792, + "theoretical_loss": 4.147586401238396, + "tokens_seen": 317456384 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045645937813440317, + "loss": 1.5978, + "theoretical_loss": 4.147487557380705, + "tokens_seen": 317521920 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004564493480441324, + "loss": 1.6115, + "theoretical_loss": 4.147388739633113, + "tokens_seen": 317587456 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045643931795386164, + "loss": 1.7498, + "theoretical_loss": 4.147289947983337, + "tokens_seen": 317652992 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045642928786359077, + "loss": 1.5678, + "theoretical_loss": 4.1471911824191015, + "tokens_seen": 317718528 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045641925777332, + "loss": 1.6398, + "theoretical_loss": 4.147092442928141, + "tokens_seen": 317784064 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 111554, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6084914207458496, + "objective/train/theoretical_loss": 4.146993729498197, + "objective/train/tokens_used": 317849600, + "theoretical_loss": 4.146993729498197, + "tokens_seen": 317849600 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045640922768304913, + "loss": 1.6106, + "theoretical_loss": 4.146993729498197, + "tokens_seen": 317849600 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045639919759277837, + "loss": 1.6349, + "theoretical_loss": 4.146895042117021, + "tokens_seen": 317915136 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045638916750250755, + "loss": 1.56, + "theoretical_loss": 4.146796380772369, + "tokens_seen": 317980672 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045637913741223673, + "loss": 1.57, + "theoretical_loss": 4.146697745452011, + "tokens_seen": 318046208 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004563691073219659, + "loss": 1.6006, + "theoretical_loss": 4.146599136143719, + "tokens_seen": 318111744 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045635907723169515, + "loss": 1.6107, + "theoretical_loss": 4.146500552835278, + "tokens_seen": 318177280 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045634904714142427, + "loss": 1.7911, + "theoretical_loss": 4.146401995514479, + "tokens_seen": 318242816 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004563390170511535, + "loss": 1.7416, + "theoretical_loss": 4.146303464169123, + "tokens_seen": 318308352 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045632898696088263, + "loss": 1.8469, + "theoretical_loss": 4.146204958787017, + "tokens_seen": 318373888 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045631895687061187, + "loss": 1.7508, + "theoretical_loss": 4.146106479355978, + "tokens_seen": 318439424 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045630892678034105, + "loss": 1.8755, + "theoretical_loss": 4.1460080258638285, + "tokens_seen": 318504960 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045629889669007023, + "loss": 1.671, + "theoretical_loss": 4.145909598298404, + "tokens_seen": 318570496 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004562888665997994, + "loss": 1.7213, + "theoretical_loss": 4.145811196647544, + "tokens_seen": 318636032 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004562788365095286, + "loss": 1.8306, + "theoretical_loss": 4.145712820899098, + "tokens_seen": 318701568 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004562688064192578, + "loss": 1.7508, + "theoretical_loss": 4.145614471040923, + "tokens_seen": 318767104 + }, + { + "epoch": 0.1, + "learning_rate": 0.000456258776328987, + "loss": 1.9216, + "theoretical_loss": 4.145516147060884, + "tokens_seen": 318832640 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045624874623871614, + "loss": 1.7113, + "theoretical_loss": 4.145417848946857, + "tokens_seen": 318898176 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004562387161484454, + "loss": 1.7214, + "theoretical_loss": 4.145319576686721, + "tokens_seen": 318963712 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004562286860581745, + "loss": 1.7344, + "theoretical_loss": 4.1452213302683685, + "tokens_seen": 319029248 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045621865596790374, + "loss": 1.7218, + "theoretical_loss": 4.145123109679696, + "tokens_seen": 319094784 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004562086258776329, + "loss": 1.8017, + "theoretical_loss": 4.1450249149086105, + "tokens_seen": 319160320 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004561985957873621, + "loss": 1.7483, + "theoretical_loss": 4.144926745943026, + "tokens_seen": 319225856 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004561885656970913, + "loss": 1.6548, + "theoretical_loss": 4.1448286027708665, + "tokens_seen": 319291392 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004561785356068205, + "loss": 1.7032, + "theoretical_loss": 4.144730485380062, + "tokens_seen": 319356928 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045616850551654964, + "loss": 1.7371, + "theoretical_loss": 4.144632393758551, + "tokens_seen": 319422464 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004561584754262789, + "loss": 1.6246, + "theoretical_loss": 4.144534327894281, + "tokens_seen": 319488000 + }, + { + "epoch": 0.1, + "learning_rate": 0.000456148445336008, + "loss": 1.6343, + "theoretical_loss": 4.144436287775206, + "tokens_seen": 319553536 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045613841524573724, + "loss": 1.7076, + "theoretical_loss": 4.144338273389291, + "tokens_seen": 319619072 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004561283851554664, + "loss": 1.7891, + "theoretical_loss": 4.144240284724505, + "tokens_seen": 319684608 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004561183550651956, + "loss": 1.6602, + "theoretical_loss": 4.144142321768831, + "tokens_seen": 319750144 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004561083249749248, + "loss": 1.7408, + "theoretical_loss": 4.144044384510253, + "tokens_seen": 319815680 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045609829488465396, + "loss": 1.7387, + "theoretical_loss": 4.1439464729367685, + "tokens_seen": 319881216 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045608826479438314, + "loss": 1.6257, + "theoretical_loss": 4.14384858703638, + "tokens_seen": 319946752 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004560782347041124, + "loss": 1.6989, + "theoretical_loss": 4.1437507267971, + "tokens_seen": 320012288 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004560682046138415, + "loss": 1.7421, + "theoretical_loss": 4.143652892206948, + "tokens_seen": 320077824 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045605817452357074, + "loss": 1.7041, + "theoretical_loss": 4.1435550832539505, + "tokens_seen": 320143360 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045604814443329987, + "loss": 1.6031, + "theoretical_loss": 4.143457299926146, + "tokens_seen": 320208896 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004560381143430291, + "loss": 1.6285, + "theoretical_loss": 4.143359542211576, + "tokens_seen": 320274432 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004560280842527583, + "loss": 1.8292, + "theoretical_loss": 4.143261810098293, + "tokens_seen": 320339968 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045601805416248747, + "loss": 1.6729, + "theoretical_loss": 4.1431641035743585, + "tokens_seen": 320405504 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045600802407221665, + "loss": 1.6566, + "theoretical_loss": 4.143066422627838, + "tokens_seen": 320471040 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004559979939819459, + "loss": 1.5926, + "theoretical_loss": 4.142968767246808, + "tokens_seen": 320536576 + }, + { + "epoch": 0.1, + "learning_rate": 0.000455987963891675, + "loss": 1.6242, + "theoretical_loss": 4.142871137419354, + "tokens_seen": 320602112 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045597793380140425, + "loss": 1.6901, + "theoretical_loss": 4.142773533133565, + "tokens_seen": 320667648 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045596790371113337, + "loss": 1.702, + "theoretical_loss": 4.142675954377543, + "tokens_seen": 320733184 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004559578736208626, + "loss": 1.676, + "theoretical_loss": 4.142578401139396, + "tokens_seen": 320798720 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004559478435305918, + "loss": 1.6491, + "theoretical_loss": 4.142480873407239, + "tokens_seen": 320864256 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045593781344032097, + "loss": 1.7252, + "theoretical_loss": 4.1423833711691955, + "tokens_seen": 320929792 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045592778335005015, + "loss": 1.6146, + "theoretical_loss": 4.142285894413398, + "tokens_seen": 320995328 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045591775325977933, + "loss": 1.661, + "theoretical_loss": 4.142188443127985, + "tokens_seen": 321060864 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 112831, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6132713556289673, + "objective/train/theoretical_loss": 4.142091017301105, + "objective/train/tokens_used": 321126400, + "theoretical_loss": 4.142091017301105, + "tokens_seen": 321126400 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004559077231695085, + "loss": 1.5752, + "theoretical_loss": 4.142091017301105, + "tokens_seen": 321126400 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045589769307923775, + "loss": 1.5947, + "theoretical_loss": 4.141993616920914, + "tokens_seen": 321191936 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004558876629889669, + "loss": 1.5956, + "theoretical_loss": 4.141896241975575, + "tokens_seen": 321257472 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004558776328986961, + "loss": 1.6384, + "theoretical_loss": 4.141798892453259, + "tokens_seen": 321323008 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045586760280842524, + "loss": 1.6168, + "theoretical_loss": 4.141701568342145, + "tokens_seen": 321388544 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004558575727181545, + "loss": 1.7069, + "theoretical_loss": 4.141604269630422, + "tokens_seen": 321454080 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045584754262788365, + "loss": 1.6556, + "theoretical_loss": 4.1415069963062825, + "tokens_seen": 321519616 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045583751253761284, + "loss": 1.6, + "theoretical_loss": 4.1414097483579315, + "tokens_seen": 321585152 + }, + { + "epoch": 0.1, + "learning_rate": 0.000455827482447342, + "loss": 1.66, + "theoretical_loss": 4.1413125257735794, + "tokens_seen": 321650688 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045581745235707125, + "loss": 1.7011, + "theoretical_loss": 4.141215328541445, + "tokens_seen": 321716224 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004558074222668004, + "loss": 1.7042, + "theoretical_loss": 4.141118156649753, + "tokens_seen": 321781760 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004557973921765296, + "loss": 1.6052, + "theoretical_loss": 4.1410210100867415, + "tokens_seen": 321847296 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045578736208625874, + "loss": 1.5521, + "theoretical_loss": 4.14092388884065, + "tokens_seen": 321912832 + }, + { + "epoch": 0.1, + "learning_rate": 0.000455777331995988, + "loss": 1.5363, + "theoretical_loss": 4.1408267928997295, + "tokens_seen": 321978368 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045576730190571716, + "loss": 1.686, + "theoretical_loss": 4.140729722252239, + "tokens_seen": 322043904 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045575727181544634, + "loss": 1.5892, + "theoretical_loss": 4.140632676886442, + "tokens_seen": 322109440 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004557472417251755, + "loss": 1.6375, + "theoretical_loss": 4.140535656790615, + "tokens_seen": 322174976 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004557372116349047, + "loss": 1.6282, + "theoretical_loss": 4.140438661953038, + "tokens_seen": 322240512 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004557271815446339, + "loss": 1.5546, + "theoretical_loss": 4.140341692362, + "tokens_seen": 322306048 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004557171514543631, + "loss": 1.6854, + "theoretical_loss": 4.140244748005799, + "tokens_seen": 322371584 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045570712136409224, + "loss": 1.5675, + "theoretical_loss": 4.14014782887274, + "tokens_seen": 322437120 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004556970912738215, + "loss": 1.5499, + "theoretical_loss": 4.140050934951135, + "tokens_seen": 322502656 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004556870611835507, + "loss": 1.775, + "theoretical_loss": 4.139954066229304, + "tokens_seen": 322568192 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045567703109327984, + "loss": 1.6147, + "theoretical_loss": 4.139857222695578, + "tokens_seen": 322633728 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004556670010030091, + "loss": 1.7651, + "theoretical_loss": 4.139760404338291, + "tokens_seen": 322699264 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004556569709127382, + "loss": 1.6329, + "theoretical_loss": 4.139663611145786, + "tokens_seen": 322764800 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045564694082246744, + "loss": 1.6526, + "theoretical_loss": 4.139566843106417, + "tokens_seen": 322830336 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004556369107321966, + "loss": 1.5711, + "theoretical_loss": 4.139470100208542, + "tokens_seen": 322895872 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004556268806419258, + "loss": 1.7093, + "theoretical_loss": 4.139373382440528, + "tokens_seen": 322961408 + }, + { + "epoch": 0.1, + "learning_rate": 0.000455616850551655, + "loss": 1.7118, + "theoretical_loss": 4.1392766897907505, + "tokens_seen": 323026944 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045560682046138416, + "loss": 1.6158, + "theoretical_loss": 4.139180022247592, + "tokens_seen": 323092480 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045559679037111334, + "loss": 1.7483, + "theoretical_loss": 4.139083379799443, + "tokens_seen": 323158016 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004555867602808426, + "loss": 1.6778, + "theoretical_loss": 4.1389867624347, + "tokens_seen": 323223552 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004555767301905717, + "loss": 1.5709, + "theoretical_loss": 4.138890170141771, + "tokens_seen": 323289088 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045556670010030094, + "loss": 1.6032, + "theoretical_loss": 4.138793602909068, + "tokens_seen": 323354624 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045555667001003007, + "loss": 1.5937, + "theoretical_loss": 4.138697060725013, + "tokens_seen": 323420160 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004555466399197593, + "loss": 1.6595, + "theoretical_loss": 4.138600543578034, + "tokens_seen": 323485696 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004555366098294885, + "loss": 1.6233, + "theoretical_loss": 4.138504051456568, + "tokens_seen": 323551232 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045552657973921767, + "loss": 1.5717, + "theoretical_loss": 4.13840758434906, + "tokens_seen": 323616768 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045551654964894685, + "loss": 1.7328, + "theoretical_loss": 4.1383111422439605, + "tokens_seen": 323682304 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004555065195586761, + "loss": 1.5811, + "theoretical_loss": 4.13821472512973, + "tokens_seen": 323747840 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004554964894684052, + "loss": 1.5996, + "theoretical_loss": 4.138118332994837, + "tokens_seen": 323813376 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045548645937813445, + "loss": 1.6788, + "theoretical_loss": 4.138021965827753, + "tokens_seen": 323878912 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045547642928786357, + "loss": 1.645, + "theoretical_loss": 4.1379256236169635, + "tokens_seen": 323944448 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004554663991975928, + "loss": 1.5548, + "theoretical_loss": 4.137829306350957, + "tokens_seen": 324009984 + }, + { + "epoch": 0.1, + "learning_rate": 0.000455456369107322, + "loss": 1.7335, + "theoretical_loss": 4.137733014018233, + "tokens_seen": 324075520 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045544633901705117, + "loss": 1.6958, + "theoretical_loss": 4.137636746607295, + "tokens_seen": 324141056 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045543630892678035, + "loss": 1.5844, + "theoretical_loss": 4.137540504106657, + "tokens_seen": 324206592 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045542627883650953, + "loss": 1.6676, + "theoretical_loss": 4.137444286504841, + "tokens_seen": 324272128 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004554162487462387, + "loss": 1.548, + "theoretical_loss": 4.137348093790372, + "tokens_seen": 324337664 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 113726, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6711912155151367, + "objective/train/theoretical_loss": 4.13725192595179, + "objective/train/tokens_used": 324403200, + "theoretical_loss": 4.13725192595179, + "tokens_seen": 324403200 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045540621865596795, + "loss": 1.7184, + "theoretical_loss": 4.13725192595179, + "tokens_seen": 324403200 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004553961885656971, + "loss": 1.6119, + "theoretical_loss": 4.1371557829776355, + "tokens_seen": 324468736 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004553861584754263, + "loss": 1.4547, + "theoretical_loss": 4.13705966485646, + "tokens_seen": 324534272 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045537612838515544, + "loss": 1.5296, + "theoretical_loss": 4.136963571576825, + "tokens_seen": 324599808 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004553660982948847, + "loss": 1.6315, + "theoretical_loss": 4.136867503127292, + "tokens_seen": 324665344 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045535606820461385, + "loss": 1.7367, + "theoretical_loss": 4.136771459496439, + "tokens_seen": 324730880 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045534603811434304, + "loss": 1.766, + "theoretical_loss": 4.136675440672844, + "tokens_seen": 324796416 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004553360080240722, + "loss": 1.6391, + "theoretical_loss": 4.136579446645098, + "tokens_seen": 324861952 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045532597793380145, + "loss": 1.6866, + "theoretical_loss": 4.136483477401798, + "tokens_seen": 324927488 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004553159478435306, + "loss": 1.6125, + "theoretical_loss": 4.136387532931546, + "tokens_seen": 324993024 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004553059177532598, + "loss": 1.6035, + "theoretical_loss": 4.136291613222955, + "tokens_seen": 325058560 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045529588766298894, + "loss": 1.5204, + "theoretical_loss": 4.136195718264644, + "tokens_seen": 325124096 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004552858575727182, + "loss": 1.5535, + "theoretical_loss": 4.136099848045239, + "tokens_seen": 325189632 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045527582748244736, + "loss": 1.6241, + "theoretical_loss": 4.136004002553375, + "tokens_seen": 325255168 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045526579739217654, + "loss": 1.5827, + "theoretical_loss": 4.135908181777693, + "tokens_seen": 325320704 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004552557673019057, + "loss": 1.5783, + "theoretical_loss": 4.135812385706842, + "tokens_seen": 325386240 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004552457372116349, + "loss": 1.6978, + "theoretical_loss": 4.135716614329479, + "tokens_seen": 325451776 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004552357071213641, + "loss": 1.5628, + "theoretical_loss": 4.135620867634269, + "tokens_seen": 325517312 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004552256770310933, + "loss": 1.4898, + "theoretical_loss": 4.135525145609881, + "tokens_seen": 325582848 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045521564694082244, + "loss": 1.5202, + "theoretical_loss": 4.135429448244997, + "tokens_seen": 325648384 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004552056168505517, + "loss": 1.6794, + "theoretical_loss": 4.135333775528302, + "tokens_seen": 325713920 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004551955867602808, + "loss": 1.6124, + "theoretical_loss": 4.13523812744849, + "tokens_seen": 325779456 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045518555667001004, + "loss": 1.5814, + "theoretical_loss": 4.135142503994263, + "tokens_seen": 325844992 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004551755265797392, + "loss": 1.5834, + "theoretical_loss": 4.13504690515433, + "tokens_seen": 325910528 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004551654964894684, + "loss": 1.6341, + "theoretical_loss": 4.134951330917408, + "tokens_seen": 325976064 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004551554663991976, + "loss": 1.586, + "theoretical_loss": 4.134855781272218, + "tokens_seen": 326041600 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004551454363089268, + "loss": 1.5441, + "theoretical_loss": 4.134760256207495, + "tokens_seen": 326107136 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045513540621865595, + "loss": 1.4265, + "theoretical_loss": 4.1346647557119764, + "tokens_seen": 326172672 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004551253761283852, + "loss": 1.5846, + "theoretical_loss": 4.134569279774407, + "tokens_seen": 326238208 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004551153460381143, + "loss": 1.656, + "theoretical_loss": 4.134473828383541, + "tokens_seen": 326303744 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045510531594784354, + "loss": 1.6634, + "theoretical_loss": 4.13437840152814, + "tokens_seen": 326369280 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004550952858575727, + "loss": 1.6658, + "theoretical_loss": 4.134282999196973, + "tokens_seen": 326434816 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004550852557673019, + "loss": 1.6055, + "theoretical_loss": 4.134187621378813, + "tokens_seen": 326500352 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004550752256770311, + "loss": 1.6075, + "theoretical_loss": 4.134092268062446, + "tokens_seen": 326565888 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045506519558676027, + "loss": 1.5734, + "theoretical_loss": 4.133996939236661, + "tokens_seen": 326631424 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045505516549648945, + "loss": 1.521, + "theoretical_loss": 4.133901634890256, + "tokens_seen": 326696960 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004550451354062187, + "loss": 1.5859, + "theoretical_loss": 4.1338063550120365, + "tokens_seen": 326762496 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004550351053159478, + "loss": 1.5425, + "theoretical_loss": 4.133711099590815, + "tokens_seen": 326828032 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045502507522567705, + "loss": 1.4998, + "theoretical_loss": 4.133615868615411, + "tokens_seen": 326893568 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045501504513540623, + "loss": 1.5348, + "theoretical_loss": 4.133520662074654, + "tokens_seen": 326959104 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004550050150451354, + "loss": 1.6815, + "theoretical_loss": 4.133425479957375, + "tokens_seen": 327024640 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004549949849548646, + "loss": 1.5246, + "theoretical_loss": 4.133330322252419, + "tokens_seen": 327090176 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045498495486459377, + "loss": 1.5819, + "theoretical_loss": 4.1332351889486345, + "tokens_seen": 327155712 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045497492477432295, + "loss": 1.536, + "theoretical_loss": 4.133140080034878, + "tokens_seen": 327221248 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004549648946840522, + "loss": 1.6072, + "theoretical_loss": 4.133044995500013, + "tokens_seen": 327286784 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045495486459378137, + "loss": 1.5206, + "theoretical_loss": 4.1329499353329116, + "tokens_seen": 327352320 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045494483450351055, + "loss": 1.6907, + "theoretical_loss": 4.132854899522453, + "tokens_seen": 327417856 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045493480441323973, + "loss": 1.5213, + "theoretical_loss": 4.132759888057521, + "tokens_seen": 327483392 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004549247743229689, + "loss": 1.499, + "theoretical_loss": 4.13266490092701, + "tokens_seen": 327548928 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045491474423269815, + "loss": 1.522, + "theoretical_loss": 4.132569938119821, + "tokens_seen": 327614464 + }, + { + "debugging/length": 576.4, + "debugging/num_segments": 5, + "debugging/score": 0.06423571687175252, + "debugging/score_std": 0.017873980887275653, + "epoch": 0.1, + "objective/train/docs_used": 114982, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.432810664176941, + "objective/train/theoretical_loss": 4.132474999624861, + "objective/train/tokens_used": 327680000, + "theoretical_loss": 4.132474999624861, + "tokens_seen": 327680000 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004549047141424273, + "loss": 1.5265, + "theoretical_loss": 4.132474999624861, + "tokens_seen": 327680000 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004548946840521565, + "loss": 1.5169, + "theoretical_loss": 4.132380085431046, + "tokens_seen": 327745536 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045488465396188564, + "loss": 1.4656, + "theoretical_loss": 4.132285195527297, + "tokens_seen": 327811072 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004548746238716149, + "loss": 1.539, + "theoretical_loss": 4.132190329902544, + "tokens_seen": 327876608 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045486459378134405, + "loss": 1.568, + "theoretical_loss": 4.132095488545724, + "tokens_seen": 327942144 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045485456369107324, + "loss": 1.5642, + "theoretical_loss": 4.132000671445781, + "tokens_seen": 328007680 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004548445336008024, + "loss": 1.6231, + "theoretical_loss": 4.131905878591667, + "tokens_seen": 328073216 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045483450351053165, + "loss": 1.6249, + "theoretical_loss": 4.13181110997234, + "tokens_seen": 328138752 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004548244734202608, + "loss": 1.5255, + "theoretical_loss": 4.131716365576765, + "tokens_seen": 328204288 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045481444332999, + "loss": 1.5973, + "theoretical_loss": 4.131621645393917, + "tokens_seen": 328269824 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045480441323971914, + "loss": 1.5628, + "theoretical_loss": 4.131526949412774, + "tokens_seen": 328335360 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004547943831494484, + "loss": 1.5219, + "theoretical_loss": 4.131432277622325, + "tokens_seen": 328400896 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045478435305917756, + "loss": 1.5896, + "theoretical_loss": 4.131337630011564, + "tokens_seen": 328466432 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045477432296890674, + "loss": 1.6238, + "theoretical_loss": 4.131243006569493, + "tokens_seen": 328531968 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004547642928786359, + "loss": 1.5227, + "theoretical_loss": 4.131148407285121, + "tokens_seen": 328597504 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004547542627883651, + "loss": 1.5952, + "theoretical_loss": 4.131053832147465, + "tokens_seen": 328663040 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004547442326980943, + "loss": 1.6073, + "theoretical_loss": 4.130959281145547, + "tokens_seen": 328728576 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004547342026078235, + "loss": 1.513, + "theoretical_loss": 4.130864754268399, + "tokens_seen": 328794112 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045472417251755264, + "loss": 1.4314, + "theoretical_loss": 4.130770251505058, + "tokens_seen": 328859648 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004547141424272819, + "loss": 1.5379, + "theoretical_loss": 4.130675772844569, + "tokens_seen": 328925184 + }, + { + "epoch": 0.1, + "learning_rate": 0.000454704112337011, + "loss": 1.5875, + "theoretical_loss": 4.130581318275984, + "tokens_seen": 328990720 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045469408224674024, + "loss": 1.5614, + "theoretical_loss": 4.130486887788362, + "tokens_seen": 329056256 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004546840521564694, + "loss": 1.5762, + "theoretical_loss": 4.130392481370771, + "tokens_seen": 329121792 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004546740220661986, + "loss": 1.7185, + "theoretical_loss": 4.130298099012283, + "tokens_seen": 329187328 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004546639919759278, + "loss": 1.5525, + "theoretical_loss": 4.130203740701979, + "tokens_seen": 329252864 + }, + { + "epoch": 0.1, + "learning_rate": 0.000454653961885657, + "loss": 1.5323, + "theoretical_loss": 4.1301094064289465, + "tokens_seen": 329318400 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045464393179538615, + "loss": 1.5203, + "theoretical_loss": 4.130015096182282, + "tokens_seen": 329383936 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004546339017051154, + "loss": 1.5127, + "theoretical_loss": 4.129920809951085, + "tokens_seen": 329449472 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004546238716148445, + "loss": 1.5853, + "theoretical_loss": 4.129826547724467, + "tokens_seen": 329515008 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045461384152457374, + "loss": 1.6096, + "theoretical_loss": 4.129732309491542, + "tokens_seen": 329580544 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004546038114343029, + "loss": 1.6237, + "theoretical_loss": 4.129638095241436, + "tokens_seen": 329646080 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004545937813440321, + "loss": 1.5991, + "theoretical_loss": 4.129543904963278, + "tokens_seen": 329711616 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004545837512537613, + "loss": 1.5855, + "theoretical_loss": 4.129449738646205, + "tokens_seen": 329777152 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045457372116349047, + "loss": 1.6456, + "theoretical_loss": 4.129355596279362, + "tokens_seen": 329842688 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045456369107321965, + "loss": 1.5347, + "theoretical_loss": 4.129261477851902, + "tokens_seen": 329908224 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004545536609829489, + "loss": 1.5537, + "theoretical_loss": 4.129167383352982, + "tokens_seen": 329973760 + }, + { + "epoch": 0.1, + "learning_rate": 0.000454543630892678, + "loss": 1.6173, + "theoretical_loss": 4.129073312771768, + "tokens_seen": 330039296 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045453360080240725, + "loss": 1.5783, + "theoretical_loss": 4.128979266097433, + "tokens_seen": 330104832 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045452357071213643, + "loss": 1.5895, + "theoretical_loss": 4.128885243319158, + "tokens_seen": 330170368 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004545135406218656, + "loss": 1.6294, + "theoretical_loss": 4.12879124442613, + "tokens_seen": 330235904 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004545035105315948, + "loss": 1.5519, + "theoretical_loss": 4.128697269407541, + "tokens_seen": 330301440 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045449348044132397, + "loss": 1.5069, + "theoretical_loss": 4.128603318252593, + "tokens_seen": 330366976 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045448345035105315, + "loss": 1.5834, + "theoretical_loss": 4.128509390950494, + "tokens_seen": 330432512 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004544734202607824, + "loss": 1.4825, + "theoretical_loss": 4.128415487490461, + "tokens_seen": 330498048 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004544633901705115, + "loss": 1.4924, + "theoretical_loss": 4.128321607861713, + "tokens_seen": 330563584 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045445336008024075, + "loss": 1.5645, + "theoretical_loss": 4.128227752053482, + "tokens_seen": 330629120 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004544433299899699, + "loss": 1.5696, + "theoretical_loss": 4.128133920055001, + "tokens_seen": 330694656 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004544332998996991, + "loss": 1.5666, + "theoretical_loss": 4.1280401118555154, + "tokens_seen": 330760192 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004544232698094283, + "loss": 1.5163, + "theoretical_loss": 4.127946327444274, + "tokens_seen": 330825728 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004544132397191575, + "loss": 1.6175, + "theoretical_loss": 4.127852566810535, + "tokens_seen": 330891264 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 116187, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4776533842086792, + "objective/train/theoretical_loss": 4.127758829943562, + "objective/train/tokens_used": 330956800, + "theoretical_loss": 4.127758829943562, + "tokens_seen": 330956800 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045440320962888666, + "loss": 1.548, + "theoretical_loss": 4.127758829943562, + "tokens_seen": 330956800 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045439317953861584, + "loss": 1.5597, + "theoretical_loss": 4.127665116832626, + "tokens_seen": 331022336 + }, + { + "epoch": 0.1, + "learning_rate": 0.000454383149448345, + "loss": 1.6722, + "theoretical_loss": 4.127571427467005, + "tokens_seen": 331087872 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045437311935807425, + "loss": 1.5302, + "theoretical_loss": 4.1274777618359835, + "tokens_seen": 331153408 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004543630892678034, + "loss": 1.6125, + "theoretical_loss": 4.127384119928854, + "tokens_seen": 331218944 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004543530591775326, + "loss": 1.6081, + "theoretical_loss": 4.127290501734914, + "tokens_seen": 331284480 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004543430290872618, + "loss": 1.6737, + "theoretical_loss": 4.127196907243472, + "tokens_seen": 331350016 + }, + { + "epoch": 0.1, + "learning_rate": 0.000454332998996991, + "loss": 1.5832, + "theoretical_loss": 4.127103336443839, + "tokens_seen": 331415552 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045432296890672016, + "loss": 1.4612, + "theoretical_loss": 4.127009789325334, + "tokens_seen": 331481088 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045431293881644934, + "loss": 1.5909, + "theoretical_loss": 4.126916265877284, + "tokens_seen": 331546624 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004543029087261785, + "loss": 1.5638, + "theoretical_loss": 4.126822766089024, + "tokens_seen": 331612160 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045429287863590776, + "loss": 1.6065, + "theoretical_loss": 4.126729289949893, + "tokens_seen": 331677696 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004542828485456369, + "loss": 1.5742, + "theoretical_loss": 4.126635837449238, + "tokens_seen": 331743232 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004542728184553661, + "loss": 1.5235, + "theoretical_loss": 4.126542408576414, + "tokens_seen": 331808768 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045426278836509525, + "loss": 1.7276, + "theoretical_loss": 4.126449003320783, + "tokens_seen": 331874304 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004542527582748245, + "loss": 1.627, + "theoretical_loss": 4.126355621671711, + "tokens_seen": 331939840 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045424272818455366, + "loss": 1.5936, + "theoretical_loss": 4.126262263618574, + "tokens_seen": 332005376 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045423269809428284, + "loss": 1.5519, + "theoretical_loss": 4.1261689291507535, + "tokens_seen": 332070912 + }, + { + "epoch": 0.1, + "learning_rate": 0.000454222668004012, + "loss": 1.6403, + "theoretical_loss": 4.126075618257639, + "tokens_seen": 332136448 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004542126379137412, + "loss": 1.6292, + "theoretical_loss": 4.125982330928624, + "tokens_seen": 332201984 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045420260782347044, + "loss": 1.5634, + "theoretical_loss": 4.125889067153113, + "tokens_seen": 332267520 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004541925777331996, + "loss": 1.5716, + "theoretical_loss": 4.125795826920514, + "tokens_seen": 332333056 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004541825476429288, + "loss": 1.5756, + "theoretical_loss": 4.125702610220244, + "tokens_seen": 332398592 + }, + { + "epoch": 0.1, + "learning_rate": 0.000454172517552658, + "loss": 1.5498, + "theoretical_loss": 4.1256094170417255, + "tokens_seen": 332464128 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004541624874623872, + "loss": 1.4833, + "theoretical_loss": 4.125516247374387, + "tokens_seen": 332529664 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045415245737211635, + "loss": 1.6832, + "theoretical_loss": 4.125423101207668, + "tokens_seen": 332595200 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004541424272818456, + "loss": 1.6404, + "theoretical_loss": 4.125329978531009, + "tokens_seen": 332660736 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004541323971915747, + "loss": 1.5596, + "theoretical_loss": 4.125236879333861, + "tokens_seen": 332726272 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045412236710130395, + "loss": 1.5852, + "theoretical_loss": 4.125143803605683, + "tokens_seen": 332791808 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004541123370110331, + "loss": 1.7014, + "theoretical_loss": 4.125050751335936, + "tokens_seen": 332857344 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004541023069207623, + "loss": 1.6175, + "theoretical_loss": 4.124957722514091, + "tokens_seen": 332922880 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004540922768304915, + "loss": 1.5757, + "theoretical_loss": 4.124864717129629, + "tokens_seen": 332988416 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045408224674022067, + "loss": 1.5818, + "theoretical_loss": 4.124771735172031, + "tokens_seen": 333053952 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045407221664994985, + "loss": 1.5609, + "theoretical_loss": 4.1246787766307875, + "tokens_seen": 333119488 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004540621865596791, + "loss": 1.6013, + "theoretical_loss": 4.124585841495398, + "tokens_seen": 333185024 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004540521564694082, + "loss": 1.685, + "theoretical_loss": 4.124492929755367, + "tokens_seen": 333250560 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045404212637913745, + "loss": 1.593, + "theoretical_loss": 4.1244000414002056, + "tokens_seen": 333316096 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045403209628886663, + "loss": 1.597, + "theoretical_loss": 4.124307176419432, + "tokens_seen": 333381632 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004540220661985958, + "loss": 1.7173, + "theoretical_loss": 4.12421433480257, + "tokens_seen": 333447168 + }, + { + "epoch": 0.1, + "learning_rate": 0.000454012036108325, + "loss": 1.554, + "theoretical_loss": 4.124121516539153, + "tokens_seen": 333512704 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045400200601805417, + "loss": 1.6274, + "theoretical_loss": 4.124028721618719, + "tokens_seen": 333578240 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045399197592778335, + "loss": 1.513, + "theoretical_loss": 4.123935950030812, + "tokens_seen": 333643776 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004539819458375126, + "loss": 1.518, + "theoretical_loss": 4.123843201764984, + "tokens_seen": 333709312 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004539719157472417, + "loss": 1.6528, + "theoretical_loss": 4.1237504768107955, + "tokens_seen": 333774848 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045396188565697095, + "loss": 1.5385, + "theoretical_loss": 4.12365777515781, + "tokens_seen": 333840384 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004539518555667001, + "loss": 1.637, + "theoretical_loss": 4.1235650967956, + "tokens_seen": 333905920 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004539418254764293, + "loss": 1.5463, + "theoretical_loss": 4.123472441713743, + "tokens_seen": 333971456 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004539317953861585, + "loss": 1.635, + "theoretical_loss": 4.123379809901827, + "tokens_seen": 334036992 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004539217652958877, + "loss": 1.5657, + "theoretical_loss": 4.123287201349443, + "tokens_seen": 334102528 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045391173520561686, + "loss": 1.5541, + "theoretical_loss": 4.123194616046189, + "tokens_seen": 334168064 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 117142, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5620770454406738, + "objective/train/theoretical_loss": 4.123102053981672, + "objective/train/tokens_used": 334233600, + "theoretical_loss": 4.123102053981672, + "tokens_seen": 334233600 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045390170511534604, + "loss": 1.5517, + "theoretical_loss": 4.123102053981672, + "tokens_seen": 334233600 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004538916750250752, + "loss": 1.5529, + "theoretical_loss": 4.123009515145503, + "tokens_seen": 334299136 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045388164493480445, + "loss": 1.6521, + "theoretical_loss": 4.122916999527301, + "tokens_seen": 334364672 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004538716148445336, + "loss": 1.6622, + "theoretical_loss": 4.122824507116691, + "tokens_seen": 334430208 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004538615847542628, + "loss": 1.5905, + "theoretical_loss": 4.122732037903308, + "tokens_seen": 334495744 + }, + { + "epoch": 0.1, + "learning_rate": 0.000453851554663992, + "loss": 1.7014, + "theoretical_loss": 4.122639591876789, + "tokens_seen": 334561280 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004538415245737212, + "loss": 1.5839, + "theoretical_loss": 4.122547169026778, + "tokens_seen": 334626816 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045383149448345036, + "loss": 1.4278, + "theoretical_loss": 4.12245476934293, + "tokens_seen": 334692352 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045382146439317954, + "loss": 1.4481, + "theoretical_loss": 4.122362392814903, + "tokens_seen": 334757888 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004538114343029087, + "loss": 1.5861, + "theoretical_loss": 4.122270039432362, + "tokens_seen": 334823424 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045380140421263796, + "loss": 1.5191, + "theoretical_loss": 4.122177709184978, + "tokens_seen": 334888960 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004537913741223671, + "loss": 1.6191, + "theoretical_loss": 4.1220854020624325, + "tokens_seen": 334954496 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004537813440320963, + "loss": 1.6818, + "theoretical_loss": 4.121993118054409, + "tokens_seen": 335020032 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045377131394182545, + "loss": 1.5412, + "theoretical_loss": 4.1219008571505995, + "tokens_seen": 335085568 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004537612838515547, + "loss": 1.5763, + "theoretical_loss": 4.121808619340704, + "tokens_seen": 335151104 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045375125376128386, + "loss": 1.5632, + "theoretical_loss": 4.121716404614426, + "tokens_seen": 335216640 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045374122367101304, + "loss": 1.5401, + "theoretical_loss": 4.121624212961478, + "tokens_seen": 335282176 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004537311935807422, + "loss": 1.5725, + "theoretical_loss": 4.121532044371579, + "tokens_seen": 335347712 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004537211634904714, + "loss": 1.5557, + "theoretical_loss": 4.121439898834454, + "tokens_seen": 335413248 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004537111334002006, + "loss": 1.6665, + "theoretical_loss": 4.1213477763398325, + "tokens_seen": 335478784 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004537011033099298, + "loss": 1.625, + "theoretical_loss": 4.121255676877455, + "tokens_seen": 335544320 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045369107321965895, + "loss": 1.4813, + "theoretical_loss": 4.1211636004370655, + "tokens_seen": 335609856 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004536810431293882, + "loss": 1.5441, + "theoretical_loss": 4.121071547008415, + "tokens_seen": 335675392 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045367101303911737, + "loss": 1.4705, + "theoretical_loss": 4.120979516581262, + "tokens_seen": 335740928 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045366098294884655, + "loss": 1.6103, + "theoretical_loss": 4.120887509145371, + "tokens_seen": 335806464 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045365095285857573, + "loss": 1.641, + "theoretical_loss": 4.120795524690512, + "tokens_seen": 335872000 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004536409227683049, + "loss": 1.5691, + "theoretical_loss": 4.1207035632064635, + "tokens_seen": 335937536 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004536308926780341, + "loss": 1.6178, + "theoretical_loss": 4.120611624683008, + "tokens_seen": 336003072 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004536208625877633, + "loss": 1.5236, + "theoretical_loss": 4.120519709109938, + "tokens_seen": 336068608 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045361083249749245, + "loss": 1.6263, + "theoretical_loss": 4.12042781647705, + "tokens_seen": 336134144 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004536008024072217, + "loss": 1.694, + "theoretical_loss": 4.120335946774147, + "tokens_seen": 336199680 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004535907723169508, + "loss": 1.6498, + "theoretical_loss": 4.12024409999104, + "tokens_seen": 336265216 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045358074222668005, + "loss": 1.6912, + "theoretical_loss": 4.120152276117545, + "tokens_seen": 336330752 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045357071213640923, + "loss": 1.6189, + "theoretical_loss": 4.120060475143486, + "tokens_seen": 336396288 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004535606820461384, + "loss": 1.5395, + "theoretical_loss": 4.119968697058692, + "tokens_seen": 336461824 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004535506519558676, + "loss": 1.4985, + "theoretical_loss": 4.119876941852999, + "tokens_seen": 336527360 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045354062186559683, + "loss": 1.5829, + "theoretical_loss": 4.11978520951625, + "tokens_seen": 336592896 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045353059177532596, + "loss": 1.6051, + "theoretical_loss": 4.119693500038294, + "tokens_seen": 336658432 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004535205616850552, + "loss": 1.5563, + "theoretical_loss": 4.119601813408988, + "tokens_seen": 336723968 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004535105315947843, + "loss": 1.4467, + "theoretical_loss": 4.1195101496181925, + "tokens_seen": 336789504 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045350050150451355, + "loss": 1.5656, + "theoretical_loss": 4.119418508655777, + "tokens_seen": 336855040 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045349047141424274, + "loss": 1.7038, + "theoretical_loss": 4.119326890511615, + "tokens_seen": 336920576 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004534804413239719, + "loss": 1.5501, + "theoretical_loss": 4.1192352951755895, + "tokens_seen": 336986112 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004534704112337011, + "loss": 1.6031, + "theoretical_loss": 4.119143722637588, + "tokens_seen": 337051648 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004534603811434303, + "loss": 1.4672, + "theoretical_loss": 4.119052172887506, + "tokens_seen": 337117184 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004534503510531595, + "loss": 1.4788, + "theoretical_loss": 4.118960645915243, + "tokens_seen": 337182720 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004534403209628887, + "loss": 1.7154, + "theoretical_loss": 4.118869141710706, + "tokens_seen": 337248256 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004534302908726179, + "loss": 1.5196, + "theoretical_loss": 4.11877766026381, + "tokens_seen": 337313792 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045342026078234706, + "loss": 1.5648, + "theoretical_loss": 4.118686201564475, + "tokens_seen": 337379328 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045341023069207624, + "loss": 1.5748, + "theoretical_loss": 4.1185947656026265, + "tokens_seen": 337444864 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 118287, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.7597241401672363, + "objective/train/theoretical_loss": 4.118503352368197, + "objective/train/tokens_used": 337510400, + "theoretical_loss": 4.118503352368197, + "tokens_seen": 337510400 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004534002006018054, + "loss": 1.6375, + "theoretical_loss": 4.118503352368197, + "tokens_seen": 337510400 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045339017051153465, + "loss": 1.6169, + "theoretical_loss": 4.118411961851129, + "tokens_seen": 337575936 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004533801404212638, + "loss": 1.6769, + "theoretical_loss": 4.118320594041366, + "tokens_seen": 337641472 + }, + { + "epoch": 0.1, + "learning_rate": 0.000453370110330993, + "loss": 1.6596, + "theoretical_loss": 4.118229248928859, + "tokens_seen": 337707008 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004533600802407222, + "loss": 1.5539, + "theoretical_loss": 4.11813792650357, + "tokens_seen": 337772544 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004533500501504514, + "loss": 1.5119, + "theoretical_loss": 4.118046626755461, + "tokens_seen": 337838080 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045334002006018056, + "loss": 1.6415, + "theoretical_loss": 4.117955349674505, + "tokens_seen": 337903616 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045332998996990974, + "loss": 1.6657, + "theoretical_loss": 4.117864095250679, + "tokens_seen": 337969152 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004533199598796389, + "loss": 1.6002, + "theoretical_loss": 4.117772863473968, + "tokens_seen": 338034688 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045330992978936816, + "loss": 1.5779, + "theoretical_loss": 4.117681654334361, + "tokens_seen": 338100224 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004532998996990973, + "loss": 1.5488, + "theoretical_loss": 4.117590467821857, + "tokens_seen": 338165760 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004532898696088265, + "loss": 1.5241, + "theoretical_loss": 4.117499303926457, + "tokens_seen": 338231296 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045327983951855565, + "loss": 1.5888, + "theoretical_loss": 4.1174081626381716, + "tokens_seen": 338296832 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004532698094282849, + "loss": 1.5746, + "theoretical_loss": 4.117317043947017, + "tokens_seen": 338362368 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045325977933801406, + "loss": 1.5718, + "theoretical_loss": 4.117225947843014, + "tokens_seen": 338427904 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045324974924774324, + "loss": 1.5227, + "theoretical_loss": 4.117134874316194, + "tokens_seen": 338493440 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004532397191574724, + "loss": 1.6119, + "theoretical_loss": 4.117043823356589, + "tokens_seen": 338558976 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004532296890672016, + "loss": 1.6408, + "theoretical_loss": 4.116952794954242, + "tokens_seen": 338624512 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004532196589769308, + "loss": 1.5507, + "theoretical_loss": 4.1168617890992, + "tokens_seen": 338690048 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045320962888666, + "loss": 1.4723, + "theoretical_loss": 4.116770805781517, + "tokens_seen": 338755584 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045319959879638915, + "loss": 1.6293, + "theoretical_loss": 4.116679844991253, + "tokens_seen": 338821120 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004531895687061184, + "loss": 1.6298, + "theoretical_loss": 4.116588906718475, + "tokens_seen": 338886656 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045317953861584757, + "loss": 1.5544, + "theoretical_loss": 4.116497990953255, + "tokens_seen": 338952192 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045316950852557675, + "loss": 1.6445, + "theoretical_loss": 4.1164070976856735, + "tokens_seen": 339017728 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045315947843530593, + "loss": 1.6202, + "theoretical_loss": 4.1163162269058144, + "tokens_seen": 339083264 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004531494483450351, + "loss": 1.6473, + "theoretical_loss": 4.11622537860377, + "tokens_seen": 339148800 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004531394182547643, + "loss": 1.4794, + "theoretical_loss": 4.116134552769638, + "tokens_seen": 339214336 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004531293881644935, + "loss": 1.5199, + "theoretical_loss": 4.116043749393523, + "tokens_seen": 339279872 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045311935807422265, + "loss": 1.652, + "theoretical_loss": 4.115952968465535, + "tokens_seen": 339345408 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004531093279839519, + "loss": 1.6128, + "theoretical_loss": 4.115862209975791, + "tokens_seen": 339410944 + }, + { + "epoch": 0.1, + "learning_rate": 0.000453099297893681, + "loss": 1.5809, + "theoretical_loss": 4.115771473914415, + "tokens_seen": 339476480 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045308926780341025, + "loss": 1.5693, + "theoretical_loss": 4.115680760271535, + "tokens_seen": 339542016 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045307923771313943, + "loss": 1.5729, + "theoretical_loss": 4.115590069037287, + "tokens_seen": 339607552 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004530692076228686, + "loss": 1.4673, + "theoretical_loss": 4.115499400201813, + "tokens_seen": 339673088 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004530591775325978, + "loss": 1.6701, + "theoretical_loss": 4.1154087537552595, + "tokens_seen": 339738624 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045304914744232703, + "loss": 1.5709, + "theoretical_loss": 4.1153181296877825, + "tokens_seen": 339804160 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045303911735205616, + "loss": 1.5478, + "theoretical_loss": 4.115227527989543, + "tokens_seen": 339869696 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004530290872617854, + "loss": 1.6966, + "theoretical_loss": 4.115136948650705, + "tokens_seen": 339935232 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004530190571715145, + "loss": 1.4248, + "theoretical_loss": 4.115046391661443, + "tokens_seen": 340000768 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045300902708124375, + "loss": 1.5547, + "theoretical_loss": 4.114955857011937, + "tokens_seen": 340066304 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045299899699097294, + "loss": 1.7133, + "theoretical_loss": 4.114865344692371, + "tokens_seen": 340131840 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004529889669007021, + "loss": 1.699, + "theoretical_loss": 4.1147748546929375, + "tokens_seen": 340197376 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004529789368104313, + "loss": 1.59, + "theoretical_loss": 4.114684387003833, + "tokens_seen": 340262912 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004529689067201605, + "loss": 1.5272, + "theoretical_loss": 4.114593941615262, + "tokens_seen": 340328448 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045295887662988966, + "loss": 1.4372, + "theoretical_loss": 4.114503518517435, + "tokens_seen": 340393984 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004529488465396189, + "loss": 1.7183, + "theoretical_loss": 4.114413117700568, + "tokens_seen": 340459520 + }, + { + "epoch": 0.1, + "learning_rate": 0.000452938816449348, + "loss": 1.6407, + "theoretical_loss": 4.114322739154883, + "tokens_seen": 340525056 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045292878635907726, + "loss": 1.6288, + "theoretical_loss": 4.114232382870609, + "tokens_seen": 340590592 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004529187562688064, + "loss": 1.5348, + "theoretical_loss": 4.114142048837982, + "tokens_seen": 340656128 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004529087261785356, + "loss": 1.6813, + "theoretical_loss": 4.114051737047241, + "tokens_seen": 340721664 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 119240, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4965146780014038, + "objective/train/theoretical_loss": 4.1139614474886335, + "objective/train/tokens_used": 340787200, + "theoretical_loss": 4.1139614474886335, + "tokens_seen": 340787200 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004528986960882648, + "loss": 1.6435, + "theoretical_loss": 4.1139614474886335, + "tokens_seen": 340787200 + }, + { + "epoch": 0.1, + "learning_rate": 0.000452888665997994, + "loss": 1.6395, + "theoretical_loss": 4.1138711801524135, + "tokens_seen": 340852736 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045287863590772316, + "loss": 1.5064, + "theoretical_loss": 4.11378093502884, + "tokens_seen": 340918272 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004528686058174524, + "loss": 1.5521, + "theoretical_loss": 4.113690712108179, + "tokens_seen": 340983808 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004528585757271815, + "loss": 1.6846, + "theoretical_loss": 4.113600511380701, + "tokens_seen": 341049344 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045284854563691076, + "loss": 1.6307, + "theoretical_loss": 4.113510332836685, + "tokens_seen": 341114880 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004528385155466399, + "loss": 1.6096, + "theoretical_loss": 4.1134201764664144, + "tokens_seen": 341180416 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004528284854563691, + "loss": 1.6446, + "theoretical_loss": 4.113330042260181, + "tokens_seen": 341245952 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004528184553660983, + "loss": 1.494, + "theoretical_loss": 4.113239930208278, + "tokens_seen": 341311488 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004528084252758275, + "loss": 1.5864, + "theoretical_loss": 4.113149840301009, + "tokens_seen": 341377024 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045279839518555667, + "loss": 1.5146, + "theoretical_loss": 4.113059772528683, + "tokens_seen": 341442560 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045278836509528585, + "loss": 1.5502, + "theoretical_loss": 4.112969726881613, + "tokens_seen": 341508096 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045277833500501503, + "loss": 1.4604, + "theoretical_loss": 4.112879703350122, + "tokens_seen": 341573632 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045276830491474426, + "loss": 1.6511, + "theoretical_loss": 4.112789701924535, + "tokens_seen": 341639168 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004527582748244734, + "loss": 1.7004, + "theoretical_loss": 4.112699722595185, + "tokens_seen": 341704704 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004527482447342026, + "loss": 1.5712, + "theoretical_loss": 4.11260976535241, + "tokens_seen": 341770240 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045273821464393175, + "loss": 1.4884, + "theoretical_loss": 4.112519830186557, + "tokens_seen": 341835776 + }, + { + "epoch": 0.1, + "learning_rate": 0.000452728184553661, + "loss": 1.6567, + "theoretical_loss": 4.112429917087974, + "tokens_seen": 341901312 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045271815446339017, + "loss": 1.5806, + "theoretical_loss": 4.112340026047022, + "tokens_seen": 341966848 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045270812437311935, + "loss": 1.5784, + "theoretical_loss": 4.112250157054061, + "tokens_seen": 342032384 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004526980942828486, + "loss": 1.5498, + "theoretical_loss": 4.112160310099461, + "tokens_seen": 342097920 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045268806419257777, + "loss": 1.5501, + "theoretical_loss": 4.112070485173598, + "tokens_seen": 342163456 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045267803410230695, + "loss": 1.6474, + "theoretical_loss": 4.111980682266853, + "tokens_seen": 342228992 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045266800401203613, + "loss": 1.6607, + "theoretical_loss": 4.111890901369612, + "tokens_seen": 342294528 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004526579739217653, + "loss": 1.5962, + "theoretical_loss": 4.11180114247227, + "tokens_seen": 342360064 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004526479438314945, + "loss": 1.5217, + "theoretical_loss": 4.1117114055652255, + "tokens_seen": 342425600 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004526379137412237, + "loss": 1.672, + "theoretical_loss": 4.111621690638884, + "tokens_seen": 342491136 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045262788365095285, + "loss": 1.6674, + "theoretical_loss": 4.111531997683658, + "tokens_seen": 342556672 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004526178535606821, + "loss": 1.5918, + "theoretical_loss": 4.1114423266899625, + "tokens_seen": 342622208 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004526078234704112, + "loss": 1.5535, + "theoretical_loss": 4.111352677648224, + "tokens_seen": 342687744 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045259779338014045, + "loss": 1.5797, + "theoretical_loss": 4.111263050548869, + "tokens_seen": 342753280 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045258776328986963, + "loss": 1.5196, + "theoretical_loss": 4.111173445382335, + "tokens_seen": 342818816 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004525777331995988, + "loss": 1.6325, + "theoretical_loss": 4.111083862139063, + "tokens_seen": 342884352 + }, + { + "epoch": 0.1, + "learning_rate": 0.000452567703109328, + "loss": 1.385, + "theoretical_loss": 4.110994300809501, + "tokens_seen": 342949888 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045255767301905723, + "loss": 1.6766, + "theoretical_loss": 4.1109047613841, + "tokens_seen": 343015424 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045254764292878636, + "loss": 1.5155, + "theoretical_loss": 4.110815243853323, + "tokens_seen": 343080960 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004525376128385156, + "loss": 1.6615, + "theoretical_loss": 4.110725748207633, + "tokens_seen": 343146496 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004525275827482447, + "loss": 1.7356, + "theoretical_loss": 4.110636274437502, + "tokens_seen": 343212032 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045251755265797395, + "loss": 1.608, + "theoretical_loss": 4.110546822533407, + "tokens_seen": 343277568 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045250752256770314, + "loss": 1.7833, + "theoretical_loss": 4.110457392485832, + "tokens_seen": 343343104 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004524974924774323, + "loss": 1.5136, + "theoretical_loss": 4.110367984285265, + "tokens_seen": 343408640 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004524874623871615, + "loss": 1.6956, + "theoretical_loss": 4.1102785979222025, + "tokens_seen": 343474176 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004524774322968907, + "loss": 1.5996, + "theoretical_loss": 4.110189233387146, + "tokens_seen": 343539712 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045246740220661986, + "loss": 1.5225, + "theoretical_loss": 4.1100998906706, + "tokens_seen": 343605248 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004524573721163491, + "loss": 1.5537, + "theoretical_loss": 4.110010569763081, + "tokens_seen": 343670784 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004524473420260782, + "loss": 1.5219, + "theoretical_loss": 4.109921270655106, + "tokens_seen": 343736320 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045243731193580746, + "loss": 1.61, + "theoretical_loss": 4.109831993337201, + "tokens_seen": 343801856 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004524272818455366, + "loss": 1.5486, + "theoretical_loss": 4.109742737799895, + "tokens_seen": 343867392 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004524172517552658, + "loss": 1.5923, + "theoretical_loss": 4.109653504033726, + "tokens_seen": 343932928 + }, + { + "epoch": 0.1, + "learning_rate": 0.000452407221664995, + "loss": 1.5747, + "theoretical_loss": 4.109564292029235, + "tokens_seen": 343998464 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 120518, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.662473201751709, + "objective/train/theoretical_loss": 4.109475101776974, + "objective/train/tokens_used": 344064000, + "theoretical_loss": 4.109475101776974, + "tokens_seen": 344064000 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004523971915747242, + "loss": 1.5818, + "theoretical_loss": 4.109475101776974, + "tokens_seen": 344064000 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045238716148445336, + "loss": 1.5809, + "theoretical_loss": 4.109385933267495, + "tokens_seen": 344129536 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004523771313941826, + "loss": 1.6153, + "theoretical_loss": 4.109296786491359, + "tokens_seen": 344195072 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004523671013039117, + "loss": 1.4936, + "theoretical_loss": 4.1092076614391315, + "tokens_seen": 344260608 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045235707121364096, + "loss": 1.5847, + "theoretical_loss": 4.109118558101386, + "tokens_seen": 344326144 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004523470411233701, + "loss": 1.5848, + "theoretical_loss": 4.1090294764687, + "tokens_seen": 344391680 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004523370110330993, + "loss": 1.437, + "theoretical_loss": 4.108940416531656, + "tokens_seen": 344457216 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004523269809428285, + "loss": 1.6305, + "theoretical_loss": 4.108851378280846, + "tokens_seen": 344522752 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004523169508525577, + "loss": 1.6806, + "theoretical_loss": 4.108762361706865, + "tokens_seen": 344588288 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045230692076228687, + "loss": 1.6411, + "theoretical_loss": 4.108673366800313, + "tokens_seen": 344653824 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045229689067201605, + "loss": 1.5574, + "theoretical_loss": 4.1085843935518, + "tokens_seen": 344719360 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045228686058174523, + "loss": 1.6092, + "theoretical_loss": 4.108495441951937, + "tokens_seen": 344784896 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045227683049147446, + "loss": 1.6241, + "theoretical_loss": 4.108406511991344, + "tokens_seen": 344850432 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004522668004012036, + "loss": 1.5431, + "theoretical_loss": 4.108317603660646, + "tokens_seen": 344915968 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004522567703109328, + "loss": 1.5397, + "theoretical_loss": 4.108228716950474, + "tokens_seen": 344981504 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045224674022066195, + "loss": 1.694, + "theoretical_loss": 4.108139851851464, + "tokens_seen": 345047040 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004522367101303912, + "loss": 1.6686, + "theoretical_loss": 4.108051008354258, + "tokens_seen": 345112576 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045222668004012037, + "loss": 1.6444, + "theoretical_loss": 4.107962186449506, + "tokens_seen": 345178112 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045221664994984955, + "loss": 1.5705, + "theoretical_loss": 4.107873386127861, + "tokens_seen": 345243648 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045220661985957873, + "loss": 1.6183, + "theoretical_loss": 4.107784607379983, + "tokens_seen": 345309184 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045219658976930797, + "loss": 1.6743, + "theoretical_loss": 4.107695850196539, + "tokens_seen": 345374720 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004521865596790371, + "loss": 1.7029, + "theoretical_loss": 4.107607114568197, + "tokens_seen": 345440256 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045217652958876633, + "loss": 1.6633, + "theoretical_loss": 4.107518400485638, + "tokens_seen": 345505792 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045216649949849546, + "loss": 1.5906, + "theoretical_loss": 4.107429707939545, + "tokens_seen": 345571328 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004521564694082247, + "loss": 1.6064, + "theoretical_loss": 4.107341036920604, + "tokens_seen": 345636864 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045214643931795387, + "loss": 1.5375, + "theoretical_loss": 4.107252387419513, + "tokens_seen": 345702400 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045213640922768305, + "loss": 1.5951, + "theoretical_loss": 4.107163759426971, + "tokens_seen": 345767936 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045212637913741223, + "loss": 1.5762, + "theoretical_loss": 4.1070751529336835, + "tokens_seen": 345833472 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004521163490471414, + "loss": 1.4337, + "theoretical_loss": 4.106986567930364, + "tokens_seen": 345899008 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004521063189568706, + "loss": 1.4729, + "theoretical_loss": 4.1068980044077295, + "tokens_seen": 345964544 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045209628886659983, + "loss": 1.5804, + "theoretical_loss": 4.1068094623565035, + "tokens_seen": 346030080 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045208625877632896, + "loss": 1.6157, + "theoretical_loss": 4.106720941767417, + "tokens_seen": 346095616 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004520762286860582, + "loss": 1.5588, + "theoretical_loss": 4.106632442631203, + "tokens_seen": 346161152 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004520661985957873, + "loss": 1.5581, + "theoretical_loss": 4.106543964938604, + "tokens_seen": 346226688 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045205616850551656, + "loss": 1.756, + "theoretical_loss": 4.106455508680366, + "tokens_seen": 346292224 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045204613841524574, + "loss": 1.5503, + "theoretical_loss": 4.106367073847241, + "tokens_seen": 346357760 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004520361083249749, + "loss": 1.5105, + "theoretical_loss": 4.106278660429988, + "tokens_seen": 346423296 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004520260782347041, + "loss": 1.6252, + "theoretical_loss": 4.10619026841937, + "tokens_seen": 346488832 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045201604814443334, + "loss": 1.5548, + "theoretical_loss": 4.106101897806156, + "tokens_seen": 346554368 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045200601805416246, + "loss": 1.5702, + "theoretical_loss": 4.1060135485811236, + "tokens_seen": 346619904 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004519959879638917, + "loss": 1.4933, + "theoretical_loss": 4.105925220735052, + "tokens_seen": 346685440 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004519859578736208, + "loss": 1.5635, + "theoretical_loss": 4.105836914258728, + "tokens_seen": 346750976 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045197592778335006, + "loss": 1.4648, + "theoretical_loss": 4.105748629142945, + "tokens_seen": 346816512 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045196589769307924, + "loss": 1.4401, + "theoretical_loss": 4.1056603653785, + "tokens_seen": 346882048 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004519558676028084, + "loss": 1.5557, + "theoretical_loss": 4.1055721229561986, + "tokens_seen": 346947584 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045194583751253766, + "loss": 1.682, + "theoretical_loss": 4.105483901866848, + "tokens_seen": 347013120 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004519358074222668, + "loss": 1.6695, + "theoretical_loss": 4.105395702101266, + "tokens_seen": 347078656 + }, + { + "epoch": 0.11, + "learning_rate": 0.000451925777331996, + "loss": 1.5188, + "theoretical_loss": 4.1053075236502705, + "tokens_seen": 347144192 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004519157472417252, + "loss": 1.4555, + "theoretical_loss": 4.1052193665046905, + "tokens_seen": 347209728 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004519057171514544, + "loss": 1.5348, + "theoretical_loss": 4.105131230655358, + "tokens_seen": 347275264 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 121413, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.7423319816589355, + "objective/train/theoretical_loss": 4.1050431160931105, + "objective/train/tokens_used": 347340800, + "theoretical_loss": 4.1050431160931105, + "tokens_seen": 347340800 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045189568706118356, + "loss": 1.7282, + "theoretical_loss": 4.1050431160931105, + "tokens_seen": 347340800 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004518856569709128, + "loss": 1.5723, + "theoretical_loss": 4.104955022808792, + "tokens_seen": 347406336 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004518756268806419, + "loss": 1.718, + "theoretical_loss": 4.104866950793251, + "tokens_seen": 347471872 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045186559679037116, + "loss": 1.6176, + "theoretical_loss": 4.104778900037343, + "tokens_seen": 347537408 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004518555667001003, + "loss": 1.604, + "theoretical_loss": 4.104690870531929, + "tokens_seen": 347602944 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004518455366098295, + "loss": 1.6331, + "theoretical_loss": 4.104602862267875, + "tokens_seen": 347668480 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004518355065195587, + "loss": 1.6437, + "theoretical_loss": 4.1045148752360525, + "tokens_seen": 347734016 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004518254764292879, + "loss": 1.5134, + "theoretical_loss": 4.104426909427339, + "tokens_seen": 347799552 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045181544633901707, + "loss": 1.6216, + "theoretical_loss": 4.104338964832619, + "tokens_seen": 347865088 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045180541624874625, + "loss": 1.6033, + "theoretical_loss": 4.10425104144278, + "tokens_seen": 347930624 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045179538615847543, + "loss": 1.6518, + "theoretical_loss": 4.104163139248717, + "tokens_seen": 347996160 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045178535606820466, + "loss": 1.4809, + "theoretical_loss": 4.1040752582413305, + "tokens_seen": 348061696 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004517753259779338, + "loss": 1.5989, + "theoretical_loss": 4.103987398411524, + "tokens_seen": 348127232 + }, + { + "epoch": 0.11, + "learning_rate": 0.000451765295887663, + "loss": 1.5483, + "theoretical_loss": 4.103899559750213, + "tokens_seen": 348192768 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045175526579739215, + "loss": 1.5606, + "theoretical_loss": 4.10381174224831, + "tokens_seen": 348258304 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004517452357071214, + "loss": 1.6182, + "theoretical_loss": 4.10372394589674, + "tokens_seen": 348323840 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045173520561685057, + "loss": 1.7054, + "theoretical_loss": 4.10363617068643, + "tokens_seen": 348389376 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045172517552657975, + "loss": 1.6053, + "theoretical_loss": 4.103548416608316, + "tokens_seen": 348454912 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045171514543630893, + "loss": 1.6833, + "theoretical_loss": 4.103460683653335, + "tokens_seen": 348520448 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045170511534603817, + "loss": 1.5052, + "theoretical_loss": 4.103372971812432, + "tokens_seen": 348585984 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004516950852557673, + "loss": 1.6192, + "theoretical_loss": 4.103285281076558, + "tokens_seen": 348651520 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045168505516549653, + "loss": 1.5811, + "theoretical_loss": 4.10319761143667, + "tokens_seen": 348717056 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045167502507522566, + "loss": 1.5252, + "theoretical_loss": 4.103109962883728, + "tokens_seen": 348782592 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004516649949849549, + "loss": 1.4692, + "theoretical_loss": 4.1030223354087, + "tokens_seen": 348848128 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045165496489468407, + "loss": 1.6349, + "theoretical_loss": 4.102934729002559, + "tokens_seen": 348913664 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045164493480441325, + "loss": 1.5107, + "theoretical_loss": 4.102847143656284, + "tokens_seen": 348979200 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045163490471414244, + "loss": 1.6313, + "theoretical_loss": 4.102759579360857, + "tokens_seen": 349044736 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004516248746238716, + "loss": 1.623, + "theoretical_loss": 4.102672036107269, + "tokens_seen": 349110272 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004516148445336008, + "loss": 1.5288, + "theoretical_loss": 4.102584513886514, + "tokens_seen": 349175808 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045160481444333003, + "loss": 1.5163, + "theoretical_loss": 4.102497012689594, + "tokens_seen": 349241344 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045159478435305916, + "loss": 1.6743, + "theoretical_loss": 4.102409532507513, + "tokens_seen": 349306880 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004515847542627884, + "loss": 1.5098, + "theoretical_loss": 4.102322073331285, + "tokens_seen": 349372416 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004515747241725175, + "loss": 1.6046, + "theoretical_loss": 4.102234635151925, + "tokens_seen": 349437952 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045156469408224676, + "loss": 1.5493, + "theoretical_loss": 4.102147217960457, + "tokens_seen": 349503488 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045155466399197594, + "loss": 1.6437, + "theoretical_loss": 4.1020598217479085, + "tokens_seen": 349569024 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004515446339017051, + "loss": 1.5481, + "theoretical_loss": 4.101972446505314, + "tokens_seen": 349634560 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004515346038114343, + "loss": 1.5726, + "theoretical_loss": 4.101885092223712, + "tokens_seen": 349700096 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045152457372116354, + "loss": 1.6227, + "theoretical_loss": 4.101797758894147, + "tokens_seen": 349765632 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045151454363089266, + "loss": 1.6024, + "theoretical_loss": 4.101710446507669, + "tokens_seen": 349831168 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004515045135406219, + "loss": 1.5442, + "theoretical_loss": 4.101623155055336, + "tokens_seen": 349896704 + }, + { + "epoch": 0.11, + "learning_rate": 0.000451494483450351, + "loss": 1.4992, + "theoretical_loss": 4.101535884528206, + "tokens_seen": 349962240 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045148445336008026, + "loss": 1.6243, + "theoretical_loss": 4.1014486349173485, + "tokens_seen": 350027776 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045147442326980944, + "loss": 1.4905, + "theoretical_loss": 4.101361406213834, + "tokens_seen": 350093312 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004514643931795386, + "loss": 1.5099, + "theoretical_loss": 4.101274198408741, + "tokens_seen": 350158848 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004514543630892678, + "loss": 1.6524, + "theoretical_loss": 4.101187011493153, + "tokens_seen": 350224384 + }, + { + "epoch": 0.11, + "learning_rate": 0.000451444332998997, + "loss": 1.534, + "theoretical_loss": 4.101099845458156, + "tokens_seen": 350289920 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045143430290872617, + "loss": 1.5128, + "theoretical_loss": 4.101012700294848, + "tokens_seen": 350355456 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004514242728184554, + "loss": 1.5749, + "theoretical_loss": 4.100925575994325, + "tokens_seen": 350420992 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045141424272818453, + "loss": 1.6098, + "theoretical_loss": 4.100838472547695, + "tokens_seen": 350486528 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045140421263791376, + "loss": 1.5902, + "theoretical_loss": 4.100751389946067, + "tokens_seen": 350552064 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 122656, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4169642925262451, + "objective/train/theoretical_loss": 4.100664328180557, + "objective/train/tokens_used": 350617600, + "theoretical_loss": 4.100664328180557, + "tokens_seen": 350617600 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045139418254764294, + "loss": 1.5604, + "theoretical_loss": 4.100664328180557, + "tokens_seen": 350617600 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004513841524573721, + "loss": 1.6409, + "theoretical_loss": 4.100577287242286, + "tokens_seen": 350683136 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004513741223671013, + "loss": 1.6483, + "theoretical_loss": 4.100490267122381, + "tokens_seen": 350748672 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004513640922768305, + "loss": 1.5455, + "theoretical_loss": 4.100403267811975, + "tokens_seen": 350814208 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045135406218655967, + "loss": 1.5287, + "theoretical_loss": 4.100316289302206, + "tokens_seen": 350879744 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004513440320962889, + "loss": 1.7353, + "theoretical_loss": 4.100229331584215, + "tokens_seen": 350945280 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045133400200601803, + "loss": 1.6837, + "theoretical_loss": 4.100142394649152, + "tokens_seen": 351010816 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045132397191574727, + "loss": 1.737, + "theoretical_loss": 4.100055478488171, + "tokens_seen": 351076352 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004513139418254764, + "loss": 1.5871, + "theoretical_loss": 4.099968583092431, + "tokens_seen": 351141888 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045130391173520563, + "loss": 1.6543, + "theoretical_loss": 4.099881708453097, + "tokens_seen": 351207424 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004512938816449348, + "loss": 1.5577, + "theoretical_loss": 4.099794854561339, + "tokens_seen": 351272960 + }, + { + "epoch": 0.11, + "learning_rate": 0.000451283851554664, + "loss": 1.5818, + "theoretical_loss": 4.0997080214083335, + "tokens_seen": 351338496 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045127382146439317, + "loss": 1.5526, + "theoretical_loss": 4.09962120898526, + "tokens_seen": 351404032 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045126379137412235, + "loss": 1.5047, + "theoretical_loss": 4.099534417283305, + "tokens_seen": 351469568 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045125376128385153, + "loss": 1.5786, + "theoretical_loss": 4.099447646293662, + "tokens_seen": 351535104 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045124373119358077, + "loss": 1.6405, + "theoretical_loss": 4.099360896007527, + "tokens_seen": 351600640 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004512337011033099, + "loss": 1.5766, + "theoretical_loss": 4.099274166416102, + "tokens_seen": 351666176 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045122367101303913, + "loss": 1.6069, + "theoretical_loss": 4.099187457510595, + "tokens_seen": 351731712 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004512136409227683, + "loss": 1.5677, + "theoretical_loss": 4.099100769282221, + "tokens_seen": 351797248 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004512036108324975, + "loss": 1.5319, + "theoretical_loss": 4.099014101722196, + "tokens_seen": 351862784 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045119358074222673, + "loss": 1.5613, + "theoretical_loss": 4.098927454821747, + "tokens_seen": 351928320 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045118355065195586, + "loss": 1.4478, + "theoretical_loss": 4.098840828572101, + "tokens_seen": 351993856 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004511735205616851, + "loss": 1.6299, + "theoretical_loss": 4.0987542229644935, + "tokens_seen": 352059392 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004511634904714143, + "loss": 1.5931, + "theoretical_loss": 4.0986676379901645, + "tokens_seen": 352124928 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045115346038114345, + "loss": 1.5762, + "theoretical_loss": 4.098581073640361, + "tokens_seen": 352190464 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045114343029087264, + "loss": 1.6273, + "theoretical_loss": 4.098494529906331, + "tokens_seen": 352256000 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004511334002006018, + "loss": 1.7129, + "theoretical_loss": 4.098408006779332, + "tokens_seen": 352321536 + }, + { + "epoch": 0.11, + "learning_rate": 0.000451123370110331, + "loss": 1.5016, + "theoretical_loss": 4.098321504250626, + "tokens_seen": 352387072 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045111334002006023, + "loss": 1.524, + "theoretical_loss": 4.098235022311479, + "tokens_seen": 352452608 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045110330992978936, + "loss": 1.6896, + "theoretical_loss": 4.098148560953163, + "tokens_seen": 352518144 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004510932798395186, + "loss": 1.4581, + "theoretical_loss": 4.098062120166956, + "tokens_seen": 352583680 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004510832497492477, + "loss": 1.5437, + "theoretical_loss": 4.097975699944141, + "tokens_seen": 352649216 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045107321965897696, + "loss": 1.4746, + "theoretical_loss": 4.097889300276005, + "tokens_seen": 352714752 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045106318956870614, + "loss": 1.5003, + "theoretical_loss": 4.097802921153842, + "tokens_seen": 352780288 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004510531594784353, + "loss": 1.5524, + "theoretical_loss": 4.097716562568951, + "tokens_seen": 352845824 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004510431293881645, + "loss": 1.512, + "theoretical_loss": 4.097630224512635, + "tokens_seen": 352911360 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045103309929789374, + "loss": 1.5676, + "theoretical_loss": 4.097543906976204, + "tokens_seen": 352976896 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045102306920762286, + "loss": 1.5536, + "theoretical_loss": 4.097457609950973, + "tokens_seen": 353042432 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004510130391173521, + "loss": 1.4303, + "theoretical_loss": 4.09737133342826, + "tokens_seen": 353107968 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004510030090270812, + "loss": 1.5323, + "theoretical_loss": 4.097285077399392, + "tokens_seen": 353173504 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045099297893681046, + "loss": 1.5401, + "theoretical_loss": 4.097198841855699, + "tokens_seen": 353239040 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045098294884653964, + "loss": 1.5319, + "theoretical_loss": 4.097112626788516, + "tokens_seen": 353304576 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004509729187562688, + "loss": 1.4244, + "theoretical_loss": 4.097026432189185, + "tokens_seen": 353370112 + }, + { + "epoch": 0.11, + "learning_rate": 0.000450962888665998, + "loss": 1.6138, + "theoretical_loss": 4.096940258049052, + "tokens_seen": 353435648 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004509528585757272, + "loss": 1.5574, + "theoretical_loss": 4.096854104359467, + "tokens_seen": 353501184 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045094282848545637, + "loss": 1.5093, + "theoretical_loss": 4.096767971111789, + "tokens_seen": 353566720 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004509327983951856, + "loss": 1.5065, + "theoretical_loss": 4.096681858297378, + "tokens_seen": 353632256 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045092276830491473, + "loss": 1.4662, + "theoretical_loss": 4.096595765907603, + "tokens_seen": 353697792 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045091273821464396, + "loss": 1.6537, + "theoretical_loss": 4.096509693933835, + "tokens_seen": 353763328 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045090270812437314, + "loss": 1.6065, + "theoretical_loss": 4.096423642367453, + "tokens_seen": 353828864 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 123582, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5701985359191895, + "objective/train/theoretical_loss": 4.096337611199838, + "objective/train/tokens_used": 353894400, + "theoretical_loss": 4.096337611199838, + "tokens_seen": 353894400 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004508926780341023, + "loss": 1.6235, + "theoretical_loss": 4.096337611199838, + "tokens_seen": 353894400 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004508826479438315, + "loss": 1.5227, + "theoretical_loss": 4.096251600422381, + "tokens_seen": 353959936 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004508726178535607, + "loss": 1.5594, + "theoretical_loss": 4.096165610026474, + "tokens_seen": 354025472 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045086258776328987, + "loss": 1.576, + "theoretical_loss": 4.096079640003515, + "tokens_seen": 354091008 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004508525576730191, + "loss": 1.5002, + "theoretical_loss": 4.095993690344908, + "tokens_seen": 354156544 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045084252758274823, + "loss": 1.4621, + "theoretical_loss": 4.0959077610420636, + "tokens_seen": 354222080 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045083249749247747, + "loss": 1.6288, + "theoretical_loss": 4.095821852086394, + "tokens_seen": 354287616 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004508224674022066, + "loss": 1.5624, + "theoretical_loss": 4.095735963469322, + "tokens_seen": 354353152 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045081243731193583, + "loss": 1.526, + "theoretical_loss": 4.095650095182268, + "tokens_seen": 354418688 + }, + { + "epoch": 0.11, + "learning_rate": 0.000450802407221665, + "loss": 1.4962, + "theoretical_loss": 4.0955642472166645, + "tokens_seen": 354484224 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004507923771313942, + "loss": 1.654, + "theoretical_loss": 4.095478419563947, + "tokens_seen": 354549760 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045078234704112337, + "loss": 1.6113, + "theoretical_loss": 4.095392612215555, + "tokens_seen": 354615296 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045077231695085255, + "loss": 1.4832, + "theoretical_loss": 4.095306825162933, + "tokens_seen": 354680832 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045076228686058173, + "loss": 1.5627, + "theoretical_loss": 4.095221058397534, + "tokens_seen": 354746368 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045075225677031097, + "loss": 1.5206, + "theoretical_loss": 4.095135311910812, + "tokens_seen": 354811904 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004507422266800401, + "loss": 1.6112, + "theoretical_loss": 4.095049585694229, + "tokens_seen": 354877440 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045073219658976933, + "loss": 1.4829, + "theoretical_loss": 4.09496387973925, + "tokens_seen": 354942976 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004507221664994985, + "loss": 1.5501, + "theoretical_loss": 4.094878194037348, + "tokens_seen": 355008512 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004507121364092277, + "loss": 1.5152, + "theoretical_loss": 4.094792528579999, + "tokens_seen": 355074048 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004507021063189569, + "loss": 1.5963, + "theoretical_loss": 4.094706883358684, + "tokens_seen": 355139584 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045069207622868606, + "loss": 1.6573, + "theoretical_loss": 4.09462125836489, + "tokens_seen": 355205120 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045068204613841524, + "loss": 1.5778, + "theoretical_loss": 4.0945356535901105, + "tokens_seen": 355270656 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004506720160481445, + "loss": 1.645, + "theoretical_loss": 4.094450069025841, + "tokens_seen": 355336192 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004506619859578736, + "loss": 1.5367, + "theoretical_loss": 4.094364504663584, + "tokens_seen": 355401728 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045065195586760284, + "loss": 1.5004, + "theoretical_loss": 4.094278960494847, + "tokens_seen": 355467264 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045064192577733196, + "loss": 1.5522, + "theoretical_loss": 4.094193436511143, + "tokens_seen": 355532800 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004506318956870612, + "loss": 1.6221, + "theoretical_loss": 4.094107932703991, + "tokens_seen": 355598336 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004506218655967904, + "loss": 1.5044, + "theoretical_loss": 4.09402244906491, + "tokens_seen": 355663872 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045061183550651956, + "loss": 1.5314, + "theoretical_loss": 4.093936985585432, + "tokens_seen": 355729408 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045060180541624874, + "loss": 1.5585, + "theoretical_loss": 4.093851542257088, + "tokens_seen": 355794944 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004505917753259779, + "loss": 1.5467, + "theoretical_loss": 4.093766119071416, + "tokens_seen": 355860480 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004505817452357071, + "loss": 1.5573, + "theoretical_loss": 4.093680716019961, + "tokens_seen": 355926016 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045057171514543634, + "loss": 1.4683, + "theoretical_loss": 4.093595333094269, + "tokens_seen": 355991552 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045056168505516547, + "loss": 1.4855, + "theoretical_loss": 4.093509970285895, + "tokens_seen": 356057088 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004505516549648947, + "loss": 1.5711, + "theoretical_loss": 4.093424627586399, + "tokens_seen": 356122624 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004505416248746239, + "loss": 1.6613, + "theoretical_loss": 4.093339304987341, + "tokens_seen": 356188160 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045053159478435306, + "loss": 1.5612, + "theoretical_loss": 4.093254002480293, + "tokens_seen": 356253696 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045052156469408224, + "loss": 1.5888, + "theoretical_loss": 4.093168720056828, + "tokens_seen": 356319232 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004505115346038114, + "loss": 1.5492, + "theoretical_loss": 4.093083457708524, + "tokens_seen": 356384768 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004505015045135406, + "loss": 1.5609, + "theoretical_loss": 4.092998215426967, + "tokens_seen": 356450304 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045049147442326984, + "loss": 1.6102, + "theoretical_loss": 4.092912993203744, + "tokens_seen": 356515840 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045048144433299897, + "loss": 1.5154, + "theoretical_loss": 4.09282779103045, + "tokens_seen": 356581376 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004504714142427282, + "loss": 1.5172, + "theoretical_loss": 4.092742608898685, + "tokens_seen": 356646912 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045046138415245733, + "loss": 1.7942, + "theoretical_loss": 4.092657446800052, + "tokens_seen": 356712448 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045045135406218657, + "loss": 1.5954, + "theoretical_loss": 4.092572304726161, + "tokens_seen": 356777984 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004504413239719158, + "loss": 1.5975, + "theoretical_loss": 4.092487182668627, + "tokens_seen": 356843520 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045043129388164493, + "loss": 1.4778, + "theoretical_loss": 4.092402080619069, + "tokens_seen": 356909056 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045042126379137416, + "loss": 1.6664, + "theoretical_loss": 4.09231699856911, + "tokens_seen": 356974592 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045041123370110334, + "loss": 1.4746, + "theoretical_loss": 4.092231936510383, + "tokens_seen": 357040128 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004504012036108325, + "loss": 1.6017, + "theoretical_loss": 4.09214689443452, + "tokens_seen": 357105664 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 124803, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6355619430541992, + "objective/train/theoretical_loss": 4.0920618723331605, + "objective/train/tokens_used": 357171200, + "theoretical_loss": 4.0920618723331605, + "tokens_seen": 357171200 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004503911735205617, + "loss": 1.6165, + "theoretical_loss": 4.0920618723331605, + "tokens_seen": 357171200 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004503811434302909, + "loss": 1.6085, + "theoretical_loss": 4.091976870197951, + "tokens_seen": 357236736 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045037111334002007, + "loss": 1.5419, + "theoretical_loss": 4.091891888020539, + "tokens_seen": 357302272 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004503610832497493, + "loss": 1.6157, + "theoretical_loss": 4.091806925792582, + "tokens_seen": 357367808 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045035105315947843, + "loss": 1.5786, + "theoretical_loss": 4.091721983505737, + "tokens_seen": 357433344 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045034102306920767, + "loss": 1.5588, + "theoretical_loss": 4.091637061151671, + "tokens_seen": 357498880 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004503309929789368, + "loss": 1.6274, + "theoretical_loss": 4.091552158722052, + "tokens_seen": 357564416 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045032096288866603, + "loss": 1.5065, + "theoretical_loss": 4.091467276208554, + "tokens_seen": 357629952 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004503109327983952, + "loss": 1.5213, + "theoretical_loss": 4.091382413602861, + "tokens_seen": 357695488 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004503009027081244, + "loss": 1.5664, + "theoretical_loss": 4.0912975708966535, + "tokens_seen": 357761024 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045029087261785357, + "loss": 1.5835, + "theoretical_loss": 4.091212748081622, + "tokens_seen": 357826560 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045028084252758275, + "loss": 1.6185, + "theoretical_loss": 4.091127945149463, + "tokens_seen": 357892096 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045027081243731193, + "loss": 1.4874, + "theoretical_loss": 4.091043162091876, + "tokens_seen": 357957632 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045026078234704117, + "loss": 1.5879, + "theoretical_loss": 4.090958398900564, + "tokens_seen": 358023168 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004502507522567703, + "loss": 1.673, + "theoretical_loss": 4.090873655567238, + "tokens_seen": 358088704 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045024072216649953, + "loss": 1.4972, + "theoretical_loss": 4.090788932083613, + "tokens_seen": 358154240 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004502306920762287, + "loss": 1.5225, + "theoretical_loss": 4.090704228441408, + "tokens_seen": 358219776 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004502206619859579, + "loss": 1.649, + "theoretical_loss": 4.090619544632347, + "tokens_seen": 358285312 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004502106318956871, + "loss": 1.5802, + "theoretical_loss": 4.0905348806481605, + "tokens_seen": 358350848 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045020060180541626, + "loss": 1.4518, + "theoretical_loss": 4.090450236480583, + "tokens_seen": 358416384 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045019057171514544, + "loss": 1.4866, + "theoretical_loss": 4.090365612121355, + "tokens_seen": 358481920 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004501805416248747, + "loss": 1.5242, + "theoretical_loss": 4.0902810075622185, + "tokens_seen": 358547456 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004501705115346038, + "loss": 1.5772, + "theoretical_loss": 4.090196422794925, + "tokens_seen": 358612992 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045016048144433304, + "loss": 1.498, + "theoretical_loss": 4.0901118578112285, + "tokens_seen": 358678528 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045015045135406216, + "loss": 1.4635, + "theoretical_loss": 4.090027312602888, + "tokens_seen": 358744064 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004501404212637914, + "loss": 1.5891, + "theoretical_loss": 4.089942787161667, + "tokens_seen": 358809600 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004501303911735206, + "loss": 1.3697, + "theoretical_loss": 4.089858281479335, + "tokens_seen": 358875136 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045012036108324976, + "loss": 1.5907, + "theoretical_loss": 4.089773795547668, + "tokens_seen": 358940672 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045011033099297894, + "loss": 1.5886, + "theoretical_loss": 4.089689329358443, + "tokens_seen": 359006208 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004501003009027081, + "loss": 1.5845, + "theoretical_loss": 4.089604882903444, + "tokens_seen": 359071744 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004500902708124373, + "loss": 1.5623, + "theoretical_loss": 4.089520456174461, + "tokens_seen": 359137280 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045008024072216654, + "loss": 1.5522, + "theoretical_loss": 4.089436049163287, + "tokens_seen": 359202816 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045007021063189567, + "loss": 1.5097, + "theoretical_loss": 4.089351661861722, + "tokens_seen": 359268352 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004500601805416249, + "loss": 1.5967, + "theoretical_loss": 4.089267294261566, + "tokens_seen": 359333888 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004500501504513541, + "loss": 1.5708, + "theoretical_loss": 4.089182946354631, + "tokens_seen": 359399424 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045004012036108326, + "loss": 1.4238, + "theoretical_loss": 4.08909861813273, + "tokens_seen": 359464960 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045003009027081244, + "loss": 1.4989, + "theoretical_loss": 4.0890143095876805, + "tokens_seen": 359530496 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004500200601805416, + "loss": 1.4486, + "theoretical_loss": 4.088930020711305, + "tokens_seen": 359596032 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004500100300902708, + "loss": 1.5798, + "theoretical_loss": 4.088845751495433, + "tokens_seen": 359661568 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045000000000000004, + "loss": 1.6207, + "theoretical_loss": 4.088761501931897, + "tokens_seen": 359727104 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044998996990972917, + "loss": 1.5856, + "theoretical_loss": 4.088677272012535, + "tokens_seen": 359792640 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004499799398194584, + "loss": 1.5541, + "theoretical_loss": 4.088593061729188, + "tokens_seen": 359858176 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044996990972918753, + "loss": 1.4925, + "theoretical_loss": 4.0885088710737065, + "tokens_seen": 359923712 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044995987963891677, + "loss": 1.5874, + "theoretical_loss": 4.088424700037941, + "tokens_seen": 359989248 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044994984954864595, + "loss": 1.5248, + "theoretical_loss": 4.08834054861375, + "tokens_seen": 360054784 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044993981945837513, + "loss": 1.6519, + "theoretical_loss": 4.088256416792994, + "tokens_seen": 360120320 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004499297893681043, + "loss": 1.4692, + "theoretical_loss": 4.088172304567541, + "tokens_seen": 360185856 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044991975927783355, + "loss": 1.6297, + "theoretical_loss": 4.088088211929263, + "tokens_seen": 360251392 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044990972918756267, + "loss": 1.7128, + "theoretical_loss": 4.088004138870037, + "tokens_seen": 360316928 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004498996990972919, + "loss": 1.6464, + "theoretical_loss": 4.087920085381745, + "tokens_seen": 360382464 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 126091, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5474098920822144, + "objective/train/theoretical_loss": 4.0878360514562715, + "objective/train/tokens_used": 360448000, + "theoretical_loss": 4.0878360514562715, + "tokens_seen": 360448000 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044988966900702103, + "loss": 1.5425, + "theoretical_loss": 4.0878360514562715, + "tokens_seen": 360448000 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044987963891675027, + "loss": 1.5086, + "theoretical_loss": 4.087752037085509, + "tokens_seen": 360513536 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044986960882647945, + "loss": 1.5521, + "theoretical_loss": 4.087668042261353, + "tokens_seen": 360579072 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044985957873620863, + "loss": 1.5649, + "theoretical_loss": 4.087584066975706, + "tokens_seen": 360644608 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004498495486459378, + "loss": 1.6492, + "theoretical_loss": 4.087500111220472, + "tokens_seen": 360710144 + }, + { + "epoch": 0.11, + "learning_rate": 0.000449839518555667, + "loss": 1.445, + "theoretical_loss": 4.087416174987563, + "tokens_seen": 360775680 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004498294884653962, + "loss": 1.5976, + "theoretical_loss": 4.087332258268893, + "tokens_seen": 360841216 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004498194583751254, + "loss": 1.5866, + "theoretical_loss": 4.087248361056384, + "tokens_seen": 360906752 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044980942828485454, + "loss": 1.5621, + "theoretical_loss": 4.087164483341959, + "tokens_seen": 360972288 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044979939819458377, + "loss": 1.5998, + "theoretical_loss": 4.087080625117549, + "tokens_seen": 361037824 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004497893681043129, + "loss": 1.5553, + "theoretical_loss": 4.086996786375089, + "tokens_seen": 361103360 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044977933801404213, + "loss": 1.6413, + "theoretical_loss": 4.0869129671065165, + "tokens_seen": 361168896 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004497693079237713, + "loss": 1.5333, + "theoretical_loss": 4.086829167303778, + "tokens_seen": 361234432 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004497592778335005, + "loss": 1.4999, + "theoretical_loss": 4.086745386958821, + "tokens_seen": 361299968 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004497492477432297, + "loss": 1.4616, + "theoretical_loss": 4.0866616260636, + "tokens_seen": 361365504 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004497392176529589, + "loss": 1.5814, + "theoretical_loss": 4.086577884610074, + "tokens_seen": 361431040 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044972918756268804, + "loss": 1.6191, + "theoretical_loss": 4.086494162590206, + "tokens_seen": 361496576 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004497191574724173, + "loss": 1.5831, + "theoretical_loss": 4.086410459995964, + "tokens_seen": 361562112 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004497091273821464, + "loss": 1.5419, + "theoretical_loss": 4.0863267768193205, + "tokens_seen": 361627648 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044969909729187564, + "loss": 1.6164, + "theoretical_loss": 4.086243113052254, + "tokens_seen": 361693184 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004496890672016049, + "loss": 1.5787, + "theoretical_loss": 4.0861594686867475, + "tokens_seen": 361758720 + }, + { + "epoch": 0.11, + "learning_rate": 0.000449679037111334, + "loss": 1.4165, + "theoretical_loss": 4.086075843714786, + "tokens_seen": 361824256 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044966900702106324, + "loss": 1.545, + "theoretical_loss": 4.085992238128364, + "tokens_seen": 361889792 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044965897693079236, + "loss": 1.6059, + "theoretical_loss": 4.085908651919477, + "tokens_seen": 361955328 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004496489468405216, + "loss": 1.7121, + "theoretical_loss": 4.085825085080128, + "tokens_seen": 362020864 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004496389167502508, + "loss": 1.5628, + "theoretical_loss": 4.08574153760232, + "tokens_seen": 362086400 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044962888665997996, + "loss": 1.6078, + "theoretical_loss": 4.085658009478067, + "tokens_seen": 362151936 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044961885656970914, + "loss": 1.4843, + "theoretical_loss": 4.0855745006993835, + "tokens_seen": 362217472 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004496088264794383, + "loss": 1.7199, + "theoretical_loss": 4.08549101125829, + "tokens_seen": 362283008 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004495987963891675, + "loss": 1.5691, + "theoretical_loss": 4.085407541146812, + "tokens_seen": 362348544 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044958876629889674, + "loss": 1.5746, + "theoretical_loss": 4.085324090356979, + "tokens_seen": 362414080 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044957873620862587, + "loss": 1.5743, + "theoretical_loss": 4.085240658880827, + "tokens_seen": 362479616 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004495687061183551, + "loss": 1.4746, + "theoretical_loss": 4.085157246710394, + "tokens_seen": 362545152 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004495586760280843, + "loss": 1.4852, + "theoretical_loss": 4.0850738538377245, + "tokens_seen": 362610688 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044954864593781346, + "loss": 1.5581, + "theoretical_loss": 4.084990480254867, + "tokens_seen": 362676224 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044953861584754264, + "loss": 1.6136, + "theoretical_loss": 4.084907125953876, + "tokens_seen": 362741760 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004495285857572718, + "loss": 1.5929, + "theoretical_loss": 4.0848237909268095, + "tokens_seen": 362807296 + }, + { + "epoch": 0.11, + "learning_rate": 0.000449518555667001, + "loss": 1.5237, + "theoretical_loss": 4.084740475165729, + "tokens_seen": 362872832 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044950852557673024, + "loss": 1.5097, + "theoretical_loss": 4.084657178662704, + "tokens_seen": 362938368 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044949849548645937, + "loss": 1.5293, + "theoretical_loss": 4.084573901409807, + "tokens_seen": 363003904 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004494884653961886, + "loss": 1.5142, + "theoretical_loss": 4.084490643399113, + "tokens_seen": 363069440 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044947843530591773, + "loss": 1.4847, + "theoretical_loss": 4.084407404622706, + "tokens_seen": 363134976 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044946840521564697, + "loss": 1.4854, + "theoretical_loss": 4.084324185072671, + "tokens_seen": 363200512 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044945837512537615, + "loss": 1.4949, + "theoretical_loss": 4.084240984741099, + "tokens_seen": 363266048 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044944834503510533, + "loss": 1.481, + "theoretical_loss": 4.084157803620086, + "tokens_seen": 363331584 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004494383149448345, + "loss": 1.522, + "theoretical_loss": 4.084074641701734, + "tokens_seen": 363397120 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044942828485456375, + "loss": 1.5929, + "theoretical_loss": 4.083991498978146, + "tokens_seen": 363462656 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044941825476429287, + "loss": 1.5507, + "theoretical_loss": 4.083908375441433, + "tokens_seen": 363528192 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004494082246740221, + "loss": 1.4718, + "theoretical_loss": 4.083825271083709, + "tokens_seen": 363593728 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044939819458375123, + "loss": 1.5024, + "theoretical_loss": 4.083742185897093, + "tokens_seen": 363659264 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 126988, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.7686326503753662, + "objective/train/theoretical_loss": 4.08365911987371, + "objective/train/tokens_used": 363724800, + "theoretical_loss": 4.08365911987371, + "tokens_seen": 363724800 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044938816449348047, + "loss": 1.6423, + "theoretical_loss": 4.08365911987371, + "tokens_seen": 363724800 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044937813440320965, + "loss": 1.6174, + "theoretical_loss": 4.083576073005687, + "tokens_seen": 363790336 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044936810431293883, + "loss": 1.503, + "theoretical_loss": 4.083493045285158, + "tokens_seen": 363855872 + }, + { + "epoch": 0.11, + "learning_rate": 0.000449358074222668, + "loss": 1.5769, + "theoretical_loss": 4.08341003670426, + "tokens_seen": 363921408 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004493480441323972, + "loss": 1.4759, + "theoretical_loss": 4.0833270472551355, + "tokens_seen": 363986944 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004493380140421264, + "loss": 1.5533, + "theoretical_loss": 4.083244076929932, + "tokens_seen": 364052480 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004493279839518556, + "loss": 1.5346, + "theoretical_loss": 4.083161125720801, + "tokens_seen": 364118016 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044931795386158474, + "loss": 1.6489, + "theoretical_loss": 4.083078193619899, + "tokens_seen": 364183552 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044930792377131397, + "loss": 1.6187, + "theoretical_loss": 4.082995280619386, + "tokens_seen": 364249088 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004492978936810431, + "loss": 1.5457, + "theoretical_loss": 4.0829123867114285, + "tokens_seen": 364314624 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044928786359077234, + "loss": 1.6712, + "theoretical_loss": 4.082829511888197, + "tokens_seen": 364380160 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004492778335005015, + "loss": 1.7054, + "theoretical_loss": 4.082746656141865, + "tokens_seen": 364445696 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004492678034102307, + "loss": 1.5925, + "theoretical_loss": 4.082663819464613, + "tokens_seen": 364511232 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004492577733199599, + "loss": 1.5323, + "theoretical_loss": 4.082581001848624, + "tokens_seen": 364576768 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004492477432296891, + "loss": 1.5417, + "theoretical_loss": 4.082498203286088, + "tokens_seen": 364642304 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044923771313941824, + "loss": 1.6421, + "theoretical_loss": 4.082415423769197, + "tokens_seen": 364707840 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004492276830491475, + "loss": 1.6375, + "theoretical_loss": 4.082332663290149, + "tokens_seen": 364773376 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004492176529588766, + "loss": 1.5082, + "theoretical_loss": 4.082249921841147, + "tokens_seen": 364838912 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044920762286860584, + "loss": 1.6361, + "theoretical_loss": 4.082167199414398, + "tokens_seen": 364904448 + }, + { + "epoch": 0.11, + "learning_rate": 0.000449197592778335, + "loss": 1.5691, + "theoretical_loss": 4.082084496002113, + "tokens_seen": 364969984 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004491875626880642, + "loss": 1.5412, + "theoretical_loss": 4.082001811596509, + "tokens_seen": 365035520 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004491775325977934, + "loss": 1.6397, + "theoretical_loss": 4.081919146189806, + "tokens_seen": 365101056 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044916750250752256, + "loss": 1.5986, + "theoretical_loss": 4.08183649977423, + "tokens_seen": 365166592 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044915747241725174, + "loss": 1.5451, + "theoretical_loss": 4.08175387234201, + "tokens_seen": 365232128 + }, + { + "epoch": 0.11, + "learning_rate": 0.000449147442326981, + "loss": 1.4709, + "theoretical_loss": 4.081671263885381, + "tokens_seen": 365297664 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004491374122367101, + "loss": 1.4173, + "theoretical_loss": 4.081588674396583, + "tokens_seen": 365363200 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044912738214643934, + "loss": 1.6548, + "theoretical_loss": 4.081506103867858, + "tokens_seen": 365428736 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044911735205616847, + "loss": 1.5756, + "theoretical_loss": 4.081423552291456, + "tokens_seen": 365494272 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004491073219658977, + "loss": 1.4895, + "theoretical_loss": 4.081341019659628, + "tokens_seen": 365559808 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004490972918756269, + "loss": 1.554, + "theoretical_loss": 4.081258505964633, + "tokens_seen": 365625344 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044908726178535607, + "loss": 1.6263, + "theoretical_loss": 4.081176011198731, + "tokens_seen": 365690880 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044907723169508525, + "loss": 1.6191, + "theoretical_loss": 4.08109353535419, + "tokens_seen": 365756416 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004490672016048145, + "loss": 1.5593, + "theoretical_loss": 4.0810110784232805, + "tokens_seen": 365821952 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004490571715145436, + "loss": 1.5679, + "theoretical_loss": 4.080928640398278, + "tokens_seen": 365887488 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044904714142427284, + "loss": 1.5465, + "theoretical_loss": 4.080846221271461, + "tokens_seen": 365953024 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044903711133400197, + "loss": 1.5551, + "theoretical_loss": 4.0807638210351165, + "tokens_seen": 366018560 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004490270812437312, + "loss": 1.6805, + "theoretical_loss": 4.080681439681533, + "tokens_seen": 366084096 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004490170511534604, + "loss": 1.5005, + "theoretical_loss": 4.080599077203003, + "tokens_seen": 366149632 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044900702106318957, + "loss": 1.5862, + "theoretical_loss": 4.080516733591827, + "tokens_seen": 366215168 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044899699097291875, + "loss": 1.5727, + "theoretical_loss": 4.080434408840304, + "tokens_seen": 366280704 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044898696088264793, + "loss": 1.4679, + "theoretical_loss": 4.080352102940744, + "tokens_seen": 366346240 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004489769307923771, + "loss": 1.4936, + "theoretical_loss": 4.080269815885458, + "tokens_seen": 366411776 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044896690070210635, + "loss": 1.477, + "theoretical_loss": 4.080187547666763, + "tokens_seen": 366477312 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004489568706118355, + "loss": 1.6069, + "theoretical_loss": 4.080105298276978, + "tokens_seen": 366542848 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004489468405215647, + "loss": 1.673, + "theoretical_loss": 4.08002306770843, + "tokens_seen": 366608384 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044893681043129395, + "loss": 1.5075, + "theoretical_loss": 4.079940855953447, + "tokens_seen": 366673920 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044892678034102307, + "loss": 1.4959, + "theoretical_loss": 4.079858663004364, + "tokens_seen": 366739456 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004489167502507523, + "loss": 1.5827, + "theoretical_loss": 4.079776488853521, + "tokens_seen": 366804992 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044890672016048143, + "loss": 1.5781, + "theoretical_loss": 4.079694333493259, + "tokens_seen": 366870528 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044889669007021067, + "loss": 1.4909, + "theoretical_loss": 4.079612196915927, + "tokens_seen": 366936064 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 128155, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5053813457489014, + "objective/train/theoretical_loss": 4.079530079113876, + "objective/train/tokens_used": 367001600, + "theoretical_loss": 4.079530079113876, + "tokens_seen": 367001600 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044888665997993985, + "loss": 1.6068, + "theoretical_loss": 4.079530079113876, + "tokens_seen": 367001600 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044887662988966903, + "loss": 1.5385, + "theoretical_loss": 4.079447980079465, + "tokens_seen": 367067136 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004488665997993982, + "loss": 1.5331, + "theoretical_loss": 4.079365899805053, + "tokens_seen": 367132672 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004488565697091274, + "loss": 1.6098, + "theoretical_loss": 4.079283838283007, + "tokens_seen": 367198208 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004488465396188566, + "loss": 1.4712, + "theoretical_loss": 4.079201795505695, + "tokens_seen": 367263744 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004488365095285858, + "loss": 1.4846, + "theoretical_loss": 4.079119771465494, + "tokens_seen": 367329280 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044882647943831494, + "loss": 1.6004, + "theoretical_loss": 4.079037766154781, + "tokens_seen": 367394816 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004488164493480442, + "loss": 1.6359, + "theoretical_loss": 4.078955779565941, + "tokens_seen": 367460352 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004488064192577733, + "loss": 1.562, + "theoretical_loss": 4.078873811691361, + "tokens_seen": 367525888 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044879638916750254, + "loss": 1.4714, + "theoretical_loss": 4.078791862523433, + "tokens_seen": 367591424 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004487863590772317, + "loss": 1.6433, + "theoretical_loss": 4.078709932054554, + "tokens_seen": 367656960 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004487763289869609, + "loss": 1.5743, + "theoretical_loss": 4.078628020277128, + "tokens_seen": 367722496 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004487662988966901, + "loss": 1.5609, + "theoretical_loss": 4.078546127183556, + "tokens_seen": 367788032 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004487562688064193, + "loss": 1.5655, + "theoretical_loss": 4.078464252766252, + "tokens_seen": 367853568 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044874623871614844, + "loss": 1.473, + "theoretical_loss": 4.078382397017628, + "tokens_seen": 367919104 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004487362086258777, + "loss": 1.604, + "theoretical_loss": 4.0783005599301045, + "tokens_seen": 367984640 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004487261785356068, + "loss": 1.5474, + "theoretical_loss": 4.078218741496104, + "tokens_seen": 368050176 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044871614844533604, + "loss": 1.6658, + "theoretical_loss": 4.078136941708055, + "tokens_seen": 368115712 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004487061183550652, + "loss": 1.5457, + "theoretical_loss": 4.078055160558391, + "tokens_seen": 368181248 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004486960882647944, + "loss": 1.5504, + "theoretical_loss": 4.0779733980395445, + "tokens_seen": 368246784 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004486860581745236, + "loss": 1.5033, + "theoretical_loss": 4.077891654143961, + "tokens_seen": 368312320 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044867602808425276, + "loss": 1.5577, + "theoretical_loss": 4.0778099288640846, + "tokens_seen": 368377856 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044866599799398194, + "loss": 1.5948, + "theoretical_loss": 4.0777282221923645, + "tokens_seen": 368443392 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004486559679037112, + "loss": 1.5442, + "theoretical_loss": 4.077646534121256, + "tokens_seen": 368508928 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004486459378134403, + "loss": 1.485, + "theoretical_loss": 4.077564864643217, + "tokens_seen": 368574464 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044863590772316954, + "loss": 1.582, + "theoretical_loss": 4.077483213750711, + "tokens_seen": 368640000 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044862587763289867, + "loss": 1.6916, + "theoretical_loss": 4.077401581436206, + "tokens_seen": 368705536 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004486158475426279, + "loss": 1.4316, + "theoretical_loss": 4.077319967692173, + "tokens_seen": 368771072 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004486058174523571, + "loss": 1.6669, + "theoretical_loss": 4.077238372511088, + "tokens_seen": 368836608 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044859578736208627, + "loss": 1.5811, + "theoretical_loss": 4.077156795885434, + "tokens_seen": 368902144 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044858575727181545, + "loss": 1.577, + "theoretical_loss": 4.077075237807693, + "tokens_seen": 368967680 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004485757271815447, + "loss": 1.5657, + "theoretical_loss": 4.076993698270357, + "tokens_seen": 369033216 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004485656970912738, + "loss": 1.7051, + "theoretical_loss": 4.0769121772659185, + "tokens_seen": 369098752 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044855566700100304, + "loss": 1.6442, + "theoretical_loss": 4.076830674786876, + "tokens_seen": 369164288 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044854563691073217, + "loss": 1.4998, + "theoretical_loss": 4.076749190825732, + "tokens_seen": 369229824 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004485356068204614, + "loss": 1.5547, + "theoretical_loss": 4.076667725374993, + "tokens_seen": 369295360 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004485255767301906, + "loss": 1.5236, + "theoretical_loss": 4.076586278427171, + "tokens_seen": 369360896 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044851554663991977, + "loss": 1.5498, + "theoretical_loss": 4.076504849974781, + "tokens_seen": 369426432 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044850551654964895, + "loss": 1.5957, + "theoretical_loss": 4.076423440010343, + "tokens_seen": 369491968 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044849548645937813, + "loss": 1.5717, + "theoretical_loss": 4.076342048526382, + "tokens_seen": 369557504 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004484854563691073, + "loss": 1.5721, + "theoretical_loss": 4.076260675515427, + "tokens_seen": 369623040 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044847542627883655, + "loss": 1.6444, + "theoretical_loss": 4.07617932097001, + "tokens_seen": 369688576 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004484653961885657, + "loss": 1.4873, + "theoretical_loss": 4.076097984882669, + "tokens_seen": 369754112 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004484553660982949, + "loss": 1.4784, + "theoretical_loss": 4.076016667245945, + "tokens_seen": 369819648 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044844533600802404, + "loss": 1.4472, + "theoretical_loss": 4.075935368052384, + "tokens_seen": 369885184 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044843530591775327, + "loss": 1.5823, + "theoretical_loss": 4.075854087294538, + "tokens_seen": 369950720 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044842527582748245, + "loss": 1.6089, + "theoretical_loss": 4.075772824964961, + "tokens_seen": 370016256 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044841524573721163, + "loss": 1.6589, + "theoretical_loss": 4.07569158105621, + "tokens_seen": 370081792 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004484052156469408, + "loss": 1.531, + "theoretical_loss": 4.075610355560851, + "tokens_seen": 370147328 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044839518555667005, + "loss": 1.3906, + "theoretical_loss": 4.075529148471451, + "tokens_seen": 370212864 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 129204, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5357906818389893, + "objective/train/theoretical_loss": 4.075447959780581, + "objective/train/tokens_used": 370278400, + "theoretical_loss": 4.075447959780581, + "tokens_seen": 370278400 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004483851554663992, + "loss": 1.4822, + "theoretical_loss": 4.075447959780581, + "tokens_seen": 370278400 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004483751253761284, + "loss": 1.5676, + "theoretical_loss": 4.075366789480818, + "tokens_seen": 370343936 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044836509528585754, + "loss": 1.4156, + "theoretical_loss": 4.075285637564743, + "tokens_seen": 370409472 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004483550651955868, + "loss": 1.4911, + "theoretical_loss": 4.07520450402494, + "tokens_seen": 370475008 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044834503510531596, + "loss": 1.5618, + "theoretical_loss": 4.075123388854, + "tokens_seen": 370540544 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044833500501504514, + "loss": 1.4961, + "theoretical_loss": 4.075042292044515, + "tokens_seen": 370606080 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004483249749247743, + "loss": 1.5328, + "theoretical_loss": 4.074961213589082, + "tokens_seen": 370671616 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004483149448345035, + "loss": 1.5201, + "theoretical_loss": 4.074880153480304, + "tokens_seen": 370737152 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004483049147442327, + "loss": 1.5803, + "theoretical_loss": 4.074799111710789, + "tokens_seen": 370802688 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004482948846539619, + "loss": 1.5299, + "theoretical_loss": 4.074718088273144, + "tokens_seen": 370868224 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044828485456369104, + "loss": 1.5152, + "theoretical_loss": 4.074637083159988, + "tokens_seen": 370933760 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004482748244734203, + "loss": 1.558, + "theoretical_loss": 4.0745560963639385, + "tokens_seen": 370999296 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004482647943831494, + "loss": 1.5045, + "theoretical_loss": 4.0744751278776175, + "tokens_seen": 371064832 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044825476429287864, + "loss": 1.5338, + "theoretical_loss": 4.074394177693655, + "tokens_seen": 371130368 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004482447342026078, + "loss": 1.508, + "theoretical_loss": 4.074313245804683, + "tokens_seen": 371195904 + }, + { + "epoch": 0.11, + "learning_rate": 0.000448234704112337, + "loss": 1.561, + "theoretical_loss": 4.074232332203336, + "tokens_seen": 371261440 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004482246740220662, + "loss": 1.4966, + "theoretical_loss": 4.074151436882256, + "tokens_seen": 371326976 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004482146439317954, + "loss": 1.4618, + "theoretical_loss": 4.074070559834088, + "tokens_seen": 371392512 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044820461384152455, + "loss": 1.6909, + "theoretical_loss": 4.073989701051481, + "tokens_seen": 371458048 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004481945837512538, + "loss": 1.4895, + "theoretical_loss": 4.073908860527087, + "tokens_seen": 371523584 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044818455366098296, + "loss": 1.5978, + "theoretical_loss": 4.0738280382535645, + "tokens_seen": 371589120 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044817452357071214, + "loss": 1.5329, + "theoretical_loss": 4.0737472342235765, + "tokens_seen": 371654656 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004481644934804414, + "loss": 1.5107, + "theoretical_loss": 4.073666448429787, + "tokens_seen": 371720192 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004481544633901705, + "loss": 1.4544, + "theoretical_loss": 4.073585680864868, + "tokens_seen": 371785728 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044814443329989974, + "loss": 1.5128, + "theoretical_loss": 4.073504931521493, + "tokens_seen": 371851264 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044813440320962887, + "loss": 1.5647, + "theoretical_loss": 4.073424200392342, + "tokens_seen": 371916800 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004481243731193581, + "loss": 1.595, + "theoretical_loss": 4.073343487470097, + "tokens_seen": 371982336 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004481143430290873, + "loss": 1.5401, + "theoretical_loss": 4.073262792747445, + "tokens_seen": 372047872 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044810431293881647, + "loss": 1.5256, + "theoretical_loss": 4.073182116217079, + "tokens_seen": 372113408 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044809428284854565, + "loss": 1.5136, + "theoretical_loss": 4.073101457871693, + "tokens_seen": 372178944 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004480842527582749, + "loss": 1.6419, + "theoretical_loss": 4.073020817703988, + "tokens_seen": 372244480 + }, + { + "epoch": 0.11, + "learning_rate": 0.000448074222668004, + "loss": 1.6195, + "theoretical_loss": 4.072940195706668, + "tokens_seen": 372310016 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044806419257773324, + "loss": 1.6731, + "theoretical_loss": 4.072859591872441, + "tokens_seen": 372375552 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044805416248746237, + "loss": 1.6158, + "theoretical_loss": 4.07277900619402, + "tokens_seen": 372441088 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004480441323971916, + "loss": 1.5217, + "theoretical_loss": 4.072698438664121, + "tokens_seen": 372506624 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004480341023069208, + "loss": 1.5108, + "theoretical_loss": 4.072617889275467, + "tokens_seen": 372572160 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044802407221664997, + "loss": 1.5261, + "theoretical_loss": 4.07253735802078, + "tokens_seen": 372637696 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044801404212637915, + "loss": 1.621, + "theoretical_loss": 4.072456844892792, + "tokens_seen": 372703232 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044800401203610833, + "loss": 1.5235, + "theoretical_loss": 4.072376349884236, + "tokens_seen": 372768768 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004479939819458375, + "loss": 1.5141, + "theoretical_loss": 4.072295872987848, + "tokens_seen": 372834304 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044798395185556675, + "loss": 1.5522, + "theoretical_loss": 4.072215414196373, + "tokens_seen": 372899840 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004479739217652959, + "loss": 1.5395, + "theoretical_loss": 4.072134973502554, + "tokens_seen": 372965376 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004479638916750251, + "loss": 1.555, + "theoretical_loss": 4.072054550899144, + "tokens_seen": 373030912 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044795386158475424, + "loss": 1.4694, + "theoretical_loss": 4.071974146378896, + "tokens_seen": 373096448 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044794383149448347, + "loss": 1.6317, + "theoretical_loss": 4.071893759934568, + "tokens_seen": 373161984 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044793380140421265, + "loss": 1.453, + "theoretical_loss": 4.0718133915589245, + "tokens_seen": 373227520 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044792377131394183, + "loss": 1.5169, + "theoretical_loss": 4.071733041244732, + "tokens_seen": 373293056 + }, + { + "epoch": 0.11, + "learning_rate": 0.000447913741223671, + "loss": 1.5593, + "theoretical_loss": 4.071652708984761, + "tokens_seen": 373358592 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044790371113340025, + "loss": 1.5924, + "theoretical_loss": 4.0715723947717875, + "tokens_seen": 373424128 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004478936810431294, + "loss": 1.4884, + "theoretical_loss": 4.07149209859859, + "tokens_seen": 373489664 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 130311, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5853087902069092, + "objective/train/theoretical_loss": 4.0714118204579535, + "objective/train/tokens_used": 373555200, + "theoretical_loss": 4.0714118204579535, + "tokens_seen": 373555200 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004478836509528586, + "loss": 1.5572, + "theoretical_loss": 4.0714118204579535, + "tokens_seen": 373555200 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044787362086258774, + "loss": 1.4252, + "theoretical_loss": 4.0713315603426645, + "tokens_seen": 373620736 + }, + { + "epoch": 0.11, + "learning_rate": 0.000447863590772317, + "loss": 1.4828, + "theoretical_loss": 4.071251318245516, + "tokens_seen": 373686272 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044785356068204616, + "loss": 1.5571, + "theoretical_loss": 4.071171094159304, + "tokens_seen": 373751808 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044784353059177534, + "loss": 1.6185, + "theoretical_loss": 4.071090888076828, + "tokens_seen": 373817344 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004478335005015045, + "loss": 1.4361, + "theoretical_loss": 4.071010699990892, + "tokens_seen": 373882880 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004478234704112337, + "loss": 1.6708, + "theoretical_loss": 4.070930529894306, + "tokens_seen": 373948416 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004478134403209629, + "loss": 1.5995, + "theoretical_loss": 4.070850377779882, + "tokens_seen": 374013952 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004478034102306921, + "loss": 1.5955, + "theoretical_loss": 4.070770243640436, + "tokens_seen": 374079488 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044779338014042124, + "loss": 1.5185, + "theoretical_loss": 4.07069012746879, + "tokens_seen": 374145024 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004477833500501505, + "loss": 1.445, + "theoretical_loss": 4.070610029257768, + "tokens_seen": 374210560 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004477733199598796, + "loss": 1.5628, + "theoretical_loss": 4.0705299490002, + "tokens_seen": 374276096 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044776328986960884, + "loss": 1.504, + "theoretical_loss": 4.070449886688919, + "tokens_seen": 374341632 + }, + { + "epoch": 0.11, + "learning_rate": 0.000447753259779338, + "loss": 1.5512, + "theoretical_loss": 4.0703698423167625, + "tokens_seen": 374407168 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004477432296890672, + "loss": 1.5205, + "theoretical_loss": 4.070289815876571, + "tokens_seen": 374472704 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004477331995987964, + "loss": 1.3962, + "theoretical_loss": 4.070209807361191, + "tokens_seen": 374538240 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004477231695085256, + "loss": 1.5547, + "theoretical_loss": 4.0701298167634725, + "tokens_seen": 374603776 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044771313941825475, + "loss": 1.6174, + "theoretical_loss": 4.0700498440762685, + "tokens_seen": 374669312 + }, + { + "epoch": 0.11, + "learning_rate": 0.000447703109327984, + "loss": 1.5031, + "theoretical_loss": 4.069969889292437, + "tokens_seen": 374734848 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004476930792377131, + "loss": 1.7249, + "theoretical_loss": 4.069889952404841, + "tokens_seen": 374800384 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044768304914744234, + "loss": 1.4053, + "theoretical_loss": 4.069810033406345, + "tokens_seen": 374865920 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004476730190571715, + "loss": 1.4912, + "theoretical_loss": 4.06973013228982, + "tokens_seen": 374931456 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004476629889669007, + "loss": 1.4765, + "theoretical_loss": 4.0696502490481405, + "tokens_seen": 374996992 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004476529588766299, + "loss": 1.4284, + "theoretical_loss": 4.069570383674185, + "tokens_seen": 375062528 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044764292878635907, + "loss": 1.4657, + "theoretical_loss": 4.069490536160835, + "tokens_seen": 375128064 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044763289869608825, + "loss": 1.417, + "theoretical_loss": 4.069410706500978, + "tokens_seen": 375193600 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004476228686058175, + "loss": 1.6, + "theoretical_loss": 4.0693308946875035, + "tokens_seen": 375259136 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004476128385155466, + "loss": 1.4601, + "theoretical_loss": 4.069251100713307, + "tokens_seen": 375324672 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044760280842527585, + "loss": 1.5094, + "theoretical_loss": 4.069171324571288, + "tokens_seen": 375390208 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044759277833500503, + "loss": 1.6056, + "theoretical_loss": 4.069091566254347, + "tokens_seen": 375455744 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004475827482447342, + "loss": 1.5439, + "theoretical_loss": 4.069011825755393, + "tokens_seen": 375521280 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004475727181544634, + "loss": 1.6227, + "theoretical_loss": 4.068932103067335, + "tokens_seen": 375586816 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044756268806419257, + "loss": 1.5734, + "theoretical_loss": 4.06885239818309, + "tokens_seen": 375652352 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044755265797392175, + "loss": 1.4756, + "theoretical_loss": 4.068772711095575, + "tokens_seen": 375717888 + }, + { + "epoch": 0.11, + "learning_rate": 0.000447542627883651, + "loss": 1.4841, + "theoretical_loss": 4.068693041797715, + "tokens_seen": 375783424 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004475325977933801, + "loss": 1.5855, + "theoretical_loss": 4.068613390282437, + "tokens_seen": 375848960 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044752256770310935, + "loss": 1.3927, + "theoretical_loss": 4.06853375654267, + "tokens_seen": 375914496 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004475125376128385, + "loss": 1.6459, + "theoretical_loss": 4.068454140571351, + "tokens_seen": 375980032 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004475025075225677, + "loss": 1.4447, + "theoretical_loss": 4.068374542361418, + "tokens_seen": 376045568 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004474924774322969, + "loss": 1.4775, + "theoretical_loss": 4.068294961905817, + "tokens_seen": 376111104 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004474824473420261, + "loss": 1.6394, + "theoretical_loss": 4.068215399197492, + "tokens_seen": 376176640 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044747241725175526, + "loss": 1.5885, + "theoretical_loss": 4.068135854229396, + "tokens_seen": 376242176 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044746238716148444, + "loss": 1.5465, + "theoretical_loss": 4.068056326994485, + "tokens_seen": 376307712 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004474523570712136, + "loss": 1.5665, + "theoretical_loss": 4.067976817485717, + "tokens_seen": 376373248 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044744232698094285, + "loss": 1.5063, + "theoretical_loss": 4.067897325696056, + "tokens_seen": 376438784 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044743229689067204, + "loss": 1.57, + "theoretical_loss": 4.06781785161847, + "tokens_seen": 376504320 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004474222668004012, + "loss": 1.567, + "theoretical_loss": 4.067738395245929, + "tokens_seen": 376569856 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044741223671013045, + "loss": 1.6536, + "theoretical_loss": 4.067658956571409, + "tokens_seen": 376635392 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004474022066198596, + "loss": 1.5093, + "theoretical_loss": 4.067579535587891, + "tokens_seen": 376700928 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004473921765295888, + "loss": 1.5576, + "theoretical_loss": 4.067500132288357, + "tokens_seen": 376766464 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 131550, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5377223491668701, + "objective/train/theoretical_loss": 4.067420746665794, + "objective/train/tokens_used": 376832000, + "theoretical_loss": 4.067420746665794, + "tokens_seen": 376832000 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044738214643931794, + "loss": 1.5062, + "theoretical_loss": 4.067420746665794, + "tokens_seen": 376832000 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004473721163490472, + "loss": 1.5367, + "theoretical_loss": 4.067341378713195, + "tokens_seen": 376897536 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044736208625877636, + "loss": 1.5601, + "theoretical_loss": 4.0672620284235546, + "tokens_seen": 376963072 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044735205616850554, + "loss": 1.5021, + "theoretical_loss": 4.067182695789872, + "tokens_seen": 377028608 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004473420260782347, + "loss": 1.5315, + "theoretical_loss": 4.06710338080515, + "tokens_seen": 377094144 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004473319959879639, + "loss": 1.5808, + "theoretical_loss": 4.067024083462399, + "tokens_seen": 377159680 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004473219658976931, + "loss": 1.5288, + "theoretical_loss": 4.066944803754628, + "tokens_seen": 377225216 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004473119358074223, + "loss": 1.6287, + "theoretical_loss": 4.0668655416748525, + "tokens_seen": 377290752 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044730190571715144, + "loss": 1.4639, + "theoretical_loss": 4.066786297216092, + "tokens_seen": 377356288 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004472918756268807, + "loss": 1.6065, + "theoretical_loss": 4.066707070371371, + "tokens_seen": 377421824 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004472818455366098, + "loss": 1.6097, + "theoretical_loss": 4.066627861133716, + "tokens_seen": 377487360 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044727181544633904, + "loss": 1.4857, + "theoretical_loss": 4.066548669496157, + "tokens_seen": 377552896 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004472617853560682, + "loss": 1.4939, + "theoretical_loss": 4.066469495451732, + "tokens_seen": 377618432 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004472517552657974, + "loss": 1.5875, + "theoretical_loss": 4.066390338993479, + "tokens_seen": 377683968 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004472417251755266, + "loss": 1.5909, + "theoretical_loss": 4.0663112001144395, + "tokens_seen": 377749504 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004472316950852558, + "loss": 1.5713, + "theoretical_loss": 4.066232078807664, + "tokens_seen": 377815040 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044722166499498495, + "loss": 1.4496, + "theoretical_loss": 4.0661529750662, + "tokens_seen": 377880576 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004472116349047142, + "loss": 1.6113, + "theoretical_loss": 4.066073888883105, + "tokens_seen": 377946112 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004472016048144433, + "loss": 1.4681, + "theoretical_loss": 4.065994820251438, + "tokens_seen": 378011648 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044719157472417254, + "loss": 1.4275, + "theoretical_loss": 4.065915769164262, + "tokens_seen": 378077184 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004471815446339017, + "loss": 1.468, + "theoretical_loss": 4.065836735614642, + "tokens_seen": 378142720 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004471715145436309, + "loss": 1.6317, + "theoretical_loss": 4.06575771959565, + "tokens_seen": 378208256 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004471614844533601, + "loss": 1.3695, + "theoretical_loss": 4.065678721100362, + "tokens_seen": 378273792 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044715145436308927, + "loss": 1.4646, + "theoretical_loss": 4.065599740121855, + "tokens_seen": 378339328 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044714142427281845, + "loss": 1.4623, + "theoretical_loss": 4.065520776653212, + "tokens_seen": 378404864 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004471313941825477, + "loss": 1.5583, + "theoretical_loss": 4.06544183068752, + "tokens_seen": 378470400 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004471213640922768, + "loss": 1.4272, + "theoretical_loss": 4.0653629022178706, + "tokens_seen": 378535936 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044711133400200605, + "loss": 1.5187, + "theoretical_loss": 4.065283991237355, + "tokens_seen": 378601472 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044710130391173523, + "loss": 1.6449, + "theoretical_loss": 4.065205097739075, + "tokens_seen": 378667008 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004470912738214644, + "loss": 1.6074, + "theoretical_loss": 4.065126221716131, + "tokens_seen": 378732544 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004470812437311936, + "loss": 1.5718, + "theoretical_loss": 4.0650473631616295, + "tokens_seen": 378798080 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044707121364092277, + "loss": 1.6398, + "theoretical_loss": 4.064968522068681, + "tokens_seen": 378863616 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044706118355065195, + "loss": 1.4587, + "theoretical_loss": 4.0648896984304, + "tokens_seen": 378929152 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004470511534603812, + "loss": 1.5058, + "theoretical_loss": 4.064810892239903, + "tokens_seen": 378994688 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004470411233701103, + "loss": 1.4158, + "theoretical_loss": 4.064732103490312, + "tokens_seen": 379060224 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044703109327983955, + "loss": 1.6171, + "theoretical_loss": 4.064653332174753, + "tokens_seen": 379125760 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004470210631895687, + "loss": 1.6133, + "theoretical_loss": 4.064574578286356, + "tokens_seen": 379191296 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004470110330992979, + "loss": 1.5514, + "theoretical_loss": 4.064495841818255, + "tokens_seen": 379256832 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004470010030090271, + "loss": 1.5083, + "theoretical_loss": 4.064417122763587, + "tokens_seen": 379322368 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004469909729187563, + "loss": 1.5746, + "theoretical_loss": 4.064338421115491, + "tokens_seen": 379387904 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044698094282848546, + "loss": 1.4658, + "theoretical_loss": 4.064259736867116, + "tokens_seen": 379453440 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044697091273821464, + "loss": 1.4291, + "theoretical_loss": 4.0641810700116086, + "tokens_seen": 379518976 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004469608826479438, + "loss": 1.5308, + "theoretical_loss": 4.064102420542122, + "tokens_seen": 379584512 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044695085255767305, + "loss": 1.5985, + "theoretical_loss": 4.064023788451814, + "tokens_seen": 379650048 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004469408224674022, + "loss": 1.5151, + "theoretical_loss": 4.063945173733844, + "tokens_seen": 379715584 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004469307923771314, + "loss": 1.639, + "theoretical_loss": 4.063866576381377, + "tokens_seen": 379781120 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004469207622868606, + "loss": 1.4994, + "theoretical_loss": 4.063787996387583, + "tokens_seen": 379846656 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004469107321965898, + "loss": 1.4573, + "theoretical_loss": 4.063709433745632, + "tokens_seen": 379912192 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044690070210631896, + "loss": 1.5028, + "theoretical_loss": 4.063630888448701, + "tokens_seen": 379977728 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044689067201604814, + "loss": 1.5174, + "theoretical_loss": 4.06355236048997, + "tokens_seen": 380043264 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 132472, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5321382284164429, + "objective/train/theoretical_loss": 4.063473849862623, + "objective/train/tokens_used": 380108800, + "theoretical_loss": 4.063473849862623, + "tokens_seen": 380108800 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004468806419257773, + "loss": 1.4666, + "theoretical_loss": 4.063473849862623, + "tokens_seen": 380108800 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044687061183550656, + "loss": 1.5606, + "theoretical_loss": 4.063395356559848, + "tokens_seen": 380174336 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004468605817452357, + "loss": 1.6044, + "theoretical_loss": 4.063316880574836, + "tokens_seen": 380239872 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004468505516549649, + "loss": 1.5508, + "theoretical_loss": 4.063238421900783, + "tokens_seen": 380305408 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044684052156469405, + "loss": 1.6791, + "theoretical_loss": 4.063159980530887, + "tokens_seen": 380370944 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004468304914744233, + "loss": 1.5729, + "theoretical_loss": 4.063081556458353, + "tokens_seen": 380436480 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044682046138415246, + "loss": 1.4135, + "theoretical_loss": 4.063003149676387, + "tokens_seen": 380502016 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044681043129388164, + "loss": 1.4729, + "theoretical_loss": 4.062924760178198, + "tokens_seen": 380567552 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004468004012036108, + "loss": 1.5115, + "theoretical_loss": 4.062846387957005, + "tokens_seen": 380633088 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044679037111334, + "loss": 1.4622, + "theoretical_loss": 4.062768033006023, + "tokens_seen": 380698624 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004467803410230692, + "loss": 1.5525, + "theoretical_loss": 4.062689695318475, + "tokens_seen": 380764160 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004467703109327984, + "loss": 1.4867, + "theoretical_loss": 4.062611374887587, + "tokens_seen": 380829696 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044676028084252755, + "loss": 1.5339, + "theoretical_loss": 4.062533071706589, + "tokens_seen": 380895232 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004467502507522568, + "loss": 1.5951, + "theoretical_loss": 4.062454785768716, + "tokens_seen": 380960768 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044674022066198597, + "loss": 1.4324, + "theoretical_loss": 4.062376517067204, + "tokens_seen": 381026304 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044673019057171515, + "loss": 1.5639, + "theoretical_loss": 4.062298265595294, + "tokens_seen": 381091840 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044672016048144433, + "loss": 1.5366, + "theoretical_loss": 4.062220031346234, + "tokens_seen": 381157376 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004467101303911735, + "loss": 1.6231, + "theoretical_loss": 4.06214181431327, + "tokens_seen": 381222912 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004467001003009027, + "loss": 1.5167, + "theoretical_loss": 4.062063614489656, + "tokens_seen": 381288448 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004466900702106319, + "loss": 1.5711, + "theoretical_loss": 4.061985431868648, + "tokens_seen": 381353984 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004466800401203611, + "loss": 1.6051, + "theoretical_loss": 4.061907266443508, + "tokens_seen": 381419520 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004466700100300903, + "loss": 1.5014, + "theoretical_loss": 4.061829118207498, + "tokens_seen": 381485056 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044665997993981947, + "loss": 1.6702, + "theoretical_loss": 4.061750987153888, + "tokens_seen": 381550592 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044664994984954865, + "loss": 1.4612, + "theoretical_loss": 4.061672873275949, + "tokens_seen": 381616128 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004466399197592779, + "loss": 1.6288, + "theoretical_loss": 4.061594776566958, + "tokens_seen": 381681664 + }, + { + "epoch": 0.12, + "learning_rate": 0.000446629889669007, + "loss": 1.6502, + "theoretical_loss": 4.061516697020191, + "tokens_seen": 381747200 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044661985957873625, + "loss": 1.5478, + "theoretical_loss": 4.061438634628933, + "tokens_seen": 381812736 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044660982948846543, + "loss": 1.5464, + "theoretical_loss": 4.061360589386473, + "tokens_seen": 381878272 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004465997993981946, + "loss": 1.4264, + "theoretical_loss": 4.061282561286098, + "tokens_seen": 381943808 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004465897693079238, + "loss": 1.4842, + "theoretical_loss": 4.061204550321104, + "tokens_seen": 382009344 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044657973921765297, + "loss": 1.5391, + "theoretical_loss": 4.061126556484791, + "tokens_seen": 382074880 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044656970912738215, + "loss": 1.5441, + "theoretical_loss": 4.06104857977046, + "tokens_seen": 382140416 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004465596790371114, + "loss": 1.475, + "theoretical_loss": 4.060970620171415, + "tokens_seen": 382205952 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004465496489468405, + "loss": 1.5351, + "theoretical_loss": 4.060892677680967, + "tokens_seen": 382271488 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044653961885656975, + "loss": 1.4668, + "theoretical_loss": 4.06081475229243, + "tokens_seen": 382337024 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004465295887662989, + "loss": 1.5259, + "theoretical_loss": 4.060736843999121, + "tokens_seen": 382402560 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004465195586760281, + "loss": 1.4619, + "theoretical_loss": 4.060658952794359, + "tokens_seen": 382468096 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004465095285857573, + "loss": 1.7026, + "theoretical_loss": 4.06058107867147, + "tokens_seen": 382533632 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004464994984954865, + "loss": 1.5722, + "theoretical_loss": 4.060503221623783, + "tokens_seen": 382599168 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044648946840521566, + "loss": 1.5587, + "theoretical_loss": 4.060425381644629, + "tokens_seen": 382664704 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044647943831494484, + "loss": 1.4321, + "theoretical_loss": 4.060347558727344, + "tokens_seen": 382730240 + }, + { + "epoch": 0.12, + "learning_rate": 0.000446469408224674, + "loss": 1.4976, + "theoretical_loss": 4.060269752865268, + "tokens_seen": 382795776 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044645937813440325, + "loss": 1.5304, + "theoretical_loss": 4.060191964051744, + "tokens_seen": 382861312 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004464493480441324, + "loss": 1.6664, + "theoretical_loss": 4.060114192280119, + "tokens_seen": 382926848 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004464393179538616, + "loss": 1.5878, + "theoretical_loss": 4.0600364375437445, + "tokens_seen": 382992384 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004464292878635908, + "loss": 1.4531, + "theoretical_loss": 4.059958699835974, + "tokens_seen": 383057920 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044641925777332, + "loss": 1.5889, + "theoretical_loss": 4.059880979150167, + "tokens_seen": 383123456 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044640922768304916, + "loss": 1.5249, + "theoretical_loss": 4.059803275479684, + "tokens_seen": 383188992 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044639919759277834, + "loss": 1.3815, + "theoretical_loss": 4.059725588817891, + "tokens_seen": 383254528 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004463891675025075, + "loss": 1.5056, + "theoretical_loss": 4.0596479191581585, + "tokens_seen": 383320064 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 133642, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.459533929824829, + "objective/train/theoretical_loss": 4.059570266493859, + "objective/train/tokens_used": 383385600, + "theoretical_loss": 4.059570266493859, + "tokens_seen": 383385600 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044637913741223676, + "loss": 1.4779, + "theoretical_loss": 4.059570266493859, + "tokens_seen": 383385600 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004463691073219659, + "loss": 1.5942, + "theoretical_loss": 4.059492630818369, + "tokens_seen": 383451136 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004463590772316951, + "loss": 1.4633, + "theoretical_loss": 4.05941501212507, + "tokens_seen": 383516672 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044634904714142425, + "loss": 1.5901, + "theoretical_loss": 4.059337410407346, + "tokens_seen": 383582208 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004463390170511535, + "loss": 1.4232, + "theoretical_loss": 4.059259825658584, + "tokens_seen": 383647744 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044632898696088266, + "loss": 1.5188, + "theoretical_loss": 4.059182257872177, + "tokens_seen": 383713280 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044631895687061184, + "loss": 1.559, + "theoretical_loss": 4.059104707041519, + "tokens_seen": 383778816 + }, + { + "epoch": 0.12, + "learning_rate": 0.000446308926780341, + "loss": 1.6575, + "theoretical_loss": 4.059027173160011, + "tokens_seen": 383844352 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004462988966900702, + "loss": 1.3706, + "theoretical_loss": 4.0589496562210545, + "tokens_seen": 383909888 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004462888665997994, + "loss": 1.5029, + "theoretical_loss": 4.058872156218055, + "tokens_seen": 383975424 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004462788365095286, + "loss": 1.6168, + "theoretical_loss": 4.058794673144425, + "tokens_seen": 384040960 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044626880641925775, + "loss": 1.5833, + "theoretical_loss": 4.0587172069935775, + "tokens_seen": 384106496 + }, + { + "epoch": 0.12, + "learning_rate": 0.000446258776328987, + "loss": 1.5675, + "theoretical_loss": 4.05863975775893, + "tokens_seen": 384172032 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044624874623871617, + "loss": 1.5689, + "theoretical_loss": 4.058562325433902, + "tokens_seen": 384237568 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044623871614844535, + "loss": 1.5392, + "theoretical_loss": 4.058484910011922, + "tokens_seen": 384303104 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044622868605817453, + "loss": 1.624, + "theoretical_loss": 4.058407511486415, + "tokens_seen": 384368640 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004462186559679037, + "loss": 1.5188, + "theoretical_loss": 4.058330129850816, + "tokens_seen": 384434176 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004462086258776329, + "loss": 1.5276, + "theoretical_loss": 4.058252765098558, + "tokens_seen": 384499712 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004461985957873621, + "loss": 1.4701, + "theoretical_loss": 4.058175417223084, + "tokens_seen": 384565248 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044618856569709125, + "loss": 1.4407, + "theoretical_loss": 4.058098086217836, + "tokens_seen": 384630784 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004461785356068205, + "loss": 1.4798, + "theoretical_loss": 4.058020772076259, + "tokens_seen": 384696320 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004461685055165496, + "loss": 1.7, + "theoretical_loss": 4.057943474791806, + "tokens_seen": 384761856 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044615847542627885, + "loss": 1.5733, + "theoretical_loss": 4.0578661943579295, + "tokens_seen": 384827392 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044614844533600803, + "loss": 1.5421, + "theoretical_loss": 4.05778893076809, + "tokens_seen": 384892928 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004461384152457372, + "loss": 1.4653, + "theoretical_loss": 4.057711684015747, + "tokens_seen": 384958464 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004461283851554664, + "loss": 1.548, + "theoretical_loss": 4.057634454094365, + "tokens_seen": 385024000 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044611835506519563, + "loss": 1.6564, + "theoretical_loss": 4.057557240997415, + "tokens_seen": 385089536 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044610832497492476, + "loss": 1.5001, + "theoretical_loss": 4.057480044718368, + "tokens_seen": 385155072 + }, + { + "epoch": 0.12, + "learning_rate": 0.000446098294884654, + "loss": 1.5724, + "theoretical_loss": 4.057402865250701, + "tokens_seen": 385220608 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004460882647943831, + "loss": 1.4844, + "theoretical_loss": 4.057325702587893, + "tokens_seen": 385286144 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044607823470411235, + "loss": 1.5905, + "theoretical_loss": 4.057248556723429, + "tokens_seen": 385351680 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044606820461384153, + "loss": 1.5049, + "theoretical_loss": 4.057171427650793, + "tokens_seen": 385417216 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004460581745235707, + "loss": 1.5286, + "theoretical_loss": 4.0570943153634795, + "tokens_seen": 385482752 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004460481444332999, + "loss": 1.4028, + "theoretical_loss": 4.0570172198549805, + "tokens_seen": 385548288 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004460381143430291, + "loss": 1.4984, + "theoretical_loss": 4.056940141118794, + "tokens_seen": 385613824 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044602808425275826, + "loss": 1.5182, + "theoretical_loss": 4.056863079148422, + "tokens_seen": 385679360 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004460180541624875, + "loss": 1.4784, + "theoretical_loss": 4.05678603393737, + "tokens_seen": 385744896 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004460080240722166, + "loss": 1.4125, + "theoretical_loss": 4.056709005479147, + "tokens_seen": 385810432 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044599799398194586, + "loss": 1.5854, + "theoretical_loss": 4.056631993767264, + "tokens_seen": 385875968 + }, + { + "epoch": 0.12, + "learning_rate": 0.000445987963891675, + "loss": 1.4944, + "theoretical_loss": 4.056554998795239, + "tokens_seen": 385941504 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004459779338014042, + "loss": 1.4051, + "theoretical_loss": 4.05647802055659, + "tokens_seen": 386007040 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004459679037111334, + "loss": 1.5067, + "theoretical_loss": 4.056401059044841, + "tokens_seen": 386072576 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004459578736208626, + "loss": 1.4051, + "theoretical_loss": 4.056324114253519, + "tokens_seen": 386138112 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044594784353059176, + "loss": 1.4959, + "theoretical_loss": 4.056247186176154, + "tokens_seen": 386203648 + }, + { + "epoch": 0.12, + "learning_rate": 0.000445937813440321, + "loss": 1.5085, + "theoretical_loss": 4.056170274806281, + "tokens_seen": 386269184 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004459277833500502, + "loss": 1.372, + "theoretical_loss": 4.056093380137436, + "tokens_seen": 386334720 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044591775325977936, + "loss": 1.457, + "theoretical_loss": 4.056016502163162, + "tokens_seen": 386400256 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044590772316950854, + "loss": 1.4681, + "theoretical_loss": 4.055939640877003, + "tokens_seen": 386465792 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004458976930792377, + "loss": 1.5974, + "theoretical_loss": 4.055862796272508, + "tokens_seen": 386531328 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044588766298896696, + "loss": 1.5318, + "theoretical_loss": 4.055785968343228, + "tokens_seen": 386596864 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 134594, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5517709255218506, + "objective/train/theoretical_loss": 4.05570915708272, + "objective/train/tokens_used": 386662400, + "theoretical_loss": 4.05570915708272, + "tokens_seen": 386662400 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004458776328986961, + "loss": 1.6004, + "theoretical_loss": 4.05570915708272, + "tokens_seen": 386662400 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004458676028084253, + "loss": 1.5107, + "theoretical_loss": 4.055632362484542, + "tokens_seen": 386727936 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044585757271815445, + "loss": 1.4644, + "theoretical_loss": 4.055555584542257, + "tokens_seen": 386793472 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004458475426278837, + "loss": 1.3312, + "theoretical_loss": 4.055478823249432, + "tokens_seen": 386859008 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044583751253761286, + "loss": 1.5239, + "theoretical_loss": 4.055402078599638, + "tokens_seen": 386924544 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044582748244734204, + "loss": 1.4634, + "theoretical_loss": 4.055325350586445, + "tokens_seen": 386990080 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004458174523570712, + "loss": 1.5215, + "theoretical_loss": 4.055248639203433, + "tokens_seen": 387055616 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004458074222668004, + "loss": 1.4743, + "theoretical_loss": 4.055171944444182, + "tokens_seen": 387121152 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004457973921765296, + "loss": 1.4911, + "theoretical_loss": 4.055095266302276, + "tokens_seen": 387186688 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004457873620862588, + "loss": 1.5351, + "theoretical_loss": 4.055018604771302, + "tokens_seen": 387252224 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044577733199598795, + "loss": 1.5315, + "theoretical_loss": 4.054941959844853, + "tokens_seen": 387317760 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004457673019057172, + "loss": 1.4173, + "theoretical_loss": 4.054865331516523, + "tokens_seen": 387383296 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044575727181544637, + "loss": 1.483, + "theoretical_loss": 4.054788719779911, + "tokens_seen": 387448832 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044574724172517555, + "loss": 1.5481, + "theoretical_loss": 4.054712124628617, + "tokens_seen": 387514368 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044573721163490473, + "loss": 1.5853, + "theoretical_loss": 4.054635546056248, + "tokens_seen": 387579904 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004457271815446339, + "loss": 1.5489, + "theoretical_loss": 4.054558984056414, + "tokens_seen": 387645440 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004457171514543631, + "loss": 1.5644, + "theoretical_loss": 4.054482438622727, + "tokens_seen": 387710976 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004457071213640923, + "loss": 1.4944, + "theoretical_loss": 4.0544059097488026, + "tokens_seen": 387776512 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044569709127382145, + "loss": 1.6564, + "theoretical_loss": 4.05432939742826, + "tokens_seen": 387842048 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004456870611835507, + "loss": 1.4058, + "theoretical_loss": 4.054252901654724, + "tokens_seen": 387907584 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004456770310932798, + "loss": 1.4388, + "theoretical_loss": 4.054176422421821, + "tokens_seen": 387973120 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044566700100300905, + "loss": 1.4646, + "theoretical_loss": 4.05409995972318, + "tokens_seen": 388038656 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044565697091273823, + "loss": 1.5204, + "theoretical_loss": 4.054023513552437, + "tokens_seen": 388104192 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004456469408224674, + "loss": 1.5257, + "theoretical_loss": 4.0539470839032266, + "tokens_seen": 388169728 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004456369107321966, + "loss": 1.4731, + "theoretical_loss": 4.053870670769192, + "tokens_seen": 388235264 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044562688064192583, + "loss": 1.5469, + "theoretical_loss": 4.053794274143977, + "tokens_seen": 388300800 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044561685055165496, + "loss": 1.3955, + "theoretical_loss": 4.053717894021229, + "tokens_seen": 388366336 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004456068204613842, + "loss": 1.5421, + "theoretical_loss": 4.0536415303946, + "tokens_seen": 388431872 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004455967903711133, + "loss": 1.5836, + "theoretical_loss": 4.053565183257745, + "tokens_seen": 388497408 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044558676028084255, + "loss": 1.4794, + "theoretical_loss": 4.053488852604322, + "tokens_seen": 388562944 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044557673019057173, + "loss": 1.5879, + "theoretical_loss": 4.053412538427993, + "tokens_seen": 388628480 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004455667001003009, + "loss": 1.6016, + "theoretical_loss": 4.053336240722423, + "tokens_seen": 388694016 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004455566700100301, + "loss": 1.4801, + "theoretical_loss": 4.053259959481283, + "tokens_seen": 388759552 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004455466399197593, + "loss": 1.4748, + "theoretical_loss": 4.053183694698244, + "tokens_seen": 388825088 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044553660982948846, + "loss": 1.6424, + "theoretical_loss": 4.053107446366981, + "tokens_seen": 388890624 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004455265797392177, + "loss": 1.5343, + "theoretical_loss": 4.053031214481175, + "tokens_seen": 388956160 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004455165496489468, + "loss": 1.4588, + "theoretical_loss": 4.052954999034509, + "tokens_seen": 389021696 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044550651955867606, + "loss": 1.5963, + "theoretical_loss": 4.052878800020668, + "tokens_seen": 389087232 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004454964894684052, + "loss": 1.5243, + "theoretical_loss": 4.052802617433344, + "tokens_seen": 389152768 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004454864593781344, + "loss": 1.5304, + "theoretical_loss": 4.0527264512662295, + "tokens_seen": 389218304 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004454764292878636, + "loss": 1.5343, + "theoretical_loss": 4.052650301513021, + "tokens_seen": 389283840 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004454663991975928, + "loss": 1.6789, + "theoretical_loss": 4.0525741681674186, + "tokens_seen": 389349376 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044545636910732196, + "loss": 1.5596, + "theoretical_loss": 4.052498051223127, + "tokens_seen": 389414912 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004454463390170512, + "loss": 1.5239, + "theoretical_loss": 4.052421950673853, + "tokens_seen": 389480448 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004454363089267803, + "loss": 1.5335, + "theoretical_loss": 4.052345866513308, + "tokens_seen": 389545984 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044542627883650956, + "loss": 1.5571, + "theoretical_loss": 4.052269798735207, + "tokens_seen": 389611520 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004454162487462387, + "loss": 1.4793, + "theoretical_loss": 4.052193747333265, + "tokens_seen": 389677056 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004454062186559679, + "loss": 1.586, + "theoretical_loss": 4.052117712301206, + "tokens_seen": 389742592 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004453961885656971, + "loss": 1.6177, + "theoretical_loss": 4.052041693632754, + "tokens_seen": 389808128 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004453861584754263, + "loss": 1.5498, + "theoretical_loss": 4.0519656913216355, + "tokens_seen": 389873664 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 135757, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.591356635093689, + "objective/train/theoretical_loss": 4.051889705361584, + "objective/train/tokens_used": 389939200, + "theoretical_loss": 4.051889705361584, + "tokens_seen": 389939200 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044537612838515547, + "loss": 1.6268, + "theoretical_loss": 4.051889705361584, + "tokens_seen": 389939200 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044536609829488465, + "loss": 1.5783, + "theoretical_loss": 4.051813735746334, + "tokens_seen": 390004736 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044535606820461383, + "loss": 1.5255, + "theoretical_loss": 4.051737782469624, + "tokens_seen": 390070272 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044534603811434306, + "loss": 1.4042, + "theoretical_loss": 4.051661845525196, + "tokens_seen": 390135808 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004453360080240722, + "loss": 1.5671, + "theoretical_loss": 4.051585924906796, + "tokens_seen": 390201344 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004453259779338014, + "loss": 1.5253, + "theoretical_loss": 4.051510020608171, + "tokens_seen": 390266880 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044531594784353055, + "loss": 1.6389, + "theoretical_loss": 4.051434132623075, + "tokens_seen": 390332416 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004453059177532598, + "loss": 1.566, + "theoretical_loss": 4.051358260945262, + "tokens_seen": 390397952 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044529588766298897, + "loss": 1.5134, + "theoretical_loss": 4.051282405568494, + "tokens_seen": 390463488 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044528585757271815, + "loss": 1.6773, + "theoretical_loss": 4.051206566486531, + "tokens_seen": 390529024 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044527582748244733, + "loss": 1.5288, + "theoretical_loss": 4.0511307436931405, + "tokens_seen": 390594560 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044526579739217657, + "loss": 1.6464, + "theoretical_loss": 4.051054937182092, + "tokens_seen": 390660096 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004452557673019057, + "loss": 1.5114, + "theoretical_loss": 4.0509791469471566, + "tokens_seen": 390725632 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044524573721163493, + "loss": 1.5679, + "theoretical_loss": 4.050903372982113, + "tokens_seen": 390791168 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044523570712136406, + "loss": 1.5133, + "theoretical_loss": 4.050827615280739, + "tokens_seen": 390856704 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004452256770310933, + "loss": 1.6191, + "theoretical_loss": 4.050751873836819, + "tokens_seen": 390922240 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044521564694082247, + "loss": 1.5479, + "theoretical_loss": 4.050676148644139, + "tokens_seen": 390987776 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044520561685055165, + "loss": 1.4601, + "theoretical_loss": 4.050600439696489, + "tokens_seen": 391053312 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044519558676028083, + "loss": 1.4781, + "theoretical_loss": 4.050524746987664, + "tokens_seen": 391118848 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044518555667001, + "loss": 1.5653, + "theoretical_loss": 4.050449070511458, + "tokens_seen": 391184384 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044517552657973925, + "loss": 1.5547, + "theoretical_loss": 4.050373410261673, + "tokens_seen": 391249920 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044516549648946843, + "loss": 1.647, + "theoretical_loss": 4.050297766232113, + "tokens_seen": 391315456 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004451554663991976, + "loss": 1.5755, + "theoretical_loss": 4.050222138416583, + "tokens_seen": 391380992 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004451454363089268, + "loss": 1.6471, + "theoretical_loss": 4.050146526808896, + "tokens_seen": 391446528 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044513540621865603, + "loss": 1.5524, + "theoretical_loss": 4.050070931402864, + "tokens_seen": 391512064 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044512537612838516, + "loss": 1.5163, + "theoretical_loss": 4.049995352192305, + "tokens_seen": 391577600 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004451153460381144, + "loss": 1.5691, + "theoretical_loss": 4.04991978917104, + "tokens_seen": 391643136 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004451053159478435, + "loss": 1.5882, + "theoretical_loss": 4.049844242332892, + "tokens_seen": 391708672 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044509528585757275, + "loss": 1.4722, + "theoretical_loss": 4.049768711671689, + "tokens_seen": 391774208 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044508525576730194, + "loss": 1.6095, + "theoretical_loss": 4.049693197181262, + "tokens_seen": 391839744 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004450752256770311, + "loss": 1.6563, + "theoretical_loss": 4.049617698855444, + "tokens_seen": 391905280 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004450651955867603, + "loss": 1.5465, + "theoretical_loss": 4.049542216688074, + "tokens_seen": 391970816 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004450551654964895, + "loss": 1.5379, + "theoretical_loss": 4.049466750672994, + "tokens_seen": 392036352 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044504513540621866, + "loss": 1.3964, + "theoretical_loss": 4.049391300804045, + "tokens_seen": 392101888 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004450351053159479, + "loss": 1.5691, + "theoretical_loss": 4.049315867075077, + "tokens_seen": 392167424 + }, + { + "epoch": 0.12, + "learning_rate": 0.000445025075225677, + "loss": 1.565, + "theoretical_loss": 4.0492404494799406, + "tokens_seen": 392232960 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044501504513540626, + "loss": 1.5616, + "theoretical_loss": 4.049165048012489, + "tokens_seen": 392298496 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004450050150451354, + "loss": 1.4947, + "theoretical_loss": 4.049089662666582, + "tokens_seen": 392364032 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004449949849548646, + "loss": 1.477, + "theoretical_loss": 4.04901429343608, + "tokens_seen": 392429568 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004449849548645938, + "loss": 1.5079, + "theoretical_loss": 4.048938940314848, + "tokens_seen": 392495104 + }, + { + "epoch": 0.12, + "learning_rate": 0.000444974924774323, + "loss": 1.5291, + "theoretical_loss": 4.048863603296753, + "tokens_seen": 392560640 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044496489468405216, + "loss": 1.5071, + "theoretical_loss": 4.048788282375666, + "tokens_seen": 392626176 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004449548645937814, + "loss": 1.5569, + "theoretical_loss": 4.048712977545463, + "tokens_seen": 392691712 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004449448345035105, + "loss": 1.525, + "theoretical_loss": 4.048637688800021, + "tokens_seen": 392757248 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044493480441323976, + "loss": 1.4859, + "theoretical_loss": 4.048562416133221, + "tokens_seen": 392822784 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004449247743229689, + "loss": 1.5548, + "theoretical_loss": 4.048487159538949, + "tokens_seen": 392888320 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004449147442326981, + "loss": 1.5229, + "theoretical_loss": 4.048411919011091, + "tokens_seen": 392953856 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004449047141424273, + "loss": 1.4298, + "theoretical_loss": 4.04833669454354, + "tokens_seen": 393019392 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004448946840521565, + "loss": 1.5671, + "theoretical_loss": 4.0482614861301895, + "tokens_seen": 393084928 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044488465396188567, + "loss": 1.3627, + "theoretical_loss": 4.048186293764939, + "tokens_seen": 393150464 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 136670, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5054774284362793, + "objective/train/theoretical_loss": 4.048111117441689, + "objective/train/tokens_used": 393216000, + "theoretical_loss": 4.048111117441689, + "tokens_seen": 393216000 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044487462387161485, + "loss": 1.5785, + "theoretical_loss": 4.048111117441689, + "tokens_seen": 393216000 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044486459378134403, + "loss": 1.636, + "theoretical_loss": 4.048035957154343, + "tokens_seen": 393281536 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044485456369107326, + "loss": 1.5714, + "theoretical_loss": 4.047960812896811, + "tokens_seen": 393347072 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004448445336008024, + "loss": 1.5042, + "theoretical_loss": 4.047885684663004, + "tokens_seen": 393412608 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004448345035105316, + "loss": 1.6416, + "theoretical_loss": 4.047810572446836, + "tokens_seen": 393478144 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044482447342026075, + "loss": 1.5378, + "theoretical_loss": 4.047735476242225, + "tokens_seen": 393543680 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044481444332999, + "loss": 1.5188, + "theoretical_loss": 4.047660396043092, + "tokens_seen": 393609216 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044480441323971917, + "loss": 1.4844, + "theoretical_loss": 4.047585331843363, + "tokens_seen": 393674752 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044479438314944835, + "loss": 1.6514, + "theoretical_loss": 4.047510283636965, + "tokens_seen": 393740288 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044478435305917753, + "loss": 1.4957, + "theoretical_loss": 4.04743525141783, + "tokens_seen": 393805824 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044477432296890677, + "loss": 1.4573, + "theoretical_loss": 4.047360235179891, + "tokens_seen": 393871360 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004447642928786359, + "loss": 1.4783, + "theoretical_loss": 4.047285234917087, + "tokens_seen": 393936896 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044475426278836513, + "loss": 1.5452, + "theoretical_loss": 4.04721025062336, + "tokens_seen": 394002432 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044474423269809426, + "loss": 1.4915, + "theoretical_loss": 4.047135282292653, + "tokens_seen": 394067968 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004447342026078235, + "loss": 1.5307, + "theoretical_loss": 4.0470603299189145, + "tokens_seen": 394133504 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044472417251755267, + "loss": 1.4923, + "theoretical_loss": 4.0469853934960955, + "tokens_seen": 394199040 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044471414242728185, + "loss": 1.5811, + "theoretical_loss": 4.046910473018151, + "tokens_seen": 394264576 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044470411233701103, + "loss": 1.5517, + "theoretical_loss": 4.046835568479038, + "tokens_seen": 394330112 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004446940822467402, + "loss": 1.5439, + "theoretical_loss": 4.046760679872717, + "tokens_seen": 394395648 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004446840521564694, + "loss": 1.5844, + "theoretical_loss": 4.046685807193153, + "tokens_seen": 394461184 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044467402206619863, + "loss": 1.6145, + "theoretical_loss": 4.046610950434314, + "tokens_seen": 394526720 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044466399197592776, + "loss": 1.4823, + "theoretical_loss": 4.04653610959017, + "tokens_seen": 394592256 + }, + { + "epoch": 0.12, + "learning_rate": 0.000444653961885657, + "loss": 1.5484, + "theoretical_loss": 4.046461284654696, + "tokens_seen": 394657792 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004446439317953861, + "loss": 1.6303, + "theoretical_loss": 4.046386475621869, + "tokens_seen": 394723328 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044463390170511536, + "loss": 1.3807, + "theoretical_loss": 4.046311682485669, + "tokens_seen": 394788864 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044462387161484454, + "loss": 1.3771, + "theoretical_loss": 4.0462369052400815, + "tokens_seen": 394854400 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004446138415245737, + "loss": 1.5108, + "theoretical_loss": 4.0461621438790925, + "tokens_seen": 394919936 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004446038114343029, + "loss": 1.517, + "theoretical_loss": 4.046087398396693, + "tokens_seen": 394985472 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044459378134403214, + "loss": 1.5408, + "theoretical_loss": 4.046012668786877, + "tokens_seen": 395051008 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044458375125376126, + "loss": 1.5126, + "theoretical_loss": 4.045937955043641, + "tokens_seen": 395116544 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004445737211634905, + "loss": 1.583, + "theoretical_loss": 4.045863257160986, + "tokens_seen": 395182080 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004445636910732196, + "loss": 1.4581, + "theoretical_loss": 4.045788575132915, + "tokens_seen": 395247616 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044455366098294886, + "loss": 1.5105, + "theoretical_loss": 4.045713908953435, + "tokens_seen": 395313152 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044454363089267804, + "loss": 1.5674, + "theoretical_loss": 4.045639258616557, + "tokens_seen": 395378688 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004445336008024072, + "loss": 1.4213, + "theoretical_loss": 4.045564624116293, + "tokens_seen": 395444224 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004445235707121364, + "loss": 1.5807, + "theoretical_loss": 4.04549000544666, + "tokens_seen": 395509760 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004445135406218656, + "loss": 1.5855, + "theoretical_loss": 4.045415402601678, + "tokens_seen": 395575296 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044450351053159477, + "loss": 1.5018, + "theoretical_loss": 4.045340815575372, + "tokens_seen": 395640832 + }, + { + "epoch": 0.12, + "learning_rate": 0.000444493480441324, + "loss": 1.5846, + "theoretical_loss": 4.045266244361765, + "tokens_seen": 395706368 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044448345035105313, + "loss": 1.5937, + "theoretical_loss": 4.045191688954889, + "tokens_seen": 395771904 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044447342026078236, + "loss": 1.5971, + "theoretical_loss": 4.045117149348776, + "tokens_seen": 395837440 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004444633901705115, + "loss": 1.6648, + "theoretical_loss": 4.045042625537462, + "tokens_seen": 395902976 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004444533600802407, + "loss": 1.4526, + "theoretical_loss": 4.044968117514987, + "tokens_seen": 395968512 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044444332998996996, + "loss": 1.5182, + "theoretical_loss": 4.044893625275392, + "tokens_seen": 396034048 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004444332998996991, + "loss": 1.4569, + "theoretical_loss": 4.044819148812726, + "tokens_seen": 396099584 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004444232698094283, + "loss": 1.5163, + "theoretical_loss": 4.044744688121035, + "tokens_seen": 396165120 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004444132397191575, + "loss": 1.628, + "theoretical_loss": 4.044670243194372, + "tokens_seen": 396230656 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004444032096288867, + "loss": 1.4587, + "theoretical_loss": 4.044595814026794, + "tokens_seen": 396296192 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044439317953861587, + "loss": 1.4448, + "theoretical_loss": 4.044521400612357, + "tokens_seen": 396361728 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044438314944834505, + "loss": 1.4345, + "theoretical_loss": 4.044447002945125, + "tokens_seen": 396427264 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 137835, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.397544026374817, + "objective/train/theoretical_loss": 4.044372621019162, + "objective/train/tokens_used": 396492800, + "theoretical_loss": 4.044372621019162, + "tokens_seen": 396492800 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044437311935807423, + "loss": 1.4317, + "theoretical_loss": 4.044372621019162, + "tokens_seen": 396492800 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044436308926780346, + "loss": 1.5847, + "theoretical_loss": 4.044298254828538, + "tokens_seen": 396558336 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004443530591775326, + "loss": 1.6433, + "theoretical_loss": 4.044223904367324, + "tokens_seen": 396623872 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004443430290872618, + "loss": 1.492, + "theoretical_loss": 4.044149569629594, + "tokens_seen": 396689408 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044433299899699095, + "loss": 1.4914, + "theoretical_loss": 4.044075250609426, + "tokens_seen": 396754944 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004443229689067202, + "loss": 1.519, + "theoretical_loss": 4.044000947300901, + "tokens_seen": 396820480 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044431293881644937, + "loss": 1.6079, + "theoretical_loss": 4.043926659698106, + "tokens_seen": 396886016 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044430290872617855, + "loss": 1.4939, + "theoretical_loss": 4.043852387795125, + "tokens_seen": 396951552 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044429287863590773, + "loss": 1.5158, + "theoretical_loss": 4.043778131586052, + "tokens_seen": 397017088 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044428284854563697, + "loss": 1.5452, + "theoretical_loss": 4.043703891064979, + "tokens_seen": 397082624 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004442728184553661, + "loss": 1.5609, + "theoretical_loss": 4.043629666226003, + "tokens_seen": 397148160 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044426278836509533, + "loss": 1.572, + "theoretical_loss": 4.043555457063226, + "tokens_seen": 397213696 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044425275827482446, + "loss": 1.4924, + "theoretical_loss": 4.043481263570751, + "tokens_seen": 397279232 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004442427281845537, + "loss": 1.6289, + "theoretical_loss": 4.043407085742684, + "tokens_seen": 397344768 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044423269809428287, + "loss": 1.4874, + "theoretical_loss": 4.0433329235731374, + "tokens_seen": 397410304 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044422266800401205, + "loss": 1.5208, + "theoretical_loss": 4.043258777056221, + "tokens_seen": 397475840 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044421263791374123, + "loss": 1.4754, + "theoretical_loss": 4.043184646186053, + "tokens_seen": 397541376 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004442026078234704, + "loss": 1.4105, + "theoretical_loss": 4.043110530956754, + "tokens_seen": 397606912 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004441925777331996, + "loss": 1.3824, + "theoretical_loss": 4.043036431362444, + "tokens_seen": 397672448 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044418254764292883, + "loss": 1.402, + "theoretical_loss": 4.042962347397252, + "tokens_seen": 397737984 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044417251755265796, + "loss": 1.6016, + "theoretical_loss": 4.042888279055304, + "tokens_seen": 397803520 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004441624874623872, + "loss": 1.464, + "theoretical_loss": 4.0428142263307345, + "tokens_seen": 397869056 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004441524573721163, + "loss": 1.5456, + "theoretical_loss": 4.042740189217677, + "tokens_seen": 397934592 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044414242728184556, + "loss": 1.5557, + "theoretical_loss": 4.042666167710273, + "tokens_seen": 398000128 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044413239719157474, + "loss": 1.5962, + "theoretical_loss": 4.042592161802662, + "tokens_seen": 398065664 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004441223671013039, + "loss": 1.5746, + "theoretical_loss": 4.042518171488989, + "tokens_seen": 398131200 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004441123370110331, + "loss": 1.5736, + "theoretical_loss": 4.042444196763403, + "tokens_seen": 398196736 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044410230692076234, + "loss": 1.7201, + "theoretical_loss": 4.042370237620053, + "tokens_seen": 398262272 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044409227683049146, + "loss": 1.495, + "theoretical_loss": 4.042296294053097, + "tokens_seen": 398327808 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004440822467402207, + "loss": 1.3646, + "theoretical_loss": 4.0422223660566905, + "tokens_seen": 398393344 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004440722166499498, + "loss": 1.4828, + "theoretical_loss": 4.0421484536249945, + "tokens_seen": 398458880 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044406218655967906, + "loss": 1.5022, + "theoretical_loss": 4.042074556752172, + "tokens_seen": 398524416 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044405215646940824, + "loss": 1.5518, + "theoretical_loss": 4.042000675432391, + "tokens_seen": 398589952 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004440421263791374, + "loss": 1.456, + "theoretical_loss": 4.041926809659821, + "tokens_seen": 398655488 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004440320962888666, + "loss": 1.4952, + "theoretical_loss": 4.0418529594286365, + "tokens_seen": 398721024 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004440220661985958, + "loss": 1.4988, + "theoretical_loss": 4.041779124733012, + "tokens_seen": 398786560 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044401203610832497, + "loss": 1.6147, + "theoretical_loss": 4.041705305567129, + "tokens_seen": 398852096 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004440020060180542, + "loss": 1.624, + "theoretical_loss": 4.041631501925169, + "tokens_seen": 398917632 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044399197592778333, + "loss": 1.546, + "theoretical_loss": 4.041557713801319, + "tokens_seen": 398983168 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044398194583751256, + "loss": 1.5724, + "theoretical_loss": 4.041483941189766, + "tokens_seen": 399048704 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004439719157472417, + "loss": 1.5263, + "theoretical_loss": 4.041410184084703, + "tokens_seen": 399114240 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004439618856569709, + "loss": 1.6068, + "theoretical_loss": 4.041336442480326, + "tokens_seen": 399179776 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004439518555667001, + "loss": 1.571, + "theoretical_loss": 4.041262716370833, + "tokens_seen": 399245312 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004439418254764293, + "loss": 1.6306, + "theoretical_loss": 4.041189005750424, + "tokens_seen": 399310848 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044393179538615847, + "loss": 1.5305, + "theoretical_loss": 4.041115310613307, + "tokens_seen": 399376384 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004439217652958877, + "loss": 1.5947, + "theoretical_loss": 4.0410416309536865, + "tokens_seen": 399441920 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044391173520561683, + "loss": 1.5673, + "theoretical_loss": 4.040967966765774, + "tokens_seen": 399507456 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044390170511534607, + "loss": 1.5538, + "theoretical_loss": 4.040894318043785, + "tokens_seen": 399572992 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004438916750250752, + "loss": 1.528, + "theoretical_loss": 4.040820684781934, + "tokens_seen": 399638528 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044388164493480443, + "loss": 1.4456, + "theoretical_loss": 4.040747066974443, + "tokens_seen": 399704064 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 139056, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4374828338623047, + "objective/train/theoretical_loss": 4.040673464615535, + "objective/train/tokens_used": 399769600, + "theoretical_loss": 4.040673464615535, + "tokens_seen": 399769600 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004438716148445336, + "loss": 1.4772, + "theoretical_loss": 4.040673464615535, + "tokens_seen": 399769600 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004438615847542628, + "loss": 1.438, + "theoretical_loss": 4.040599877699437, + "tokens_seen": 399835136 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044385155466399197, + "loss": 1.619, + "theoretical_loss": 4.040526306220377, + "tokens_seen": 399900672 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044384152457372115, + "loss": 1.5826, + "theoretical_loss": 4.040452750172588, + "tokens_seen": 399966208 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044383149448345033, + "loss": 1.5823, + "theoretical_loss": 4.040379209550307, + "tokens_seen": 400031744 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044382146439317957, + "loss": 1.5789, + "theoretical_loss": 4.0403056843477705, + "tokens_seen": 400097280 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004438114343029087, + "loss": 1.5358, + "theoretical_loss": 4.040232174559223, + "tokens_seen": 400162816 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044380140421263793, + "loss": 1.59, + "theoretical_loss": 4.040158680178907, + "tokens_seen": 400228352 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004437913741223671, + "loss": 1.5874, + "theoretical_loss": 4.040085201201072, + "tokens_seen": 400293888 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004437813440320963, + "loss": 1.6324, + "theoretical_loss": 4.0400117376199685, + "tokens_seen": 400359424 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004437713139418255, + "loss": 1.6565, + "theoretical_loss": 4.039938289429852, + "tokens_seen": 400424960 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044376128385155466, + "loss": 1.587, + "theoretical_loss": 4.039864856624979, + "tokens_seen": 400490496 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044375125376128384, + "loss": 1.5356, + "theoretical_loss": 4.039791439199609, + "tokens_seen": 400556032 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044374122367101307, + "loss": 1.4873, + "theoretical_loss": 4.039718037148006, + "tokens_seen": 400621568 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004437311935807422, + "loss": 1.4919, + "theoretical_loss": 4.039644650464438, + "tokens_seen": 400687104 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044372116349047143, + "loss": 1.4303, + "theoretical_loss": 4.039571279143172, + "tokens_seen": 400752640 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044371113340020056, + "loss": 1.4451, + "theoretical_loss": 4.039497923178484, + "tokens_seen": 400818176 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004437011033099298, + "loss": 1.3529, + "theoretical_loss": 4.039424582564646, + "tokens_seen": 400883712 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044369107321965903, + "loss": 1.6616, + "theoretical_loss": 4.03935125729594, + "tokens_seen": 400949248 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044368104312938816, + "loss": 1.569, + "theoretical_loss": 4.039277947366647, + "tokens_seen": 401014784 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004436710130391174, + "loss": 1.5697, + "theoretical_loss": 4.039204652771051, + "tokens_seen": 401080320 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004436609829488465, + "loss": 1.5246, + "theoretical_loss": 4.039131373503441, + "tokens_seen": 401145856 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044365095285857576, + "loss": 1.5858, + "theoretical_loss": 4.039058109558107, + "tokens_seen": 401211392 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044364092276830494, + "loss": 1.61, + "theoretical_loss": 4.038984860929346, + "tokens_seen": 401276928 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004436308926780341, + "loss": 1.5175, + "theoretical_loss": 4.038911627611451, + "tokens_seen": 401342464 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004436208625877633, + "loss": 1.5831, + "theoretical_loss": 4.038838409598727, + "tokens_seen": 401408000 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044361083249749254, + "loss": 1.5563, + "theoretical_loss": 4.038765206885472, + "tokens_seen": 401473536 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044360080240722166, + "loss": 1.5138, + "theoretical_loss": 4.038692019465997, + "tokens_seen": 401539072 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004435907723169509, + "loss": 1.5018, + "theoretical_loss": 4.038618847334609, + "tokens_seen": 401604608 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044358074222668, + "loss": 1.525, + "theoretical_loss": 4.038545690485622, + "tokens_seen": 401670144 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044357071213640926, + "loss": 1.554, + "theoretical_loss": 4.038472548913349, + "tokens_seen": 401735680 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044356068204613844, + "loss": 1.4465, + "theoretical_loss": 4.038399422612112, + "tokens_seen": 401801216 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004435506519558676, + "loss": 1.5704, + "theoretical_loss": 4.038326311576229, + "tokens_seen": 401866752 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004435406218655968, + "loss": 1.5109, + "theoretical_loss": 4.038253215800028, + "tokens_seen": 401932288 + }, + { + "epoch": 0.12, + "learning_rate": 0.000443530591775326, + "loss": 1.5868, + "theoretical_loss": 4.038180135277834, + "tokens_seen": 401997824 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044352056168505517, + "loss": 1.6071, + "theoretical_loss": 4.038107070003979, + "tokens_seen": 402063360 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004435105315947844, + "loss": 1.5317, + "theoretical_loss": 4.038034019972796, + "tokens_seen": 402128896 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044350050150451353, + "loss": 1.5342, + "theoretical_loss": 4.037960985178623, + "tokens_seen": 402194432 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044349047141424276, + "loss": 1.5101, + "theoretical_loss": 4.037887965615798, + "tokens_seen": 402259968 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004434804413239719, + "loss": 1.5556, + "theoretical_loss": 4.0378149612786665, + "tokens_seen": 402325504 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004434704112337011, + "loss": 1.5572, + "theoretical_loss": 4.037741972161571, + "tokens_seen": 402391040 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004434603811434303, + "loss": 1.5328, + "theoretical_loss": 4.037668998258863, + "tokens_seen": 402456576 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004434503510531595, + "loss": 1.5985, + "theoretical_loss": 4.037596039564892, + "tokens_seen": 402522112 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044344032096288867, + "loss": 1.5196, + "theoretical_loss": 4.0375230960740165, + "tokens_seen": 402587648 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004434302908726179, + "loss": 1.5648, + "theoretical_loss": 4.03745016778059, + "tokens_seen": 402653184 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044342026078234703, + "loss": 1.5146, + "theoretical_loss": 4.037377254678977, + "tokens_seen": 402718720 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044341023069207627, + "loss": 1.6344, + "theoretical_loss": 4.037304356763539, + "tokens_seen": 402784256 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004434002006018054, + "loss": 1.5612, + "theoretical_loss": 4.037231474028644, + "tokens_seen": 402849792 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044339017051153463, + "loss": 1.3994, + "theoretical_loss": 4.037158606468663, + "tokens_seen": 402915328 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004433801404212638, + "loss": 1.5022, + "theoretical_loss": 4.037085754077966, + "tokens_seen": 402980864 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 139972, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5354070663452148, + "objective/train/theoretical_loss": 4.037012916850932, + "objective/train/tokens_used": 403046400, + "theoretical_loss": 4.037012916850932, + "tokens_seen": 403046400 + }, + { + "epoch": 0.12, + "learning_rate": 0.000443370110330993, + "loss": 1.4989, + "theoretical_loss": 4.037012916850932, + "tokens_seen": 403046400 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044336008024072217, + "loss": 1.5922, + "theoretical_loss": 4.036940094781937, + "tokens_seen": 403111936 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044335005015045135, + "loss": 1.6183, + "theoretical_loss": 4.036867287865366, + "tokens_seen": 403177472 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044334002006018053, + "loss": 1.5719, + "theoretical_loss": 4.036794496095602, + "tokens_seen": 403243008 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044332998996990977, + "loss": 1.645, + "theoretical_loss": 4.0367217194670335, + "tokens_seen": 403308544 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004433199598796389, + "loss": 1.4671, + "theoretical_loss": 4.036648957974052, + "tokens_seen": 403374080 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044330992978936813, + "loss": 1.6085, + "theoretical_loss": 4.03657621161105, + "tokens_seen": 403439616 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004432998996990973, + "loss": 1.4985, + "theoretical_loss": 4.036503480372424, + "tokens_seen": 403505152 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004432898696088265, + "loss": 1.6496, + "theoretical_loss": 4.036430764252577, + "tokens_seen": 403570688 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004432798395185557, + "loss": 1.545, + "theoretical_loss": 4.036358063245908, + "tokens_seen": 403636224 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044326980942828486, + "loss": 1.5457, + "theoretical_loss": 4.036285377346826, + "tokens_seen": 403701760 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044325977933801404, + "loss": 1.5401, + "theoretical_loss": 4.036212706549738, + "tokens_seen": 403767296 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044324974924774327, + "loss": 1.475, + "theoretical_loss": 4.036140050849056, + "tokens_seen": 403832832 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004432397191574724, + "loss": 1.5558, + "theoretical_loss": 4.036067410239196, + "tokens_seen": 403898368 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044322968906720163, + "loss": 1.6139, + "theoretical_loss": 4.035994784714574, + "tokens_seen": 403963904 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044321965897693076, + "loss": 1.5542, + "theoretical_loss": 4.035922174269611, + "tokens_seen": 404029440 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044320962888666, + "loss": 1.5327, + "theoretical_loss": 4.035849578898731, + "tokens_seen": 404094976 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004431995987963892, + "loss": 1.546, + "theoretical_loss": 4.035776998596361, + "tokens_seen": 404160512 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044318956870611836, + "loss": 1.4902, + "theoretical_loss": 4.035704433356932, + "tokens_seen": 404226048 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044317953861584754, + "loss": 1.624, + "theoretical_loss": 4.035631883174873, + "tokens_seen": 404291584 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004431695085255767, + "loss": 1.5716, + "theoretical_loss": 4.035559348044622, + "tokens_seen": 404357120 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004431594784353059, + "loss": 1.5897, + "theoretical_loss": 4.035486827960617, + "tokens_seen": 404422656 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044314944834503514, + "loss": 1.5664, + "theoretical_loss": 4.035414322917299, + "tokens_seen": 404488192 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044313941825476427, + "loss": 1.5171, + "theoretical_loss": 4.035341832909114, + "tokens_seen": 404553728 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004431293881644935, + "loss": 1.4374, + "theoretical_loss": 4.035269357930507, + "tokens_seen": 404619264 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004431193580742227, + "loss": 1.4508, + "theoretical_loss": 4.03519689797593, + "tokens_seen": 404684800 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044310932798395186, + "loss": 1.6103, + "theoretical_loss": 4.035124453039836, + "tokens_seen": 404750336 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044309929789368104, + "loss": 1.4898, + "theoretical_loss": 4.035052023116679, + "tokens_seen": 404815872 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004430892678034102, + "loss": 1.5773, + "theoretical_loss": 4.0349796082009215, + "tokens_seen": 404881408 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004430792377131394, + "loss": 1.5632, + "theoretical_loss": 4.034907208287024, + "tokens_seen": 404946944 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044306920762286864, + "loss": 1.5212, + "theoretical_loss": 4.034834823369451, + "tokens_seen": 405012480 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044305917753259777, + "loss": 1.4518, + "theoretical_loss": 4.034762453442672, + "tokens_seen": 405078016 + }, + { + "epoch": 0.12, + "learning_rate": 0.000443049147442327, + "loss": 1.4974, + "theoretical_loss": 4.034690098501155, + "tokens_seen": 405143552 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044303911735205613, + "loss": 1.5894, + "theoretical_loss": 4.034617758539377, + "tokens_seen": 405209088 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044302908726178537, + "loss": 1.5364, + "theoretical_loss": 4.034545433551814, + "tokens_seen": 405274624 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044301905717151455, + "loss": 1.5317, + "theoretical_loss": 4.034473123532944, + "tokens_seen": 405340160 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044300902708124373, + "loss": 1.6389, + "theoretical_loss": 4.034400828477251, + "tokens_seen": 405405696 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004429989969909729, + "loss": 1.6977, + "theoretical_loss": 4.03432854837922, + "tokens_seen": 405471232 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004429889669007021, + "loss": 1.442, + "theoretical_loss": 4.03425628323334, + "tokens_seen": 405536768 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044297893681043127, + "loss": 1.6353, + "theoretical_loss": 4.034184033034102, + "tokens_seen": 405602304 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004429689067201605, + "loss": 1.4222, + "theoretical_loss": 4.034111797776001, + "tokens_seen": 405667840 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044295887662988963, + "loss": 1.5153, + "theoretical_loss": 4.034039577453533, + "tokens_seen": 405733376 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044294884653961887, + "loss": 1.631, + "theoretical_loss": 4.033967372061199, + "tokens_seen": 405798912 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004429388164493481, + "loss": 1.5461, + "theoretical_loss": 4.033895181593501, + "tokens_seen": 405864448 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044292878635907723, + "loss": 1.4419, + "theoretical_loss": 4.033823006044947, + "tokens_seen": 405929984 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044291875626880647, + "loss": 1.5077, + "theoretical_loss": 4.033750845410044, + "tokens_seen": 405995520 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004429087261785356, + "loss": 1.5479, + "theoretical_loss": 4.033678699683303, + "tokens_seen": 406061056 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044289869608826483, + "loss": 1.5545, + "theoretical_loss": 4.033606568859241, + "tokens_seen": 406126592 + }, + { + "epoch": 0.12, + "learning_rate": 0.000442888665997994, + "loss": 1.4311, + "theoretical_loss": 4.033534452932376, + "tokens_seen": 406192128 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004428786359077232, + "loss": 1.5005, + "theoretical_loss": 4.033462351897225, + "tokens_seen": 406257664 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 141185, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5115940570831299, + "objective/train/theoretical_loss": 4.033390265748315, + "objective/train/tokens_used": 406323200, + "theoretical_loss": 4.033390265748315, + "tokens_seen": 406323200 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044286860581745237, + "loss": 1.4341, + "theoretical_loss": 4.033390265748315, + "tokens_seen": 406323200 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044285857572718155, + "loss": 1.4713, + "theoretical_loss": 4.033318194480169, + "tokens_seen": 406388736 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044284854563691073, + "loss": 1.4078, + "theoretical_loss": 4.0332461380873195, + "tokens_seen": 406454272 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044283851554663997, + "loss": 1.5959, + "theoretical_loss": 4.033174096564295, + "tokens_seen": 406519808 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004428284854563691, + "loss": 1.5006, + "theoretical_loss": 4.033102069905635, + "tokens_seen": 406585344 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044281845536609833, + "loss": 1.5604, + "theoretical_loss": 4.033030058105872, + "tokens_seen": 406650880 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004428084252758275, + "loss": 1.5541, + "theoretical_loss": 4.032958061159551, + "tokens_seen": 406716416 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004427983951855567, + "loss": 1.6844, + "theoretical_loss": 4.032886079061214, + "tokens_seen": 406781952 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004427883650952859, + "loss": 1.542, + "theoretical_loss": 4.032814111805407, + "tokens_seen": 406847488 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044277833500501506, + "loss": 1.5396, + "theoretical_loss": 4.03274215938668, + "tokens_seen": 406913024 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044276830491474424, + "loss": 1.4746, + "theoretical_loss": 4.0326702217995845, + "tokens_seen": 406978560 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044275827482447347, + "loss": 1.5218, + "theoretical_loss": 4.0325982990386775, + "tokens_seen": 407044096 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004427482447342026, + "loss": 1.4011, + "theoretical_loss": 4.032526391098515, + "tokens_seen": 407109632 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044273821464393184, + "loss": 1.5889, + "theoretical_loss": 4.032454497973659, + "tokens_seen": 407175168 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044272818455366096, + "loss": 1.4982, + "theoretical_loss": 4.032382619658673, + "tokens_seen": 407240704 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004427181544633902, + "loss": 1.4472, + "theoretical_loss": 4.0323107561481235, + "tokens_seen": 407306240 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004427081243731194, + "loss": 1.7538, + "theoretical_loss": 4.0322389074365805, + "tokens_seen": 407371776 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044269809428284856, + "loss": 1.5698, + "theoretical_loss": 4.032167073518615, + "tokens_seen": 407437312 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044268806419257774, + "loss": 1.5477, + "theoretical_loss": 4.032095254388803, + "tokens_seen": 407502848 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004426780341023069, + "loss": 1.4743, + "theoretical_loss": 4.032023450041724, + "tokens_seen": 407568384 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004426680040120361, + "loss": 1.411, + "theoretical_loss": 4.031951660471956, + "tokens_seen": 407633920 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044265797392176534, + "loss": 1.5144, + "theoretical_loss": 4.031879885674085, + "tokens_seen": 407699456 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044264794383149447, + "loss": 1.5213, + "theoretical_loss": 4.031808125642697, + "tokens_seen": 407764992 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004426379137412237, + "loss": 1.5599, + "theoretical_loss": 4.031736380372382, + "tokens_seen": 407830528 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004426278836509529, + "loss": 1.5485, + "theoretical_loss": 4.031664649857731, + "tokens_seen": 407896064 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044261785356068206, + "loss": 1.4667, + "theoretical_loss": 4.031592934093341, + "tokens_seen": 407961600 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044260782347041124, + "loss": 1.4454, + "theoretical_loss": 4.031521233073809, + "tokens_seen": 408027136 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004425977933801404, + "loss": 1.6082, + "theoretical_loss": 4.031449546793736, + "tokens_seen": 408092672 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004425877632898696, + "loss": 1.5325, + "theoretical_loss": 4.031377875247726, + "tokens_seen": 408158208 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044257773319959884, + "loss": 1.5204, + "theoretical_loss": 4.031306218430385, + "tokens_seen": 408223744 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044256770310932797, + "loss": 1.5907, + "theoretical_loss": 4.031234576336322, + "tokens_seen": 408289280 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004425576730190572, + "loss": 1.5063, + "theoretical_loss": 4.0311629489601515, + "tokens_seen": 408354816 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044254764292878633, + "loss": 1.5777, + "theoretical_loss": 4.031091336296487, + "tokens_seen": 408420352 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044253761283851557, + "loss": 1.4583, + "theoretical_loss": 4.031019738339946, + "tokens_seen": 408485888 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044252758274824475, + "loss": 1.6549, + "theoretical_loss": 4.03094815508515, + "tokens_seen": 408551424 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044251755265797393, + "loss": 1.6061, + "theoretical_loss": 4.030876586526722, + "tokens_seen": 408616960 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004425075225677031, + "loss": 1.5664, + "theoretical_loss": 4.0308050326592895, + "tokens_seen": 408682496 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004424974924774323, + "loss": 1.4476, + "theoretical_loss": 4.030733493477481, + "tokens_seen": 408748032 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044248746238716147, + "loss": 1.5364, + "theoretical_loss": 4.030661968975929, + "tokens_seen": 408813568 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004424774322968907, + "loss": 1.5089, + "theoretical_loss": 4.030590459149267, + "tokens_seen": 408879104 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044246740220661983, + "loss": 1.4911, + "theoretical_loss": 4.030518963992135, + "tokens_seen": 408944640 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044245737211634907, + "loss": 1.4922, + "theoretical_loss": 4.030447483499171, + "tokens_seen": 409010176 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044244734202607825, + "loss": 1.5267, + "theoretical_loss": 4.03037601766502, + "tokens_seen": 409075712 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044243731193580743, + "loss": 1.4985, + "theoretical_loss": 4.030304566484329, + "tokens_seen": 409141248 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004424272818455366, + "loss": 1.4987, + "theoretical_loss": 4.030233129951744, + "tokens_seen": 409206784 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004424172517552658, + "loss": 1.4347, + "theoretical_loss": 4.03016170806192, + "tokens_seen": 409272320 + }, + { + "epoch": 0.12, + "learning_rate": 0.000442407221664995, + "loss": 1.4861, + "theoretical_loss": 4.030090300809509, + "tokens_seen": 409337856 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004423971915747242, + "loss": 1.5175, + "theoretical_loss": 4.03001890818917, + "tokens_seen": 409403392 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044238716148445334, + "loss": 1.4977, + "theoretical_loss": 4.029947530195563, + "tokens_seen": 409468928 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044237713139418257, + "loss": 1.5173, + "theoretical_loss": 4.029876166823351, + "tokens_seen": 409534464 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 142155, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.380353331565857, + "objective/train/theoretical_loss": 4.029804818067199, + "objective/train/tokens_used": 409600000, + "theoretical_loss": 4.029804818067199, + "tokens_seen": 409600000 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004423671013039117, + "loss": 1.4497, + "theoretical_loss": 4.029804818067199, + "tokens_seen": 409600000 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044235707121364093, + "loss": 1.5525, + "theoretical_loss": 4.029733483921777, + "tokens_seen": 409665536 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004423470411233701, + "loss": 1.473, + "theoretical_loss": 4.0296621643817545, + "tokens_seen": 409731072 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004423370110330993, + "loss": 1.6142, + "theoretical_loss": 4.029590859441807, + "tokens_seen": 409796608 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004423269809428285, + "loss": 1.5042, + "theoretical_loss": 4.029519569096612, + "tokens_seen": 409862144 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004423169508525577, + "loss": 1.3519, + "theoretical_loss": 4.029448293340847, + "tokens_seen": 409927680 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044230692076228684, + "loss": 1.5341, + "theoretical_loss": 4.029377032169197, + "tokens_seen": 409993216 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004422968906720161, + "loss": 1.66, + "theoretical_loss": 4.029305785576347, + "tokens_seen": 410058752 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004422868605817452, + "loss": 1.4982, + "theoretical_loss": 4.029234553556985, + "tokens_seen": 410124288 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044227683049147444, + "loss": 1.5046, + "theoretical_loss": 4.029163336105801, + "tokens_seen": 410189824 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004422668004012036, + "loss": 1.495, + "theoretical_loss": 4.029092133217489, + "tokens_seen": 410255360 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004422567703109328, + "loss": 1.5445, + "theoretical_loss": 4.029020944886746, + "tokens_seen": 410320896 + }, + { + "epoch": 0.12, + "learning_rate": 0.000442246740220662, + "loss": 1.5008, + "theoretical_loss": 4.0289497711082705, + "tokens_seen": 410386432 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044223671013039116, + "loss": 1.5213, + "theoretical_loss": 4.028878611876765, + "tokens_seen": 410451968 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044222668004012034, + "loss": 1.5729, + "theoretical_loss": 4.028807467186935, + "tokens_seen": 410517504 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004422166499498496, + "loss": 1.5616, + "theoretical_loss": 4.028736337033488, + "tokens_seen": 410583040 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004422066198595787, + "loss": 1.544, + "theoretical_loss": 4.028665221411133, + "tokens_seen": 410648576 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044219658976930794, + "loss": 1.5141, + "theoretical_loss": 4.028594120314585, + "tokens_seen": 410714112 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004421865596790371, + "loss": 1.4864, + "theoretical_loss": 4.028523033738558, + "tokens_seen": 410779648 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004421765295887663, + "loss": 1.5275, + "theoretical_loss": 4.028451961677772, + "tokens_seen": 410845184 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044216649949849554, + "loss": 1.6322, + "theoretical_loss": 4.028380904126948, + "tokens_seen": 410910720 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044215646940822467, + "loss": 1.5197, + "theoretical_loss": 4.0283098610808095, + "tokens_seen": 410976256 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004421464393179539, + "loss": 1.5011, + "theoretical_loss": 4.028238832534084, + "tokens_seen": 411041792 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004421364092276831, + "loss": 1.5438, + "theoretical_loss": 4.028167818481502, + "tokens_seen": 411107328 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044212637913741226, + "loss": 1.6619, + "theoretical_loss": 4.0280968189177955, + "tokens_seen": 411172864 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044211634904714144, + "loss": 1.5473, + "theoretical_loss": 4.028025833837699, + "tokens_seen": 411238400 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004421063189568706, + "loss": 1.4922, + "theoretical_loss": 4.027954863235951, + "tokens_seen": 411303936 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004420962888665998, + "loss": 1.4744, + "theoretical_loss": 4.0278839071072925, + "tokens_seen": 411369472 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044208625877632904, + "loss": 1.3638, + "theoretical_loss": 4.027812965446467, + "tokens_seen": 411435008 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044207622868605817, + "loss": 1.5487, + "theoretical_loss": 4.02774203824822, + "tokens_seen": 411500544 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004420661985957874, + "loss": 1.4603, + "theoretical_loss": 4.0276711255073, + "tokens_seen": 411566080 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044205616850551653, + "loss": 1.5385, + "theoretical_loss": 4.027600227218461, + "tokens_seen": 411631616 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044204613841524577, + "loss": 1.6152, + "theoretical_loss": 4.027529343376456, + "tokens_seen": 411697152 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044203610832497495, + "loss": 1.49, + "theoretical_loss": 4.027458473976041, + "tokens_seen": 411762688 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044202607823470413, + "loss": 1.4624, + "theoretical_loss": 4.027387619011977, + "tokens_seen": 411828224 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004420160481444333, + "loss": 1.4629, + "theoretical_loss": 4.027316778479027, + "tokens_seen": 411893760 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004420060180541625, + "loss": 1.4595, + "theoretical_loss": 4.027245952371956, + "tokens_seen": 411959296 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044199598796389167, + "loss": 1.4763, + "theoretical_loss": 4.027175140685532, + "tokens_seen": 412024832 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004419859578736209, + "loss": 1.4709, + "theoretical_loss": 4.027104343414527, + "tokens_seen": 412090368 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044197592778335003, + "loss": 1.394, + "theoretical_loss": 4.027033560553713, + "tokens_seen": 412155904 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044196589769307927, + "loss": 1.3589, + "theoretical_loss": 4.026962792097867, + "tokens_seen": 412221440 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044195586760280845, + "loss": 1.491, + "theoretical_loss": 4.026892038041767, + "tokens_seen": 412286976 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044194583751253763, + "loss": 1.5068, + "theoretical_loss": 4.026821298380197, + "tokens_seen": 412352512 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004419358074222668, + "loss": 1.5605, + "theoretical_loss": 4.026750573107939, + "tokens_seen": 412418048 + }, + { + "epoch": 0.12, + "learning_rate": 0.000441925777331996, + "loss": 1.5633, + "theoretical_loss": 4.0266798622197815, + "tokens_seen": 412483584 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004419157472417252, + "loss": 1.6406, + "theoretical_loss": 4.026609165710514, + "tokens_seen": 412549120 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004419057171514544, + "loss": 1.4781, + "theoretical_loss": 4.02653848357493, + "tokens_seen": 412614656 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044189568706118354, + "loss": 1.4509, + "theoretical_loss": 4.026467815807823, + "tokens_seen": 412680192 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044188565697091277, + "loss": 1.4402, + "theoretical_loss": 4.026397162403994, + "tokens_seen": 412745728 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004418756268806419, + "loss": 1.6954, + "theoretical_loss": 4.0263265233582395, + "tokens_seen": 412811264 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 143374, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4603501558303833, + "objective/train/theoretical_loss": 4.026255898665367, + "objective/train/tokens_used": 412876800, + "theoretical_loss": 4.026255898665367, + "tokens_seen": 412876800 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044186559679037113, + "loss": 1.5047, + "theoretical_loss": 4.026255898665367, + "tokens_seen": 412876800 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004418555667001003, + "loss": 1.532, + "theoretical_loss": 4.02618528832018, + "tokens_seen": 412942336 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004418455366098295, + "loss": 1.6009, + "theoretical_loss": 4.026114692317489, + "tokens_seen": 413007872 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004418355065195587, + "loss": 1.581, + "theoretical_loss": 4.026044110652105, + "tokens_seen": 413073408 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004418254764292879, + "loss": 1.4855, + "theoretical_loss": 4.025973543318841, + "tokens_seen": 413138944 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044181544633901704, + "loss": 1.4364, + "theoretical_loss": 4.025902990312517, + "tokens_seen": 413204480 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004418054162487463, + "loss": 1.4109, + "theoretical_loss": 4.025832451627949, + "tokens_seen": 413270016 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004417953861584754, + "loss": 1.4843, + "theoretical_loss": 4.025761927259961, + "tokens_seen": 413335552 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044178535606820464, + "loss": 1.3725, + "theoretical_loss": 4.0256914172033795, + "tokens_seen": 413401088 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004417753259779338, + "loss": 1.5108, + "theoretical_loss": 4.025620921453029, + "tokens_seen": 413466624 + }, + { + "epoch": 0.13, + "learning_rate": 0.000441765295887663, + "loss": 1.4666, + "theoretical_loss": 4.025550440003744, + "tokens_seen": 413532160 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004417552657973922, + "loss": 1.5692, + "theoretical_loss": 4.025479972850354, + "tokens_seen": 413597696 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044174523570712136, + "loss": 1.5724, + "theoretical_loss": 4.025409519987696, + "tokens_seen": 413663232 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044173520561685054, + "loss": 1.5282, + "theoretical_loss": 4.025339081410609, + "tokens_seen": 413728768 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004417251755265798, + "loss": 1.4799, + "theoretical_loss": 4.025268657113934, + "tokens_seen": 413794304 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004417151454363089, + "loss": 1.5949, + "theoretical_loss": 4.025198247092514, + "tokens_seen": 413859840 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044170511534603814, + "loss": 1.4249, + "theoretical_loss": 4.025127851341196, + "tokens_seen": 413925376 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044169508525576727, + "loss": 1.4536, + "theoretical_loss": 4.025057469854831, + "tokens_seen": 413990912 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004416850551654965, + "loss": 1.554, + "theoretical_loss": 4.024987102628267, + "tokens_seen": 414056448 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004416750250752257, + "loss": 1.5333, + "theoretical_loss": 4.024916749656362, + "tokens_seen": 414121984 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044166499498495487, + "loss": 1.4371, + "theoretical_loss": 4.024846410933971, + "tokens_seen": 414187520 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044165496489468405, + "loss": 1.3872, + "theoretical_loss": 4.024776086455955, + "tokens_seen": 414253056 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004416449348044133, + "loss": 1.5184, + "theoretical_loss": 4.024705776217177, + "tokens_seen": 414318592 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004416349047141424, + "loss": 1.522, + "theoretical_loss": 4.024635480212501, + "tokens_seen": 414384128 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044162487462387164, + "loss": 1.4614, + "theoretical_loss": 4.024565198436796, + "tokens_seen": 414449664 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044161484453360077, + "loss": 1.5029, + "theoretical_loss": 4.024494930884931, + "tokens_seen": 414515200 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044160481444333, + "loss": 1.5051, + "theoretical_loss": 4.024424677551781, + "tokens_seen": 414580736 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004415947843530592, + "loss": 1.4672, + "theoretical_loss": 4.02435443843222, + "tokens_seen": 414646272 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044158475426278837, + "loss": 1.6009, + "theoretical_loss": 4.024284213521128, + "tokens_seen": 414711808 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044157472417251755, + "loss": 1.5428, + "theoretical_loss": 4.024214002813386, + "tokens_seen": 414777344 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044156469408224673, + "loss": 1.5436, + "theoretical_loss": 4.024143806303877, + "tokens_seen": 414842880 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004415546639919759, + "loss": 1.65, + "theoretical_loss": 4.024073623987487, + "tokens_seen": 414908416 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044154463390170515, + "loss": 1.4913, + "theoretical_loss": 4.024003455859107, + "tokens_seen": 414973952 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004415346038114343, + "loss": 1.4833, + "theoretical_loss": 4.023933301913628, + "tokens_seen": 415039488 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004415245737211635, + "loss": 1.4458, + "theoretical_loss": 4.023863162145943, + "tokens_seen": 415105024 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044151454363089264, + "loss": 1.3875, + "theoretical_loss": 4.023793036550952, + "tokens_seen": 415170560 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044150451354062187, + "loss": 1.5651, + "theoretical_loss": 4.023722925123551, + "tokens_seen": 415236096 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044149448345035105, + "loss": 1.5579, + "theoretical_loss": 4.023652827858646, + "tokens_seen": 415301632 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044148445336008023, + "loss": 1.6115, + "theoretical_loss": 4.023582744751138, + "tokens_seen": 415367168 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004414744232698094, + "loss": 1.4708, + "theoretical_loss": 4.0235126757959385, + "tokens_seen": 415432704 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044146439317953865, + "loss": 1.534, + "theoretical_loss": 4.0234426209879555, + "tokens_seen": 415498240 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004414543630892678, + "loss": 1.5032, + "theoretical_loss": 4.0233725803221025, + "tokens_seen": 415563776 + }, + { + "epoch": 0.13, + "learning_rate": 0.000441444332998997, + "loss": 1.5959, + "theoretical_loss": 4.023302553793295, + "tokens_seen": 415629312 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004414343029087262, + "loss": 1.4526, + "theoretical_loss": 4.023232541396451, + "tokens_seen": 415694848 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004414242728184554, + "loss": 1.5153, + "theoretical_loss": 4.023162543126491, + "tokens_seen": 415760384 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004414142427281846, + "loss": 1.5443, + "theoretical_loss": 4.023092558978339, + "tokens_seen": 415825920 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044140421263791374, + "loss": 1.5772, + "theoretical_loss": 4.023022588946921, + "tokens_seen": 415891456 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044139418254764297, + "loss": 1.6711, + "theoretical_loss": 4.022952633027164, + "tokens_seen": 415956992 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004413841524573721, + "loss": 1.4766, + "theoretical_loss": 4.022882691214002, + "tokens_seen": 416022528 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044137412236710133, + "loss": 1.5364, + "theoretical_loss": 4.022812763502367, + "tokens_seen": 416088064 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 144329, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5347439050674438, + "objective/train/theoretical_loss": 4.022742849887195, + "objective/train/tokens_used": 416153600, + "theoretical_loss": 4.022742849887195, + "tokens_seen": 416153600 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004413640922768305, + "loss": 1.4786, + "theoretical_loss": 4.022742849887195, + "tokens_seen": 416153600 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004413540621865597, + "loss": 1.542, + "theoretical_loss": 4.022672950363427, + "tokens_seen": 416219136 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004413440320962889, + "loss": 1.5409, + "theoretical_loss": 4.022603064926004, + "tokens_seen": 416284672 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004413340020060181, + "loss": 1.4414, + "theoretical_loss": 4.02253319356987, + "tokens_seen": 416350208 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044132397191574724, + "loss": 1.5611, + "theoretical_loss": 4.022463336289971, + "tokens_seen": 416415744 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004413139418254765, + "loss": 1.686, + "theoretical_loss": 4.022393493081259, + "tokens_seen": 416481280 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004413039117352056, + "loss": 1.5133, + "theoretical_loss": 4.022323663938684, + "tokens_seen": 416546816 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044129388164493484, + "loss": 1.4701, + "theoretical_loss": 4.022253848857201, + "tokens_seen": 416612352 + }, + { + "epoch": 0.13, + "learning_rate": 0.000441283851554664, + "loss": 1.449, + "theoretical_loss": 4.022184047831768, + "tokens_seen": 416677888 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004412738214643932, + "loss": 1.4277, + "theoretical_loss": 4.022114260857345, + "tokens_seen": 416743424 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004412637913741224, + "loss": 1.4362, + "theoretical_loss": 4.022044487928895, + "tokens_seen": 416808960 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044125376128385156, + "loss": 1.458, + "theoretical_loss": 4.021974729041381, + "tokens_seen": 416874496 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044124373119358074, + "loss": 1.4993, + "theoretical_loss": 4.021904984189773, + "tokens_seen": 416940032 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044123370110331, + "loss": 1.4877, + "theoretical_loss": 4.021835253369041, + "tokens_seen": 417005568 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004412236710130391, + "loss": 1.6201, + "theoretical_loss": 4.021765536574157, + "tokens_seen": 417071104 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044121364092276834, + "loss": 1.3921, + "theoretical_loss": 4.021695833800098, + "tokens_seen": 417136640 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044120361083249747, + "loss": 1.5345, + "theoretical_loss": 4.02162614504184, + "tokens_seen": 417202176 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004411935807422267, + "loss": 1.5106, + "theoretical_loss": 4.021556470294366, + "tokens_seen": 417267712 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004411835506519559, + "loss": 1.4641, + "theoretical_loss": 4.021486809552657, + "tokens_seen": 417333248 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044117352056168507, + "loss": 1.4592, + "theoretical_loss": 4.021417162811701, + "tokens_seen": 417398784 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044116349047141425, + "loss": 1.4904, + "theoretical_loss": 4.021347530066485, + "tokens_seen": 417464320 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004411534603811435, + "loss": 1.4628, + "theoretical_loss": 4.0212779113120005, + "tokens_seen": 417529856 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004411434302908726, + "loss": 1.4679, + "theoretical_loss": 4.021208306543242, + "tokens_seen": 417595392 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044113340020060184, + "loss": 1.5555, + "theoretical_loss": 4.021138715755204, + "tokens_seen": 417660928 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044112337011033097, + "loss": 1.7032, + "theoretical_loss": 4.021069138942886, + "tokens_seen": 417726464 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004411133400200602, + "loss": 1.4749, + "theoretical_loss": 4.020999576101291, + "tokens_seen": 417792000 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004411033099297894, + "loss": 1.5052, + "theoretical_loss": 4.02093002722542, + "tokens_seen": 417857536 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044109327983951857, + "loss": 1.4845, + "theoretical_loss": 4.020860492310281, + "tokens_seen": 417923072 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044108324974924775, + "loss": 1.4829, + "theoretical_loss": 4.020790971350884, + "tokens_seen": 417988608 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044107321965897693, + "loss": 1.5398, + "theoretical_loss": 4.020721464342238, + "tokens_seen": 418054144 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004410631895687061, + "loss": 1.5247, + "theoretical_loss": 4.02065197127936, + "tokens_seen": 418119680 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044105315947843535, + "loss": 1.48, + "theoretical_loss": 4.020582492157265, + "tokens_seen": 418185216 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004410431293881645, + "loss": 1.5167, + "theoretical_loss": 4.020513026970973, + "tokens_seen": 418250752 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004410330992978937, + "loss": 1.4823, + "theoretical_loss": 4.020443575715506, + "tokens_seen": 418316288 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044102306920762284, + "loss": 1.5098, + "theoretical_loss": 4.020374138385887, + "tokens_seen": 418381824 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044101303911735207, + "loss": 1.5314, + "theoretical_loss": 4.020304714977145, + "tokens_seen": 418447360 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044100300902708125, + "loss": 1.5349, + "theoretical_loss": 4.020235305484308, + "tokens_seen": 418512896 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044099297893681043, + "loss": 1.5579, + "theoretical_loss": 4.020165909902409, + "tokens_seen": 418578432 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004409829488465396, + "loss": 1.5915, + "theoretical_loss": 4.020096528226483, + "tokens_seen": 418643968 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044097291875626885, + "loss": 1.5041, + "theoretical_loss": 4.020027160451566, + "tokens_seen": 418709504 + }, + { + "epoch": 0.13, + "learning_rate": 0.000440962888665998, + "loss": 1.4118, + "theoretical_loss": 4.019957806572697, + "tokens_seen": 418775040 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004409528585757272, + "loss": 1.4964, + "theoretical_loss": 4.019888466584921, + "tokens_seen": 418840576 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044094282848545634, + "loss": 1.485, + "theoretical_loss": 4.019819140483281, + "tokens_seen": 418906112 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004409327983951856, + "loss": 1.5318, + "theoretical_loss": 4.019749828262825, + "tokens_seen": 418971648 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044092276830491476, + "loss": 1.5442, + "theoretical_loss": 4.0196805299186025, + "tokens_seen": 419037184 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044091273821464394, + "loss": 1.5464, + "theoretical_loss": 4.019611245445666, + "tokens_seen": 419102720 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004409027081243731, + "loss": 1.4519, + "theoretical_loss": 4.01954197483907, + "tokens_seen": 419168256 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004408926780341023, + "loss": 1.4951, + "theoretical_loss": 4.019472718093874, + "tokens_seen": 419233792 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004408826479438315, + "loss": 1.5897, + "theoretical_loss": 4.019403475205135, + "tokens_seen": 419299328 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004408726178535607, + "loss": 1.4991, + "theoretical_loss": 4.019334246167919, + "tokens_seen": 419364864 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 145511, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5217585563659668, + "objective/train/theoretical_loss": 4.019265030977288, + "objective/train/tokens_used": 419430400, + "theoretical_loss": 4.019265030977288, + "tokens_seen": 419430400 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044086258776328984, + "loss": 1.5607, + "theoretical_loss": 4.019265030977288, + "tokens_seen": 419430400 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004408525576730191, + "loss": 1.4171, + "theoretical_loss": 4.019195829628311, + "tokens_seen": 419495936 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004408425275827482, + "loss": 1.5081, + "theoretical_loss": 4.019126642116059, + "tokens_seen": 419561472 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044083249749247744, + "loss": 1.5877, + "theoretical_loss": 4.019057468435603, + "tokens_seen": 419627008 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004408224674022066, + "loss": 1.4374, + "theoretical_loss": 4.01898830858202, + "tokens_seen": 419692544 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004408124373119358, + "loss": 1.5274, + "theoretical_loss": 4.018919162550386, + "tokens_seen": 419758080 + }, + { + "epoch": 0.13, + "learning_rate": 0.000440802407221665, + "loss": 1.4528, + "theoretical_loss": 4.018850030335783, + "tokens_seen": 419823616 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004407923771313942, + "loss": 1.4581, + "theoretical_loss": 4.018780911933293, + "tokens_seen": 419889152 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044078234704112335, + "loss": 1.6058, + "theoretical_loss": 4.018711807338001, + "tokens_seen": 419954688 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004407723169508526, + "loss": 1.508, + "theoretical_loss": 4.018642716544994, + "tokens_seen": 420020224 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004407622868605817, + "loss": 1.4697, + "theoretical_loss": 4.018573639549364, + "tokens_seen": 420085760 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044075225677031094, + "loss": 1.475, + "theoretical_loss": 4.018504576346203, + "tokens_seen": 420151296 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004407422266800401, + "loss": 1.7133, + "theoretical_loss": 4.018435526930607, + "tokens_seen": 420216832 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004407321965897693, + "loss": 1.4801, + "theoretical_loss": 4.018366491297673, + "tokens_seen": 420282368 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004407221664994985, + "loss": 1.5576, + "theoretical_loss": 4.018297469442502, + "tokens_seen": 420347904 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044071213640922767, + "loss": 1.5528, + "theoretical_loss": 4.018228461360197, + "tokens_seen": 420413440 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044070210631895685, + "loss": 1.4913, + "theoretical_loss": 4.018159467045862, + "tokens_seen": 420478976 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004406920762286861, + "loss": 1.5387, + "theoretical_loss": 4.018090486494607, + "tokens_seen": 420544512 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044068204613841527, + "loss": 1.4793, + "theoretical_loss": 4.0180215197015405, + "tokens_seen": 420610048 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044067201604814445, + "loss": 1.5171, + "theoretical_loss": 4.017952566661776, + "tokens_seen": 420675584 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004406619859578737, + "loss": 1.5203, + "theoretical_loss": 4.017883627370431, + "tokens_seen": 420741120 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004406519558676028, + "loss": 1.438, + "theoretical_loss": 4.017814701822619, + "tokens_seen": 420806656 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044064192577733204, + "loss": 1.6724, + "theoretical_loss": 4.017745790013464, + "tokens_seen": 420872192 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044063189568706117, + "loss": 1.5149, + "theoretical_loss": 4.017676891938088, + "tokens_seen": 420937728 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004406218655967904, + "loss": 1.4569, + "theoretical_loss": 4.0176080075916145, + "tokens_seen": 421003264 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004406118355065196, + "loss": 1.5719, + "theoretical_loss": 4.017539136969174, + "tokens_seen": 421068800 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044060180541624877, + "loss": 1.542, + "theoretical_loss": 4.017470280065895, + "tokens_seen": 421134336 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044059177532597795, + "loss": 1.4966, + "theoretical_loss": 4.017401436876911, + "tokens_seen": 421199872 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044058174523570713, + "loss": 1.4919, + "theoretical_loss": 4.017332607397358, + "tokens_seen": 421265408 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004405717151454363, + "loss": 1.5964, + "theoretical_loss": 4.017263791622373, + "tokens_seen": 421330944 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044056168505516555, + "loss": 1.4962, + "theoretical_loss": 4.017194989547095, + "tokens_seen": 421396480 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004405516549648947, + "loss": 1.4972, + "theoretical_loss": 4.017126201166668, + "tokens_seen": 421462016 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004405416248746239, + "loss": 1.4776, + "theoretical_loss": 4.017057426476239, + "tokens_seen": 421527552 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044053159478435304, + "loss": 1.5162, + "theoretical_loss": 4.016988665470953, + "tokens_seen": 421593088 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044052156469408227, + "loss": 1.4657, + "theoretical_loss": 4.016919918145961, + "tokens_seen": 421658624 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044051153460381145, + "loss": 1.4303, + "theoretical_loss": 4.016851184496415, + "tokens_seen": 421724160 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044050150451354063, + "loss": 1.4863, + "theoretical_loss": 4.016782464517472, + "tokens_seen": 421789696 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004404914744232698, + "loss": 1.4546, + "theoretical_loss": 4.016713758204288, + "tokens_seen": 421855232 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044048144433299905, + "loss": 1.4942, + "theoretical_loss": 4.016645065552023, + "tokens_seen": 421920768 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004404714142427282, + "loss": 1.4378, + "theoretical_loss": 4.0165763865558395, + "tokens_seen": 421986304 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004404613841524574, + "loss": 1.5238, + "theoretical_loss": 4.016507721210904, + "tokens_seen": 422051840 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044045135406218654, + "loss": 1.4831, + "theoretical_loss": 4.016439069512382, + "tokens_seen": 422117376 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004404413239719158, + "loss": 1.5006, + "theoretical_loss": 4.016370431455444, + "tokens_seen": 422182912 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044043129388164496, + "loss": 1.5481, + "theoretical_loss": 4.016301807035264, + "tokens_seen": 422248448 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044042126379137414, + "loss": 1.3866, + "theoretical_loss": 4.016233196247014, + "tokens_seen": 422313984 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004404112337011033, + "loss": 1.473, + "theoretical_loss": 4.016164599085872, + "tokens_seen": 422379520 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004404012036108325, + "loss": 1.5919, + "theoretical_loss": 4.016096015547021, + "tokens_seen": 422445056 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004403911735205617, + "loss": 1.4829, + "theoretical_loss": 4.016027445625638, + "tokens_seen": 422510592 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004403811434302909, + "loss": 1.5345, + "theoretical_loss": 4.015958889316911, + "tokens_seen": 422576128 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044037111334002004, + "loss": 1.5484, + "theoretical_loss": 4.015890346616026, + "tokens_seen": 422641664 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 146290, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5300352573394775, + "objective/train/theoretical_loss": 4.015821817518173, + "objective/train/tokens_used": 422707200, + "theoretical_loss": 4.015821817518173, + "tokens_seen": 422707200 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004403610832497493, + "loss": 1.5503, + "theoretical_loss": 4.015821817518173, + "tokens_seen": 422707200 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004403510531594784, + "loss": 1.5246, + "theoretical_loss": 4.015753302018544, + "tokens_seen": 422772736 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044034102306920764, + "loss": 1.4514, + "theoretical_loss": 4.015684800112332, + "tokens_seen": 422838272 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004403309929789368, + "loss": 1.4617, + "theoretical_loss": 4.015616311794735, + "tokens_seen": 422903808 + }, + { + "epoch": 0.13, + "learning_rate": 0.000440320962888666, + "loss": 1.5143, + "theoretical_loss": 4.015547837060953, + "tokens_seen": 422969344 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004403109327983952, + "loss": 1.5597, + "theoretical_loss": 4.015479375906187, + "tokens_seen": 423034880 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004403009027081244, + "loss": 1.4938, + "theoretical_loss": 4.0154109283256405, + "tokens_seen": 423100416 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044029087261785355, + "loss": 1.5596, + "theoretical_loss": 4.015342494314521, + "tokens_seen": 423165952 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004402808425275828, + "loss": 1.5276, + "theoretical_loss": 4.015274073868037, + "tokens_seen": 423231488 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004402708124373119, + "loss": 1.4108, + "theoretical_loss": 4.015205666981401, + "tokens_seen": 423297024 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044026078234704114, + "loss": 1.4373, + "theoretical_loss": 4.015137273649826, + "tokens_seen": 423362560 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004402507522567703, + "loss": 1.4868, + "theoretical_loss": 4.015068893868528, + "tokens_seen": 423428096 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004402407221664995, + "loss": 1.487, + "theoretical_loss": 4.015000527632726, + "tokens_seen": 423493632 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004402306920762287, + "loss": 1.5384, + "theoretical_loss": 4.014932174937642, + "tokens_seen": 423559168 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044022066198595787, + "loss": 1.4049, + "theoretical_loss": 4.014863835778499, + "tokens_seen": 423624704 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044021063189568705, + "loss": 1.5558, + "theoretical_loss": 4.014795510150522, + "tokens_seen": 423690240 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004402006018054163, + "loss": 1.6181, + "theoretical_loss": 4.014727198048942, + "tokens_seen": 423755776 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004401905717151454, + "loss": 1.4445, + "theoretical_loss": 4.014658899468988, + "tokens_seen": 423821312 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044018054162487465, + "loss": 1.5328, + "theoretical_loss": 4.014590614405893, + "tokens_seen": 423886848 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044017051153460383, + "loss": 1.4564, + "theoretical_loss": 4.014522342854895, + "tokens_seen": 423952384 + }, + { + "epoch": 0.13, + "learning_rate": 0.000440160481444333, + "loss": 1.554, + "theoretical_loss": 4.01445408481123, + "tokens_seen": 424017920 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004401504513540622, + "loss": 1.4591, + "theoretical_loss": 4.014385840270139, + "tokens_seen": 424083456 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044014042126379137, + "loss": 1.5404, + "theoretical_loss": 4.014317609226865, + "tokens_seen": 424148992 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044013039117352055, + "loss": 1.5585, + "theoretical_loss": 4.014249391676652, + "tokens_seen": 424214528 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004401203610832498, + "loss": 1.5099, + "theoretical_loss": 4.014181187614751, + "tokens_seen": 424280064 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004401103309929789, + "loss": 1.5366, + "theoretical_loss": 4.01411299703641, + "tokens_seen": 424345600 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044010030090270815, + "loss": 1.6419, + "theoretical_loss": 4.014044819936882, + "tokens_seen": 424411136 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004400902708124373, + "loss": 1.5263, + "theoretical_loss": 4.013976656311422, + "tokens_seen": 424476672 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004400802407221665, + "loss": 1.4582, + "theoretical_loss": 4.013908506155287, + "tokens_seen": 424542208 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004400702106318957, + "loss": 1.529, + "theoretical_loss": 4.013840369463737, + "tokens_seen": 424607744 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004400601805416249, + "loss": 1.4466, + "theoretical_loss": 4.013772246232035, + "tokens_seen": 424673280 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044005015045135406, + "loss": 1.4995, + "theoretical_loss": 4.013704136455445, + "tokens_seen": 424738816 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044004012036108324, + "loss": 1.6376, + "theoretical_loss": 4.013636040129233, + "tokens_seen": 424804352 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004400300902708124, + "loss": 1.509, + "theoretical_loss": 4.0135679572486715, + "tokens_seen": 424869888 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044002006018054165, + "loss": 1.5709, + "theoretical_loss": 4.013499887809028, + "tokens_seen": 424935424 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004400100300902708, + "loss": 1.5086, + "theoretical_loss": 4.01343183180558, + "tokens_seen": 425000960 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044, + "loss": 1.486, + "theoretical_loss": 4.013363789233602, + "tokens_seen": 425066496 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004399899699097292, + "loss": 1.485, + "theoretical_loss": 4.013295760088374, + "tokens_seen": 425132032 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004399799398194584, + "loss": 1.5088, + "theoretical_loss": 4.013227744365178, + "tokens_seen": 425197568 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043996990972918756, + "loss": 1.487, + "theoretical_loss": 4.013159742059296, + "tokens_seen": 425263104 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043995987963891674, + "loss": 1.569, + "theoretical_loss": 4.013091753166015, + "tokens_seen": 425328640 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004399498495486459, + "loss": 1.4309, + "theoretical_loss": 4.013023777680624, + "tokens_seen": 425394176 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043993981945837516, + "loss": 1.5301, + "theoretical_loss": 4.0129558155984135, + "tokens_seen": 425459712 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043992978936810434, + "loss": 1.5703, + "theoretical_loss": 4.012887866914675, + "tokens_seen": 425525248 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004399197592778335, + "loss": 1.447, + "theoretical_loss": 4.0128199316247075, + "tokens_seen": 425590784 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004399097291875627, + "loss": 1.5072, + "theoretical_loss": 4.0127520097238065, + "tokens_seen": 425656320 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004398996990972919, + "loss": 1.4984, + "theoretical_loss": 4.012684101207273, + "tokens_seen": 425721856 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004398896690070211, + "loss": 1.673, + "theoretical_loss": 4.01261620607041, + "tokens_seen": 425787392 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043987963891675024, + "loss": 1.4069, + "theoretical_loss": 4.012548324308522, + "tokens_seen": 425852928 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004398696088264795, + "loss": 1.4796, + "theoretical_loss": 4.012480455916918, + "tokens_seen": 425918464 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 147611, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.7261464595794678, + "objective/train/theoretical_loss": 4.012412600890906, + "objective/train/tokens_used": 425984000, + "theoretical_loss": 4.012412600890906, + "tokens_seen": 425984000 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004398595787362086, + "loss": 1.5668, + "theoretical_loss": 4.012412600890906, + "tokens_seen": 425984000 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043984954864593784, + "loss": 1.5127, + "theoretical_loss": 4.012344759225799, + "tokens_seen": 426049536 + }, + { + "epoch": 0.13, + "learning_rate": 0.000439839518555667, + "loss": 1.4694, + "theoretical_loss": 4.0122769309169115, + "tokens_seen": 426115072 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004398294884653962, + "loss": 1.5603, + "theoretical_loss": 4.0122091159595605, + "tokens_seen": 426180608 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004398194583751254, + "loss": 1.5527, + "theoretical_loss": 4.0121413143490665, + "tokens_seen": 426246144 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004398094282848546, + "loss": 1.5464, + "theoretical_loss": 4.012073526080749, + "tokens_seen": 426311680 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043979939819458375, + "loss": 1.6912, + "theoretical_loss": 4.012005751149934, + "tokens_seen": 426377216 + }, + { + "epoch": 0.13, + "learning_rate": 0.000439789368104313, + "loss": 1.5639, + "theoretical_loss": 4.011937989551946, + "tokens_seen": 426442752 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004397793380140421, + "loss": 1.5093, + "theoretical_loss": 4.011870241282114, + "tokens_seen": 426508288 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043976930792377134, + "loss": 1.4285, + "theoretical_loss": 4.011802506335771, + "tokens_seen": 426573824 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004397592778335005, + "loss": 1.4899, + "theoretical_loss": 4.011734784708249, + "tokens_seen": 426639360 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004397492477432297, + "loss": 1.5732, + "theoretical_loss": 4.011667076394884, + "tokens_seen": 426704896 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004397392176529589, + "loss": 1.5201, + "theoretical_loss": 4.011599381391014, + "tokens_seen": 426770432 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043972918756268807, + "loss": 1.5422, + "theoretical_loss": 4.0115316996919805, + "tokens_seen": 426835968 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043971915747241725, + "loss": 1.5189, + "theoretical_loss": 4.011464031293125, + "tokens_seen": 426901504 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004397091273821465, + "loss": 1.4903, + "theoretical_loss": 4.0113963761897935, + "tokens_seen": 426967040 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004396990972918756, + "loss": 1.4017, + "theoretical_loss": 4.011328734377333, + "tokens_seen": 427032576 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043968906720160485, + "loss": 1.566, + "theoretical_loss": 4.011261105851093, + "tokens_seen": 427098112 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043967903711133403, + "loss": 1.4956, + "theoretical_loss": 4.011193490606427, + "tokens_seen": 427163648 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004396690070210632, + "loss": 1.3807, + "theoretical_loss": 4.01112588863869, + "tokens_seen": 427229184 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004396589769307924, + "loss": 1.5204, + "theoretical_loss": 4.011058299943237, + "tokens_seen": 427294720 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043964894684052157, + "loss": 1.4025, + "theoretical_loss": 4.010990724515427, + "tokens_seen": 427360256 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043963891675025075, + "loss": 1.5144, + "theoretical_loss": 4.010923162350624, + "tokens_seen": 427425792 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043962888665998, + "loss": 1.5434, + "theoretical_loss": 4.01085561344419, + "tokens_seen": 427491328 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004396188565697091, + "loss": 1.4438, + "theoretical_loss": 4.010788077791491, + "tokens_seen": 427556864 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043960882647943835, + "loss": 1.5147, + "theoretical_loss": 4.0107205553878975, + "tokens_seen": 427622400 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004395987963891675, + "loss": 1.53, + "theoretical_loss": 4.010653046228779, + "tokens_seen": 427687936 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004395887662988967, + "loss": 1.3993, + "theoretical_loss": 4.010585550309508, + "tokens_seen": 427753472 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004395787362086259, + "loss": 1.4695, + "theoretical_loss": 4.010518067625461, + "tokens_seen": 427819008 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004395687061183551, + "loss": 1.454, + "theoretical_loss": 4.010450598172016, + "tokens_seen": 427884544 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043955867602808426, + "loss": 1.4888, + "theoretical_loss": 4.010383141944553, + "tokens_seen": 427950080 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043954864593781344, + "loss": 1.4901, + "theoretical_loss": 4.010315698938455, + "tokens_seen": 428015616 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004395386158475426, + "loss": 1.4563, + "theoretical_loss": 4.0102482691491055, + "tokens_seen": 428081152 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043952858575727185, + "loss": 1.4643, + "theoretical_loss": 4.010180852571893, + "tokens_seen": 428146688 + }, + { + "epoch": 0.13, + "learning_rate": 0.000439518555667001, + "loss": 1.4007, + "theoretical_loss": 4.010113449202206, + "tokens_seen": 428212224 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004395085255767302, + "loss": 1.3895, + "theoretical_loss": 4.010046059035437, + "tokens_seen": 428277760 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004394984954864594, + "loss": 1.4821, + "theoretical_loss": 4.009978682066979, + "tokens_seen": 428343296 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004394884653961886, + "loss": 1.4968, + "theoretical_loss": 4.00991131829223, + "tokens_seen": 428408832 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043947843530591776, + "loss": 1.4584, + "theoretical_loss": 4.009843967706586, + "tokens_seen": 428474368 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043946840521564694, + "loss": 1.4957, + "theoretical_loss": 4.009776630305451, + "tokens_seen": 428539904 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004394583751253761, + "loss": 1.497, + "theoretical_loss": 4.009709306084227, + "tokens_seen": 428605440 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043944834503510536, + "loss": 1.5957, + "theoretical_loss": 4.00964199503832, + "tokens_seen": 428670976 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004394383149448345, + "loss": 1.4788, + "theoretical_loss": 4.009574697163137, + "tokens_seen": 428736512 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004394282848545637, + "loss": 1.5391, + "theoretical_loss": 4.00950741245409, + "tokens_seen": 428802048 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043941825476429285, + "loss": 1.5122, + "theoretical_loss": 4.009440140906589, + "tokens_seen": 428867584 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004394082246740221, + "loss": 1.5993, + "theoretical_loss": 4.00937288251605, + "tokens_seen": 428933120 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043939819458375126, + "loss": 1.5774, + "theoretical_loss": 4.009305637277892, + "tokens_seen": 428998656 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043938816449348044, + "loss": 1.6077, + "theoretical_loss": 4.009238405187531, + "tokens_seen": 429064192 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004393781344032096, + "loss": 1.5164, + "theoretical_loss": 4.009171186240391, + "tokens_seen": 429129728 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004393681043129388, + "loss": 1.5589, + "theoretical_loss": 4.009103980431896, + "tokens_seen": 429195264 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 148719, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.607401728630066, + "objective/train/theoretical_loss": 4.009036787757472, + "objective/train/tokens_used": 429260800, + "theoretical_loss": 4.009036787757472, + "tokens_seen": 429260800 + }, + { + "epoch": 0.13, + "learning_rate": 0.000439358074222668, + "loss": 1.6526, + "theoretical_loss": 4.009036787757472, + "tokens_seen": 429260800 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004393480441323972, + "loss": 1.4568, + "theoretical_loss": 4.0089696082125466, + "tokens_seen": 429326336 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043933801404212635, + "loss": 1.5742, + "theoretical_loss": 4.008902441792552, + "tokens_seen": 429391872 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004393279839518556, + "loss": 1.478, + "theoretical_loss": 4.00883528849292, + "tokens_seen": 429457408 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043931795386158477, + "loss": 1.4597, + "theoretical_loss": 4.008768148309087, + "tokens_seen": 429522944 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043930792377131395, + "loss": 1.5675, + "theoretical_loss": 4.00870102123649, + "tokens_seen": 429588480 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043929789368104313, + "loss": 1.5286, + "theoretical_loss": 4.008633907270571, + "tokens_seen": 429654016 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004392878635907723, + "loss": 1.4961, + "theoretical_loss": 4.008566806406771, + "tokens_seen": 429719552 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004392778335005015, + "loss": 1.5349, + "theoretical_loss": 4.0084997186405324, + "tokens_seen": 429785088 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004392678034102307, + "loss": 1.4921, + "theoretical_loss": 4.008432643967305, + "tokens_seen": 429850624 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043925777331995985, + "loss": 1.4908, + "theoretical_loss": 4.008365582382538, + "tokens_seen": 429916160 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004392477432296891, + "loss": 1.3929, + "theoretical_loss": 4.008298533881681, + "tokens_seen": 429981696 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004392377131394182, + "loss": 1.5593, + "theoretical_loss": 4.008231498460189, + "tokens_seen": 430047232 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043922768304914745, + "loss": 1.4999, + "theoretical_loss": 4.008164476113517, + "tokens_seen": 430112768 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043921765295887663, + "loss": 1.4248, + "theoretical_loss": 4.008097466837124, + "tokens_seen": 430178304 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004392076228686058, + "loss": 1.5064, + "theoretical_loss": 4.00803047062647, + "tokens_seen": 430243840 + }, + { + "epoch": 0.13, + "learning_rate": 0.000439197592778335, + "loss": 1.569, + "theoretical_loss": 4.007963487477018, + "tokens_seen": 430309376 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043918756268806423, + "loss": 1.4588, + "theoretical_loss": 4.007896517384233, + "tokens_seen": 430374912 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004391775325977934, + "loss": 1.4358, + "theoretical_loss": 4.007829560343582, + "tokens_seen": 430440448 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004391675025075226, + "loss": 1.4713, + "theoretical_loss": 4.007762616350535, + "tokens_seen": 430505984 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043915747241725177, + "loss": 1.4445, + "theoretical_loss": 4.007695685400563, + "tokens_seen": 430571520 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043914744232698095, + "loss": 1.4984, + "theoretical_loss": 4.007628767489142, + "tokens_seen": 430637056 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004391374122367102, + "loss": 1.5579, + "theoretical_loss": 4.007561862611746, + "tokens_seen": 430702592 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004391273821464393, + "loss": 1.495, + "theoretical_loss": 4.007494970763855, + "tokens_seen": 430768128 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043911735205616855, + "loss": 1.5478, + "theoretical_loss": 4.00742809194095, + "tokens_seen": 430833664 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004391073219658977, + "loss": 1.5014, + "theoretical_loss": 4.007361226138513, + "tokens_seen": 430899200 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004390972918756269, + "loss": 1.4545, + "theoretical_loss": 4.00729437335203, + "tokens_seen": 430964736 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004390872617853561, + "loss": 1.4253, + "theoretical_loss": 4.007227533576988, + "tokens_seen": 431030272 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004390772316950853, + "loss": 1.474, + "theoretical_loss": 4.007160706808879, + "tokens_seen": 431095808 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043906720160481446, + "loss": 1.5889, + "theoretical_loss": 4.007093893043192, + "tokens_seen": 431161344 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043905717151454364, + "loss": 1.4838, + "theoretical_loss": 4.007027092275424, + "tokens_seen": 431226880 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004390471414242728, + "loss": 1.5149, + "theoretical_loss": 4.006960304501069, + "tokens_seen": 431292416 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043903711133400205, + "loss": 1.6176, + "theoretical_loss": 4.0068935297156285, + "tokens_seen": 431357952 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004390270812437312, + "loss": 1.5024, + "theoretical_loss": 4.0068267679146015, + "tokens_seen": 431423488 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004390170511534604, + "loss": 1.5563, + "theoretical_loss": 4.006760019093493, + "tokens_seen": 431489024 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004390070210631896, + "loss": 1.4522, + "theoretical_loss": 4.006693283247808, + "tokens_seen": 431554560 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004389969909729188, + "loss": 1.5359, + "theoretical_loss": 4.0066265603730535, + "tokens_seen": 431620096 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043898696088264796, + "loss": 1.5452, + "theoretical_loss": 4.006559850464741, + "tokens_seen": 431685632 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043897693079237714, + "loss": 1.4854, + "theoretical_loss": 4.006493153518381, + "tokens_seen": 431751168 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004389669007021063, + "loss": 1.4247, + "theoretical_loss": 4.006426469529488, + "tokens_seen": 431816704 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043895687061183556, + "loss": 1.5344, + "theoretical_loss": 4.006359798493582, + "tokens_seen": 431882240 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004389468405215647, + "loss": 1.4508, + "theoretical_loss": 4.006293140406179, + "tokens_seen": 431947776 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004389368104312939, + "loss": 1.4775, + "theoretical_loss": 4.0062264952627995, + "tokens_seen": 432013312 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043892678034102305, + "loss": 1.4347, + "theoretical_loss": 4.00615986305897, + "tokens_seen": 432078848 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004389167502507523, + "loss": 1.5353, + "theoretical_loss": 4.006093243790214, + "tokens_seen": 432144384 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043890672016048146, + "loss": 1.4944, + "theoretical_loss": 4.0060266374520594, + "tokens_seen": 432209920 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043889669007021064, + "loss": 1.5052, + "theoretical_loss": 4.005960044040037, + "tokens_seen": 432275456 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004388866599799398, + "loss": 1.5208, + "theoretical_loss": 4.005893463549679, + "tokens_seen": 432340992 + }, + { + "epoch": 0.13, + "learning_rate": 0.000438876629889669, + "loss": 1.4472, + "theoretical_loss": 4.0058268959765195, + "tokens_seen": 432406528 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004388665997993982, + "loss": 1.4905, + "theoretical_loss": 4.005760341316096, + "tokens_seen": 432472064 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 149584, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6581988334655762, + "objective/train/theoretical_loss": 4.0056937995639466, + "objective/train/tokens_used": 432537600, + "theoretical_loss": 4.0056937995639466, + "tokens_seen": 432537600 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004388565697091274, + "loss": 1.5674, + "theoretical_loss": 4.0056937995639466, + "tokens_seen": 432537600 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043884653961885655, + "loss": 1.4984, + "theoretical_loss": 4.005627270715614, + "tokens_seen": 432603136 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004388365095285858, + "loss": 1.5005, + "theoretical_loss": 4.00556075476664, + "tokens_seen": 432668672 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043882647943831497, + "loss": 1.5191, + "theoretical_loss": 4.005494251712571, + "tokens_seen": 432734208 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043881644934804415, + "loss": 1.4978, + "theoretical_loss": 4.005427761548955, + "tokens_seen": 432799744 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043880641925777333, + "loss": 1.5434, + "theoretical_loss": 4.005361284271341, + "tokens_seen": 432865280 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004387963891675025, + "loss": 1.4649, + "theoretical_loss": 4.005294819875283, + "tokens_seen": 432930816 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004387863590772317, + "loss": 1.5563, + "theoretical_loss": 4.005228368356335, + "tokens_seen": 432996352 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004387763289869609, + "loss": 1.4229, + "theoretical_loss": 4.005161929710052, + "tokens_seen": 433061888 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043876629889669005, + "loss": 1.574, + "theoretical_loss": 4.0050955039319955, + "tokens_seen": 433127424 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004387562688064193, + "loss": 1.5592, + "theoretical_loss": 4.0050290910177235, + "tokens_seen": 433192960 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004387462387161484, + "loss": 1.4761, + "theoretical_loss": 4.004962690962802, + "tokens_seen": 433258496 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043873620862587765, + "loss": 1.5325, + "theoretical_loss": 4.004896303762797, + "tokens_seen": 433324032 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043872617853560683, + "loss": 1.5283, + "theoretical_loss": 4.0048299294132725, + "tokens_seen": 433389568 + }, + { + "epoch": 0.13, + "learning_rate": 0.000438716148445336, + "loss": 1.49, + "theoretical_loss": 4.0047635679098015, + "tokens_seen": 433455104 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004387061183550652, + "loss": 1.4476, + "theoretical_loss": 4.004697219247955, + "tokens_seen": 433520640 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043869608826479443, + "loss": 1.5897, + "theoretical_loss": 4.004630883423308, + "tokens_seen": 433586176 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043868605817452356, + "loss": 1.4757, + "theoretical_loss": 4.004564560431437, + "tokens_seen": 433651712 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004386760280842528, + "loss": 1.4068, + "theoretical_loss": 4.004498250267919, + "tokens_seen": 433717248 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004386659979939819, + "loss": 1.4429, + "theoretical_loss": 4.004431952928337, + "tokens_seen": 433782784 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043865596790371115, + "loss": 1.5654, + "theoretical_loss": 4.004365668408273, + "tokens_seen": 433848320 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043864593781344033, + "loss": 1.5925, + "theoretical_loss": 4.004299396703312, + "tokens_seen": 433913856 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004386359077231695, + "loss": 1.5723, + "theoretical_loss": 4.004233137809042, + "tokens_seen": 433979392 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004386258776328987, + "loss": 1.4885, + "theoretical_loss": 4.004166891721052, + "tokens_seen": 434044928 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004386158475426279, + "loss": 1.5817, + "theoretical_loss": 4.004100658434934, + "tokens_seen": 434110464 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043860581745235706, + "loss": 1.6338, + "theoretical_loss": 4.004034437946283, + "tokens_seen": 434176000 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004385957873620863, + "loss": 1.3572, + "theoretical_loss": 4.003968230250694, + "tokens_seen": 434241536 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004385857572718154, + "loss": 1.5995, + "theoretical_loss": 4.003902035343766, + "tokens_seen": 434307072 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043857572718154466, + "loss": 1.5035, + "theoretical_loss": 4.003835853221099, + "tokens_seen": 434372608 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004385656970912738, + "loss": 1.5349, + "theoretical_loss": 4.003769683878296, + "tokens_seen": 434438144 + }, + { + "epoch": 0.13, + "learning_rate": 0.000438555667001003, + "loss": 1.5011, + "theoretical_loss": 4.003703527310962, + "tokens_seen": 434503680 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004385456369107322, + "loss": 1.5309, + "theoretical_loss": 4.003637383514704, + "tokens_seen": 434569216 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004385356068204614, + "loss": 1.4745, + "theoretical_loss": 4.00357125248513, + "tokens_seen": 434634752 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043852557673019056, + "loss": 1.591, + "theoretical_loss": 4.003505134217853, + "tokens_seen": 434700288 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004385155466399198, + "loss": 1.5353, + "theoretical_loss": 4.0034390287084864, + "tokens_seen": 434765824 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004385055165496489, + "loss": 1.4044, + "theoretical_loss": 4.003372935952646, + "tokens_seen": 434831360 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043849548645937816, + "loss": 1.5807, + "theoretical_loss": 4.003306855945948, + "tokens_seen": 434896896 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004384854563691073, + "loss": 1.529, + "theoretical_loss": 4.003240788684015, + "tokens_seen": 434962432 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004384754262788365, + "loss": 1.5896, + "theoretical_loss": 4.003174734162467, + "tokens_seen": 435027968 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004384653961885657, + "loss": 1.4876, + "theoretical_loss": 4.00310869237693, + "tokens_seen": 435093504 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004384553660982949, + "loss": 1.3698, + "theoretical_loss": 4.003042663323029, + "tokens_seen": 435159040 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043844533600802407, + "loss": 1.5352, + "theoretical_loss": 4.0029766469963945, + "tokens_seen": 435224576 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043843530591775325, + "loss": 1.5024, + "theoretical_loss": 4.0029106433926565, + "tokens_seen": 435290112 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004384252758274825, + "loss": 1.4908, + "theoretical_loss": 4.002844652507449, + "tokens_seen": 435355648 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043841524573721166, + "loss": 1.5008, + "theoretical_loss": 4.002778674336405, + "tokens_seen": 435421184 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043840521564694084, + "loss": 1.5069, + "theoretical_loss": 4.002712708875162, + "tokens_seen": 435486720 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043839518555667, + "loss": 1.5118, + "theoretical_loss": 4.002646756119362, + "tokens_seen": 435552256 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004383851554663992, + "loss": 1.461, + "theoretical_loss": 4.002580816064645, + "tokens_seen": 435617792 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004383751253761284, + "loss": 1.4473, + "theoretical_loss": 4.002514888706655, + "tokens_seen": 435683328 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004383650952858576, + "loss": 1.5121, + "theoretical_loss": 4.002448974041037, + "tokens_seen": 435748864 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 150729, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4260175228118896, + "objective/train/theoretical_loss": 4.0023830720634415, + "objective/train/tokens_used": 435814400, + "theoretical_loss": 4.0023830720634415, + "tokens_seen": 435814400 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043835506519558675, + "loss": 1.4658, + "theoretical_loss": 4.0023830720634415, + "tokens_seen": 435814400 + }, + { + "epoch": 0.13, + "learning_rate": 0.000438345035105316, + "loss": 1.5419, + "theoretical_loss": 4.0023171827695165, + "tokens_seen": 435879936 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043833500501504517, + "loss": 1.5176, + "theoretical_loss": 4.002251306154916, + "tokens_seen": 435945472 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043832497492477435, + "loss": 1.5756, + "theoretical_loss": 4.002185442215293, + "tokens_seen": 436011008 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043831494483450353, + "loss": 1.4923, + "theoretical_loss": 4.002119590946305, + "tokens_seen": 436076544 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004383049147442327, + "loss": 1.4951, + "theoretical_loss": 4.002053752343611, + "tokens_seen": 436142080 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004382948846539619, + "loss": 1.5552, + "theoretical_loss": 4.001987926402871, + "tokens_seen": 436207616 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004382848545636911, + "loss": 1.5393, + "theoretical_loss": 4.001922113119749, + "tokens_seen": 436273152 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043827482447342025, + "loss": 1.5328, + "theoretical_loss": 4.00185631248991, + "tokens_seen": 436338688 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004382647943831495, + "loss": 1.5397, + "theoretical_loss": 4.001790524509021, + "tokens_seen": 436404224 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004382547642928786, + "loss": 1.6634, + "theoretical_loss": 4.001724749172752, + "tokens_seen": 436469760 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043824473420260785, + "loss": 1.6391, + "theoretical_loss": 4.001658986476775, + "tokens_seen": 436535296 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043823470411233703, + "loss": 1.4718, + "theoretical_loss": 4.0015932364167615, + "tokens_seen": 436600832 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004382246740220662, + "loss": 1.512, + "theoretical_loss": 4.00152749898839, + "tokens_seen": 436666368 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004382146439317954, + "loss": 1.6395, + "theoretical_loss": 4.001461774187338, + "tokens_seen": 436731904 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043820461384152463, + "loss": 1.564, + "theoretical_loss": 4.001396062009285, + "tokens_seen": 436797440 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043819458375125376, + "loss": 1.5386, + "theoretical_loss": 4.001330362449914, + "tokens_seen": 436862976 + }, + { + "epoch": 0.13, + "learning_rate": 0.000438184553660983, + "loss": 1.4296, + "theoretical_loss": 4.001264675504908, + "tokens_seen": 436928512 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004381745235707121, + "loss": 1.5022, + "theoretical_loss": 4.001199001169955, + "tokens_seen": 436994048 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043816449348044135, + "loss": 1.5518, + "theoretical_loss": 4.001133339440743, + "tokens_seen": 437059584 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043815446339017053, + "loss": 1.6337, + "theoretical_loss": 4.001067690312963, + "tokens_seen": 437125120 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004381444332998997, + "loss": 1.4313, + "theoretical_loss": 4.001002053782307, + "tokens_seen": 437190656 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004381344032096289, + "loss": 1.4185, + "theoretical_loss": 4.0009364298444705, + "tokens_seen": 437256192 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004381243731193581, + "loss": 1.5691, + "theoretical_loss": 4.000870818495152, + "tokens_seen": 437321728 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043811434302908726, + "loss": 1.4168, + "theoretical_loss": 4.000805219730049, + "tokens_seen": 437387264 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004381043129388165, + "loss": 1.5156, + "theoretical_loss": 4.000739633544863, + "tokens_seen": 437452800 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004380942828485456, + "loss": 1.5683, + "theoretical_loss": 4.000674059935298, + "tokens_seen": 437518336 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043808425275827486, + "loss": 1.5516, + "theoretical_loss": 4.000608498897059, + "tokens_seen": 437583872 + }, + { + "epoch": 0.13, + "learning_rate": 0.000438074222668004, + "loss": 1.512, + "theoretical_loss": 4.000542950425855, + "tokens_seen": 437649408 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004380641925777332, + "loss": 1.4492, + "theoretical_loss": 4.000477414517394, + "tokens_seen": 437714944 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004380541624874624, + "loss": 1.518, + "theoretical_loss": 4.000411891167389, + "tokens_seen": 437780480 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004380441323971916, + "loss": 1.608, + "theoretical_loss": 4.000346380371554, + "tokens_seen": 437846016 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043803410230692076, + "loss": 1.4663, + "theoretical_loss": 4.000280882125605, + "tokens_seen": 437911552 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043802407221665, + "loss": 1.5529, + "theoretical_loss": 4.000215396425259, + "tokens_seen": 437977088 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004380140421263791, + "loss": 1.402, + "theoretical_loss": 4.000149923266237, + "tokens_seen": 438042624 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043800401203610836, + "loss": 1.4581, + "theoretical_loss": 4.0000844626442635, + "tokens_seen": 438108160 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004379939819458375, + "loss": 1.4371, + "theoretical_loss": 4.000019014555059, + "tokens_seen": 438173696 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004379839518555667, + "loss": 1.4538, + "theoretical_loss": 3.9999535789943543, + "tokens_seen": 438239232 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004379739217652959, + "loss": 1.4979, + "theoretical_loss": 3.9998881559578754, + "tokens_seen": 438304768 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004379638916750251, + "loss": 1.3717, + "theoretical_loss": 3.999822745441353, + "tokens_seen": 438370304 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043795386158475427, + "loss": 1.4897, + "theoretical_loss": 3.9997573474405215, + "tokens_seen": 438435840 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043794383149448345, + "loss": 1.4988, + "theoretical_loss": 3.9996919619511146, + "tokens_seen": 438501376 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043793380140421263, + "loss": 1.5114, + "theoretical_loss": 3.9996265889688702, + "tokens_seen": 438566912 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043792377131394186, + "loss": 1.6067, + "theoretical_loss": 3.999561228489527, + "tokens_seen": 438632448 + }, + { + "epoch": 0.13, + "learning_rate": 0.000437913741223671, + "loss": 1.5713, + "theoretical_loss": 3.9994958805088268, + "tokens_seen": 438697984 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004379037111334002, + "loss": 1.4704, + "theoretical_loss": 3.999430545022512, + "tokens_seen": 438763520 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043789368104312935, + "loss": 1.5948, + "theoretical_loss": 3.999365222026329, + "tokens_seen": 438829056 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004378836509528586, + "loss": 1.4534, + "theoretical_loss": 3.999299911516024, + "tokens_seen": 438894592 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043787362086258777, + "loss": 1.4645, + "theoretical_loss": 3.9992346134873484, + "tokens_seen": 438960128 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043786359077231695, + "loss": 1.5693, + "theoretical_loss": 3.999169327936052, + "tokens_seen": 439025664 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 151658, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5261473655700684, + "objective/train/theoretical_loss": 3.99910405485789, + "objective/train/tokens_used": 439091200, + "theoretical_loss": 3.99910405485789, + "tokens_seen": 439091200 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043785356068204613, + "loss": 1.4769, + "theoretical_loss": 3.99910405485789, + "tokens_seen": 439091200 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043784353059177537, + "loss": 1.501, + "theoretical_loss": 3.9990387942486167, + "tokens_seen": 439156736 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004378335005015045, + "loss": 1.4941, + "theoretical_loss": 3.9989735461039917, + "tokens_seen": 439222272 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043782347041123373, + "loss": 1.5941, + "theoretical_loss": 3.9989083104197736, + "tokens_seen": 439287808 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043781344032096286, + "loss": 1.5583, + "theoretical_loss": 3.9988430871917253, + "tokens_seen": 439353344 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004378034102306921, + "loss": 1.4798, + "theoretical_loss": 3.9987778764156108, + "tokens_seen": 439418880 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043779338014042127, + "loss": 1.5402, + "theoretical_loss": 3.9987126780871955, + "tokens_seen": 439484416 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043778335005015045, + "loss": 1.5114, + "theoretical_loss": 3.998647492202249, + "tokens_seen": 439549952 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043777331995987963, + "loss": 1.5151, + "theoretical_loss": 3.99858231875654, + "tokens_seen": 439615488 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004377632898696088, + "loss": 1.5625, + "theoretical_loss": 3.9985171577458427, + "tokens_seen": 439681024 + }, + { + "epoch": 0.13, + "learning_rate": 0.000437753259779338, + "loss": 1.4615, + "theoretical_loss": 3.99845200916593, + "tokens_seen": 439746560 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043774322968906723, + "loss": 1.5148, + "theoretical_loss": 3.9983868730125796, + "tokens_seen": 439812096 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043773319959879636, + "loss": 1.475, + "theoretical_loss": 3.998321749281569, + "tokens_seen": 439877632 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004377231695085256, + "loss": 1.5357, + "theoretical_loss": 3.99825663796868, + "tokens_seen": 439943168 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004377131394182547, + "loss": 1.5577, + "theoretical_loss": 3.998191539069694, + "tokens_seen": 440008704 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043770310932798396, + "loss": 1.5122, + "theoretical_loss": 3.9981264525803972, + "tokens_seen": 440074240 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043769307923771314, + "loss": 1.4997, + "theoretical_loss": 3.9980613784965753, + "tokens_seen": 440139776 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004376830491474423, + "loss": 1.5916, + "theoretical_loss": 3.9979963168140182, + "tokens_seen": 440205312 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043767301905717155, + "loss": 1.4743, + "theoretical_loss": 3.997931267528516, + "tokens_seen": 440270848 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043766298896690073, + "loss": 1.6152, + "theoretical_loss": 3.997866230635862, + "tokens_seen": 440336384 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004376529588766299, + "loss": 1.4683, + "theoretical_loss": 3.9978012061318515, + "tokens_seen": 440401920 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004376429287863591, + "loss": 1.6061, + "theoretical_loss": 3.997736194012281, + "tokens_seen": 440467456 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004376328986960883, + "loss": 1.5521, + "theoretical_loss": 3.99767119427295, + "tokens_seen": 440532992 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043762286860581746, + "loss": 1.5703, + "theoretical_loss": 3.9976062069096603, + "tokens_seen": 440598528 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004376128385155467, + "loss": 1.5007, + "theoretical_loss": 3.9975412319182144, + "tokens_seen": 440664064 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004376028084252758, + "loss": 1.677, + "theoretical_loss": 3.9974762692944172, + "tokens_seen": 440729600 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043759277833500506, + "loss": 1.6486, + "theoretical_loss": 3.997411319034077, + "tokens_seen": 440795136 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004375827482447342, + "loss": 1.5496, + "theoretical_loss": 3.9973463811330037, + "tokens_seen": 440860672 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004375727181544634, + "loss": 1.4797, + "theoretical_loss": 3.997281455587007, + "tokens_seen": 440926208 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004375626880641926, + "loss": 1.4853, + "theoretical_loss": 3.9972165423919015, + "tokens_seen": 440991744 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004375526579739218, + "loss": 1.4837, + "theoretical_loss": 3.9971516415435024, + "tokens_seen": 441057280 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043754262788365096, + "loss": 1.44, + "theoretical_loss": 3.9970867530376273, + "tokens_seen": 441122816 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004375325977933802, + "loss": 1.5854, + "theoretical_loss": 3.9970218768700967, + "tokens_seen": 441188352 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004375225677031093, + "loss": 1.588, + "theoretical_loss": 3.99695701303673, + "tokens_seen": 441253888 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043751253761283856, + "loss": 1.4202, + "theoretical_loss": 3.9968921615333532, + "tokens_seen": 441319424 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004375025075225677, + "loss": 1.5075, + "theoretical_loss": 3.996827322355791, + "tokens_seen": 441384960 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004374924774322969, + "loss": 1.5045, + "theoretical_loss": 3.9967624954998713, + "tokens_seen": 441450496 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004374824473420261, + "loss": 1.572, + "theoretical_loss": 3.9966976809614234, + "tokens_seen": 441516032 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004374724172517553, + "loss": 1.4962, + "theoretical_loss": 3.99663287873628, + "tokens_seen": 441581568 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043746238716148447, + "loss": 1.3804, + "theoretical_loss": 3.9965680888202737, + "tokens_seen": 441647104 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043745235707121365, + "loss": 1.5703, + "theoretical_loss": 3.996503311209242, + "tokens_seen": 441712640 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043744232698094283, + "loss": 1.424, + "theoretical_loss": 3.9964385458990206, + "tokens_seen": 441778176 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043743229689067206, + "loss": 1.4798, + "theoretical_loss": 3.9963737928854517, + "tokens_seen": 441843712 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004374222668004012, + "loss": 1.5333, + "theoretical_loss": 3.996309052164376, + "tokens_seen": 441909248 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004374122367101304, + "loss": 1.5276, + "theoretical_loss": 3.996244323731638, + "tokens_seen": 441974784 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043740220661985955, + "loss": 1.4225, + "theoretical_loss": 3.996179607583082, + "tokens_seen": 442040320 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004373921765295888, + "loss": 1.4715, + "theoretical_loss": 3.9961149037145587, + "tokens_seen": 442105856 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043738214643931797, + "loss": 1.4553, + "theoretical_loss": 3.996050212121916, + "tokens_seen": 442171392 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043737211634904715, + "loss": 1.5502, + "theoretical_loss": 3.995985532801007, + "tokens_seen": 442236928 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043736208625877633, + "loss": 1.604, + "theoretical_loss": 3.995920865747685, + "tokens_seen": 442302464 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 152908, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5396764278411865, + "objective/train/theoretical_loss": 3.9958562109578066, + "objective/train/tokens_used": 442368000, + "theoretical_loss": 3.9958562109578066, + "tokens_seen": 442368000 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043735205616850557, + "loss": 1.5494, + "theoretical_loss": 3.9958562109578066, + "tokens_seen": 442368000 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004373420260782347, + "loss": 1.5064, + "theoretical_loss": 3.9957915684272294, + "tokens_seen": 442433536 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043733199598796393, + "loss": 1.4944, + "theoretical_loss": 3.995726938151815, + "tokens_seen": 442499072 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043732196589769306, + "loss": 1.4492, + "theoretical_loss": 3.995662320127423, + "tokens_seen": 442564608 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004373119358074223, + "loss": 1.5357, + "theoretical_loss": 3.9955977143499197, + "tokens_seen": 442630144 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043730190571715147, + "loss": 1.5419, + "theoretical_loss": 3.99553312081517, + "tokens_seen": 442695680 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043729187562688065, + "loss": 1.3955, + "theoretical_loss": 3.995468539519042, + "tokens_seen": 442761216 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043728184553660983, + "loss": 1.5009, + "theoretical_loss": 3.9954039704574065, + "tokens_seen": 442826752 + }, + { + "epoch": 0.13, + "learning_rate": 0.000437271815446339, + "loss": 1.4868, + "theoretical_loss": 3.9953394136261355, + "tokens_seen": 442892288 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004372617853560682, + "loss": 1.4271, + "theoretical_loss": 3.995274869021103, + "tokens_seen": 442957824 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043725175526579743, + "loss": 1.5367, + "theoretical_loss": 3.995210336638185, + "tokens_seen": 443023360 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043724172517552656, + "loss": 1.5949, + "theoretical_loss": 3.9951458164732596, + "tokens_seen": 443088896 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004372316950852558, + "loss": 1.5157, + "theoretical_loss": 3.9950813085222068, + "tokens_seen": 443154432 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004372216649949849, + "loss": 1.6261, + "theoretical_loss": 3.99501681278091, + "tokens_seen": 443219968 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043721163490471416, + "loss": 1.5602, + "theoretical_loss": 3.994952329245251, + "tokens_seen": 443285504 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043720160481444334, + "loss": 1.5866, + "theoretical_loss": 3.994887857911118, + "tokens_seen": 443351040 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004371915747241725, + "loss": 1.4679, + "theoretical_loss": 3.9948233987743986, + "tokens_seen": 443416576 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004371815446339017, + "loss": 1.4515, + "theoretical_loss": 3.994758951830982, + "tokens_seen": 443482112 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043717151454363093, + "loss": 1.4926, + "theoretical_loss": 3.9946945170767614, + "tokens_seen": 443547648 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043716148445336006, + "loss": 1.5048, + "theoretical_loss": 3.994630094507631, + "tokens_seen": 443613184 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004371514543630893, + "loss": 1.5165, + "theoretical_loss": 3.994565684119486, + "tokens_seen": 443678720 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004371414242728184, + "loss": 1.5338, + "theoretical_loss": 3.9945012859082247, + "tokens_seen": 443744256 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043713139418254766, + "loss": 1.5522, + "theoretical_loss": 3.9944368998697475, + "tokens_seen": 443809792 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043712136409227684, + "loss": 1.5761, + "theoretical_loss": 3.9943725259999563, + "tokens_seen": 443875328 + }, + { + "epoch": 0.13, + "learning_rate": 0.000437111334002006, + "loss": 1.6567, + "theoretical_loss": 3.994308164294755, + "tokens_seen": 443940864 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004371013039117352, + "loss": 1.5735, + "theoretical_loss": 3.9942438147500496, + "tokens_seen": 444006400 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004370912738214644, + "loss": 1.5383, + "theoretical_loss": 3.9941794773617487, + "tokens_seen": 444071936 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043708124373119356, + "loss": 1.5417, + "theoretical_loss": 3.9941151521257616, + "tokens_seen": 444137472 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004370712136409228, + "loss": 1.5904, + "theoretical_loss": 3.9940508390380005, + "tokens_seen": 444203008 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004370611835506519, + "loss": 1.539, + "theoretical_loss": 3.9939865380943793, + "tokens_seen": 444268544 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043705115346038116, + "loss": 1.5348, + "theoretical_loss": 3.993922249290814, + "tokens_seen": 444334080 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004370411233701103, + "loss": 1.593, + "theoretical_loss": 3.993857972623222, + "tokens_seen": 444399616 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004370310932798395, + "loss": 1.4909, + "theoretical_loss": 3.993793708087524, + "tokens_seen": 444465152 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004370210631895687, + "loss": 1.5236, + "theoretical_loss": 3.9937294556796417, + "tokens_seen": 444530688 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004370110330992979, + "loss": 1.5301, + "theoretical_loss": 3.993665215395498, + "tokens_seen": 444596224 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043700100300902707, + "loss": 1.5747, + "theoretical_loss": 3.9936009872310194, + "tokens_seen": 444661760 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004369909729187563, + "loss": 1.5156, + "theoretical_loss": 3.993536771182134, + "tokens_seen": 444727296 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043698094282848543, + "loss": 1.547, + "theoretical_loss": 3.993472567244771, + "tokens_seen": 444792832 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043697091273821467, + "loss": 1.5613, + "theoretical_loss": 3.9934083754148615, + "tokens_seen": 444858368 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004369608826479438, + "loss": 1.3831, + "theoretical_loss": 3.9933441956883398, + "tokens_seen": 444923904 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043695085255767303, + "loss": 1.5419, + "theoretical_loss": 3.9932800280611422, + "tokens_seen": 444989440 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004369408224674022, + "loss": 1.5171, + "theoretical_loss": 3.9932158725292055, + "tokens_seen": 445054976 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004369307923771314, + "loss": 1.5421, + "theoretical_loss": 3.993151729088469, + "tokens_seen": 445120512 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004369207622868606, + "loss": 1.528, + "theoretical_loss": 3.993087597734875, + "tokens_seen": 445186048 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043691073219658975, + "loss": 1.5026, + "theoretical_loss": 3.993023478464366, + "tokens_seen": 445251584 + }, + { + "epoch": 0.13, + "learning_rate": 0.000436900702106319, + "loss": 1.4372, + "theoretical_loss": 3.992959371272888, + "tokens_seen": 445317120 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043689067201604817, + "loss": 1.6446, + "theoretical_loss": 3.992895276156388, + "tokens_seen": 445382656 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043688064192577735, + "loss": 1.4492, + "theoretical_loss": 3.992831193110816, + "tokens_seen": 445448192 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043687061183550653, + "loss": 1.5813, + "theoretical_loss": 3.992767122132123, + "tokens_seen": 445513728 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043686058174523577, + "loss": 1.4135, + "theoretical_loss": 3.9927030632162612, + "tokens_seen": 445579264 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 153920, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.515016794204712, + "objective/train/theoretical_loss": 3.9926390163591874, + "objective/train/tokens_used": 445644800, + "theoretical_loss": 3.9926390163591874, + "tokens_seen": 445644800 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004368505516549649, + "loss": 1.4949, + "theoretical_loss": 3.9926390163591874, + "tokens_seen": 445644800 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043684052156469413, + "loss": 1.4923, + "theoretical_loss": 3.9925749815568583, + "tokens_seen": 445710336 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043683049147442326, + "loss": 1.5543, + "theoretical_loss": 3.9925109588052328, + "tokens_seen": 445775872 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004368204613841525, + "loss": 1.415, + "theoretical_loss": 3.9924469481002713, + "tokens_seen": 445841408 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043681043129388167, + "loss": 1.4439, + "theoretical_loss": 3.9923829494379377, + "tokens_seen": 445906944 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043680040120361085, + "loss": 1.5829, + "theoretical_loss": 3.9923189628141964, + "tokens_seen": 445972480 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043679037111334003, + "loss": 1.5016, + "theoretical_loss": 3.9922549882250147, + "tokens_seen": 446038016 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004367803410230692, + "loss": 1.5116, + "theoretical_loss": 3.9921910256663606, + "tokens_seen": 446103552 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004367703109327984, + "loss": 1.4647, + "theoretical_loss": 3.9921270751342064, + "tokens_seen": 446169088 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043676028084252763, + "loss": 1.528, + "theoretical_loss": 3.992063136624523, + "tokens_seen": 446234624 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043675025075225676, + "loss": 1.4762, + "theoretical_loss": 3.9919992101332866, + "tokens_seen": 446300160 + }, + { + "epoch": 0.14, + "learning_rate": 0.000436740220661986, + "loss": 1.5103, + "theoretical_loss": 3.991935295656473, + "tokens_seen": 446365696 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004367301905717151, + "loss": 1.5288, + "theoretical_loss": 3.9918713931900602, + "tokens_seen": 446431232 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043672016048144436, + "loss": 1.4844, + "theoretical_loss": 3.9918075027300297, + "tokens_seen": 446496768 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043671013039117354, + "loss": 1.5438, + "theoretical_loss": 3.991743624272363, + "tokens_seen": 446562304 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004367001003009027, + "loss": 1.537, + "theoretical_loss": 3.9916797578130456, + "tokens_seen": 446627840 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004366900702106319, + "loss": 1.5602, + "theoretical_loss": 3.9916159033480625, + "tokens_seen": 446693376 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043668004012036113, + "loss": 1.4712, + "theoretical_loss": 3.9915520608734028, + "tokens_seen": 446758912 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043667001003009026, + "loss": 1.5888, + "theoretical_loss": 3.991488230385056, + "tokens_seen": 446824448 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004366599799398195, + "loss": 1.4807, + "theoretical_loss": 3.991424411879014, + "tokens_seen": 446889984 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004366499498495486, + "loss": 1.4478, + "theoretical_loss": 3.9913606053512716, + "tokens_seen": 446955520 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043663991975927786, + "loss": 1.4621, + "theoretical_loss": 3.9912968107978237, + "tokens_seen": 447021056 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043662988966900704, + "loss": 1.4467, + "theoretical_loss": 3.9912330282146695, + "tokens_seen": 447086592 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004366198595787362, + "loss": 1.502, + "theoretical_loss": 3.9911692575978077, + "tokens_seen": 447152128 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004366098294884654, + "loss": 1.5804, + "theoretical_loss": 3.9911054989432397, + "tokens_seen": 447217664 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004365997993981946, + "loss": 1.4613, + "theoretical_loss": 3.9910417522469706, + "tokens_seen": 447283200 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043658976930792376, + "loss": 1.6238, + "theoretical_loss": 3.990978017505004, + "tokens_seen": 447348736 + }, + { + "epoch": 0.14, + "learning_rate": 0.000436579739217653, + "loss": 1.5694, + "theoretical_loss": 3.990914294713348, + "tokens_seen": 447414272 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043656970912738213, + "loss": 1.5414, + "theoretical_loss": 3.9908505838680126, + "tokens_seen": 447479808 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043655967903711136, + "loss": 1.5132, + "theoretical_loss": 3.9907868849650088, + "tokens_seen": 447545344 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004365496489468405, + "loss": 1.5524, + "theoretical_loss": 3.9907231980003495, + "tokens_seen": 447610880 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004365396188565697, + "loss": 1.5232, + "theoretical_loss": 3.99065952297005, + "tokens_seen": 447676416 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004365295887662989, + "loss": 1.4452, + "theoretical_loss": 3.990595859870127, + "tokens_seen": 447741952 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004365195586760281, + "loss": 1.5266, + "theoretical_loss": 3.9905322086966004, + "tokens_seen": 447807488 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043650952858575727, + "loss": 1.5988, + "theoretical_loss": 3.990468569445489, + "tokens_seen": 447873024 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004364994984954865, + "loss": 1.5187, + "theoretical_loss": 3.990404942112818, + "tokens_seen": 447938560 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043648946840521563, + "loss": 1.4949, + "theoretical_loss": 3.9903413266946104, + "tokens_seen": 448004096 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043647943831494487, + "loss": 1.5198, + "theoretical_loss": 3.9902777231868933, + "tokens_seen": 448069632 + }, + { + "epoch": 0.14, + "learning_rate": 0.000436469408224674, + "loss": 1.3968, + "theoretical_loss": 3.990214131585695, + "tokens_seen": 448135168 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043645937813440323, + "loss": 1.5089, + "theoretical_loss": 3.990150551887047, + "tokens_seen": 448200704 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004364493480441324, + "loss": 1.5188, + "theoretical_loss": 3.9900869840869793, + "tokens_seen": 448266240 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004364393179538616, + "loss": 1.3945, + "theoretical_loss": 3.9900234281815283, + "tokens_seen": 448331776 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043642928786359077, + "loss": 1.35, + "theoretical_loss": 3.989959884166728, + "tokens_seen": 448397312 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043641925777331995, + "loss": 1.4642, + "theoretical_loss": 3.9898963520386186, + "tokens_seen": 448462848 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043640922768304913, + "loss": 1.4012, + "theoretical_loss": 3.9898328317932386, + "tokens_seen": 448528384 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043639919759277837, + "loss": 1.5894, + "theoretical_loss": 3.98976932342663, + "tokens_seen": 448593920 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004363891675025075, + "loss": 1.3782, + "theoretical_loss": 3.9897058269348364, + "tokens_seen": 448659456 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043637913741223673, + "loss": 1.4088, + "theoretical_loss": 3.9896423423139034, + "tokens_seen": 448724992 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004363691073219659, + "loss": 1.474, + "theoretical_loss": 3.989578869559879, + "tokens_seen": 448790528 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004363590772316951, + "loss": 1.5504, + "theoretical_loss": 3.989515408668812, + "tokens_seen": 448856064 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 155023, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5598907470703125, + "objective/train/theoretical_loss": 3.989451959636754, + "objective/train/tokens_used": 448921600, + "theoretical_loss": 3.989451959636754, + "tokens_seen": 448921600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004363490471414243, + "loss": 1.5283, + "theoretical_loss": 3.989451959636754, + "tokens_seen": 448921600 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043633901705115346, + "loss": 1.3872, + "theoretical_loss": 3.989388522459757, + "tokens_seen": 448987136 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043632898696088264, + "loss": 1.5916, + "theoretical_loss": 3.989325097133878, + "tokens_seen": 449052672 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043631895687061187, + "loss": 1.4476, + "theoretical_loss": 3.989261683655172, + "tokens_seen": 449118208 + }, + { + "epoch": 0.14, + "learning_rate": 0.000436308926780341, + "loss": 1.4984, + "theoretical_loss": 3.989198282019699, + "tokens_seen": 449183744 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043629889669007023, + "loss": 1.5479, + "theoretical_loss": 3.989134892223519, + "tokens_seen": 449249280 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043628886659979936, + "loss": 1.5462, + "theoretical_loss": 3.9890715142626956, + "tokens_seen": 449314816 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004362788365095286, + "loss": 1.5539, + "theoretical_loss": 3.9890081481332924, + "tokens_seen": 449380352 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004362688064192578, + "loss": 1.3552, + "theoretical_loss": 3.988944793831376, + "tokens_seen": 449445888 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043625877632898696, + "loss": 1.5939, + "theoretical_loss": 3.9888814513530138, + "tokens_seen": 449511424 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043624874623871614, + "loss": 1.4095, + "theoretical_loss": 3.988818120694277, + "tokens_seen": 449576960 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004362387161484453, + "loss": 1.5046, + "theoretical_loss": 3.988754801851237, + "tokens_seen": 449642496 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004362286860581745, + "loss": 1.5107, + "theoretical_loss": 3.9886914948199683, + "tokens_seen": 449708032 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043621865596790374, + "loss": 1.4718, + "theoretical_loss": 3.988628199596546, + "tokens_seen": 449773568 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043620862587763286, + "loss": 1.5622, + "theoretical_loss": 3.9885649161770473, + "tokens_seen": 449839104 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004361985957873621, + "loss": 1.5511, + "theoretical_loss": 3.988501644557553, + "tokens_seen": 449904640 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004361885656970913, + "loss": 1.4971, + "theoretical_loss": 3.9884383847341436, + "tokens_seen": 449970176 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043617853560682046, + "loss": 1.5458, + "theoretical_loss": 3.988375136702902, + "tokens_seen": 450035712 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004361685055165497, + "loss": 1.4836, + "theoretical_loss": 3.988311900459914, + "tokens_seen": 450101248 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004361584754262788, + "loss": 1.4254, + "theoretical_loss": 3.988248676001267, + "tokens_seen": 450166784 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043614844533600806, + "loss": 1.3941, + "theoretical_loss": 3.9881854633230485, + "tokens_seen": 450232320 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043613841524573724, + "loss": 1.4198, + "theoretical_loss": 3.9881222624213497, + "tokens_seen": 450297856 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004361283851554664, + "loss": 1.5194, + "theoretical_loss": 3.9880590732922636, + "tokens_seen": 450363392 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004361183550651956, + "loss": 1.5076, + "theoretical_loss": 3.9879958959318844, + "tokens_seen": 450428928 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004361083249749248, + "loss": 1.4239, + "theoretical_loss": 3.9879327303363086, + "tokens_seen": 450494464 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043609829488465397, + "loss": 1.4064, + "theoretical_loss": 3.9878695765016334, + "tokens_seen": 450560000 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004360882647943832, + "loss": 1.4977, + "theoretical_loss": 3.98780643442396, + "tokens_seen": 450625536 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043607823470411233, + "loss": 1.389, + "theoretical_loss": 3.9877433040993897, + "tokens_seen": 450691072 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043606820461384156, + "loss": 1.5461, + "theoretical_loss": 3.9876801855240265, + "tokens_seen": 450756608 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004360581745235707, + "loss": 1.4788, + "theoretical_loss": 3.987617078693975, + "tokens_seen": 450822144 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004360481444332999, + "loss": 1.4608, + "theoretical_loss": 3.987553983605345, + "tokens_seen": 450887680 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004360381143430291, + "loss": 1.4209, + "theoretical_loss": 3.9874909002542434, + "tokens_seen": 450953216 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004360280842527583, + "loss": 1.5585, + "theoretical_loss": 3.987427828636782, + "tokens_seen": 451018752 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043601805416248747, + "loss": 1.5277, + "theoretical_loss": 3.9873647687490745, + "tokens_seen": 451084288 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004360080240722167, + "loss": 1.3353, + "theoretical_loss": 3.9873017205872356, + "tokens_seen": 451149824 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043599799398194583, + "loss": 1.5659, + "theoretical_loss": 3.9872386841473815, + "tokens_seen": 451215360 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043598796389167507, + "loss": 1.4277, + "theoretical_loss": 3.987175659425631, + "tokens_seen": 451280896 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004359779338014042, + "loss": 1.3933, + "theoretical_loss": 3.9871126464181046, + "tokens_seen": 451346432 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043596790371113343, + "loss": 1.4428, + "theoretical_loss": 3.9870496451209245, + "tokens_seen": 451411968 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004359578736208626, + "loss": 1.6216, + "theoretical_loss": 3.986986655530215, + "tokens_seen": 451477504 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004359478435305918, + "loss": 1.4383, + "theoretical_loss": 3.9869236776421015, + "tokens_seen": 451543040 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043593781344032097, + "loss": 1.4915, + "theoretical_loss": 3.986860711452713, + "tokens_seen": 451608576 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043592778335005015, + "loss": 1.5093, + "theoretical_loss": 3.986797756958178, + "tokens_seen": 451674112 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043591775325977933, + "loss": 1.4998, + "theoretical_loss": 3.9867348141546284, + "tokens_seen": 451739648 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043590772316950857, + "loss": 1.4998, + "theoretical_loss": 3.9866718830381975, + "tokens_seen": 451805184 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004358976930792377, + "loss": 1.3374, + "theoretical_loss": 3.98660896360502, + "tokens_seen": 451870720 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043588766298896693, + "loss": 1.4436, + "theoretical_loss": 3.9865460558512336, + "tokens_seen": 451936256 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004358776328986961, + "loss": 1.515, + "theoretical_loss": 3.986483159772977, + "tokens_seen": 452001792 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004358676028084253, + "loss": 1.434, + "theoretical_loss": 3.986420275366391, + "tokens_seen": 452067328 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004358575727181545, + "loss": 1.5124, + "theoretical_loss": 3.9863574026276174, + "tokens_seen": 452132864 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 156163, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4855676889419556, + "objective/train/theoretical_loss": 3.986294541552801, + "objective/train/tokens_used": 452198400, + "theoretical_loss": 3.986294541552801, + "tokens_seen": 452198400 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043584754262788366, + "loss": 1.4845, + "theoretical_loss": 3.986294541552801, + "tokens_seen": 452198400 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043583751253761284, + "loss": 1.4473, + "theoretical_loss": 3.9862316921380883, + "tokens_seen": 452263936 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043582748244734207, + "loss": 1.4501, + "theoretical_loss": 3.9861688543796268, + "tokens_seen": 452329472 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004358174523570712, + "loss": 1.4653, + "theoretical_loss": 3.986106028273567, + "tokens_seen": 452395008 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043580742226680043, + "loss": 1.4963, + "theoretical_loss": 3.9860432138160595, + "tokens_seen": 452460544 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043579739217652956, + "loss": 1.3541, + "theoretical_loss": 3.985980411003259, + "tokens_seen": 452526080 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004357873620862588, + "loss": 1.6449, + "theoretical_loss": 3.98591761983132, + "tokens_seen": 452591616 + }, + { + "epoch": 0.14, + "learning_rate": 0.000435777331995988, + "loss": 1.4496, + "theoretical_loss": 3.9858548402964, + "tokens_seen": 452657152 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043576730190571716, + "loss": 1.5354, + "theoretical_loss": 3.985792072394658, + "tokens_seen": 452722688 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043575727181544634, + "loss": 1.4486, + "theoretical_loss": 3.9857293161222542, + "tokens_seen": 452788224 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004357472417251755, + "loss": 1.5614, + "theoretical_loss": 3.985666571475352, + "tokens_seen": 452853760 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004357372116349047, + "loss": 1.5521, + "theoretical_loss": 3.985603838450116, + "tokens_seen": 452919296 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043572718154463394, + "loss": 1.4871, + "theoretical_loss": 3.9855411170427115, + "tokens_seen": 452984832 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043571715145436306, + "loss": 1.5123, + "theoretical_loss": 3.985478407249307, + "tokens_seen": 453050368 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004357071213640923, + "loss": 1.5173, + "theoretical_loss": 3.985415709066073, + "tokens_seen": 453115904 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004356970912738215, + "loss": 1.4946, + "theoretical_loss": 3.98535302248918, + "tokens_seen": 453181440 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043568706118355066, + "loss": 1.4624, + "theoretical_loss": 3.9852903475148027, + "tokens_seen": 453246976 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043567703109327984, + "loss": 1.4936, + "theoretical_loss": 3.985227684139116, + "tokens_seen": 453312512 + }, + { + "epoch": 0.14, + "learning_rate": 0.000435667001003009, + "loss": 1.509, + "theoretical_loss": 3.9851650323582977, + "tokens_seen": 453378048 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004356569709127382, + "loss": 1.4664, + "theoretical_loss": 3.9851023921685256, + "tokens_seen": 453443584 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043564694082246744, + "loss": 1.4542, + "theoretical_loss": 3.9850397635659807, + "tokens_seen": 453509120 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043563691073219657, + "loss": 1.5628, + "theoretical_loss": 3.9849771465468464, + "tokens_seen": 453574656 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004356268806419258, + "loss": 1.5541, + "theoretical_loss": 3.9849145411073064, + "tokens_seen": 453640192 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043561685055165493, + "loss": 1.5874, + "theoretical_loss": 3.984851947243547, + "tokens_seen": 453705728 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043560682046138417, + "loss": 1.4005, + "theoretical_loss": 3.984789364951757, + "tokens_seen": 453771264 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043559679037111335, + "loss": 1.4723, + "theoretical_loss": 3.9847267942281253, + "tokens_seen": 453836800 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043558676028084253, + "loss": 1.4924, + "theoretical_loss": 3.984664235068843, + "tokens_seen": 453902336 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004355767301905717, + "loss": 1.4355, + "theoretical_loss": 3.9846016874701053, + "tokens_seen": 453967872 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004355667001003009, + "loss": 1.4641, + "theoretical_loss": 3.9845391514281063, + "tokens_seen": 454033408 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043555667001003007, + "loss": 1.5101, + "theoretical_loss": 3.9844766269390437, + "tokens_seen": 454098944 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004355466399197593, + "loss": 1.5915, + "theoretical_loss": 3.984414113999115, + "tokens_seen": 454164480 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043553660982948843, + "loss": 1.4445, + "theoretical_loss": 3.984351612604522, + "tokens_seen": 454230016 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043552657973921767, + "loss": 1.5059, + "theoretical_loss": 3.984289122751467, + "tokens_seen": 454295552 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043551654964894685, + "loss": 1.4248, + "theoretical_loss": 3.984226644436154, + "tokens_seen": 454361088 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043550651955867603, + "loss": 1.3565, + "theoretical_loss": 3.9841641776547885, + "tokens_seen": 454426624 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004354964894684052, + "loss": 1.445, + "theoretical_loss": 3.98410172240358, + "tokens_seen": 454492160 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004354864593781344, + "loss": 1.6109, + "theoretical_loss": 3.984039278678736, + "tokens_seen": 454557696 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004354764292878636, + "loss": 1.5927, + "theoretical_loss": 3.9839768464764695, + "tokens_seen": 454623232 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004354663991975928, + "loss": 1.4505, + "theoretical_loss": 3.9839144257929933, + "tokens_seen": 454688768 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043545636910732194, + "loss": 1.5464, + "theoretical_loss": 3.983852016624522, + "tokens_seen": 454754304 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043544633901705117, + "loss": 1.3793, + "theoretical_loss": 3.9837896189672724, + "tokens_seen": 454819840 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004354363089267803, + "loss": 1.485, + "theoretical_loss": 3.9837272328174635, + "tokens_seen": 454885376 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043542627883650953, + "loss": 1.3087, + "theoretical_loss": 3.9836648581713154, + "tokens_seen": 454950912 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043541624874623877, + "loss": 1.5104, + "theoretical_loss": 3.9836024950250497, + "tokens_seen": 455016448 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004354062186559679, + "loss": 1.4646, + "theoretical_loss": 3.983540143374891, + "tokens_seen": 455081984 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043539618856569713, + "loss": 1.4564, + "theoretical_loss": 3.983477803217066, + "tokens_seen": 455147520 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004353861584754263, + "loss": 1.4644, + "theoretical_loss": 3.9834154745478, + "tokens_seen": 455213056 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004353761283851555, + "loss": 1.5506, + "theoretical_loss": 3.9833531573633234, + "tokens_seen": 455278592 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004353660982948847, + "loss": 1.5715, + "theoretical_loss": 3.9832908516598673, + "tokens_seen": 455344128 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043535606820461386, + "loss": 1.5342, + "theoretical_loss": 3.983228557433664, + "tokens_seen": 455409664 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 157051, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3251571655273438, + "objective/train/theoretical_loss": 3.9831662746809493, + "objective/train/tokens_used": 455475200, + "theoretical_loss": 3.9831662746809493, + "tokens_seen": 455475200 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043534603811434304, + "loss": 1.4252, + "theoretical_loss": 3.9831662746809493, + "tokens_seen": 455475200 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043533600802407227, + "loss": 1.5453, + "theoretical_loss": 3.9831040033979583, + "tokens_seen": 455540736 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004353259779338014, + "loss": 1.549, + "theoretical_loss": 3.9830417435809298, + "tokens_seen": 455606272 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043531594784353063, + "loss": 1.5145, + "theoretical_loss": 3.9829794952261035, + "tokens_seen": 455671808 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043530591775325976, + "loss": 1.5806, + "theoretical_loss": 3.9829172583297208, + "tokens_seen": 455737344 + }, + { + "epoch": 0.14, + "learning_rate": 0.000435295887662989, + "loss": 1.4258, + "theoretical_loss": 3.9828550328880263, + "tokens_seen": 455802880 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004352858575727182, + "loss": 1.6137, + "theoretical_loss": 3.982792818897264, + "tokens_seen": 455868416 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043527582748244736, + "loss": 1.5555, + "theoretical_loss": 3.9827306163536815, + "tokens_seen": 455933952 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043526579739217654, + "loss": 1.4881, + "theoretical_loss": 3.982668425253528, + "tokens_seen": 455999488 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004352557673019057, + "loss": 1.4732, + "theoretical_loss": 3.982606245593053, + "tokens_seen": 456065024 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004352457372116349, + "loss": 1.5076, + "theoretical_loss": 3.9825440773685092, + "tokens_seen": 456130560 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043523570712136414, + "loss": 1.6014, + "theoretical_loss": 3.9824819205761512, + "tokens_seen": 456196096 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043522567703109326, + "loss": 1.4899, + "theoretical_loss": 3.982419775212235, + "tokens_seen": 456261632 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004352156469408225, + "loss": 1.4539, + "theoretical_loss": 3.9823576412730173, + "tokens_seen": 456327168 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004352056168505517, + "loss": 1.5679, + "theoretical_loss": 3.982295518754758, + "tokens_seen": 456392704 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043519558676028086, + "loss": 1.612, + "theoretical_loss": 3.982233407653718, + "tokens_seen": 456458240 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043518555667001004, + "loss": 1.4486, + "theoretical_loss": 3.982171307966161, + "tokens_seen": 456523776 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004351755265797392, + "loss": 1.4742, + "theoretical_loss": 3.9821092196883505, + "tokens_seen": 456589312 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004351654964894684, + "loss": 1.5067, + "theoretical_loss": 3.982047142816554, + "tokens_seen": 456654848 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043515546639919764, + "loss": 1.5232, + "theoretical_loss": 3.981985077347039, + "tokens_seen": 456720384 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043514543630892677, + "loss": 1.5804, + "theoretical_loss": 3.981923023276075, + "tokens_seen": 456785920 + }, + { + "epoch": 0.14, + "learning_rate": 0.000435135406218656, + "loss": 1.5388, + "theoretical_loss": 3.9818609805999348, + "tokens_seen": 456851456 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043512537612838513, + "loss": 1.6126, + "theoretical_loss": 3.981798949314891, + "tokens_seen": 456916992 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043511534603811437, + "loss": 1.473, + "theoretical_loss": 3.98173692941722, + "tokens_seen": 456982528 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043510531594784355, + "loss": 1.4317, + "theoretical_loss": 3.981674920903197, + "tokens_seen": 457048064 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043509528585757273, + "loss": 1.405, + "theoretical_loss": 3.9816129237691023, + "tokens_seen": 457113600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004350852557673019, + "loss": 1.5388, + "theoretical_loss": 3.981550938011215, + "tokens_seen": 457179136 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004350752256770311, + "loss": 1.5367, + "theoretical_loss": 3.981488963625818, + "tokens_seen": 457244672 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043506519558676027, + "loss": 1.4274, + "theoretical_loss": 3.9814270006091954, + "tokens_seen": 457310208 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004350551654964895, + "loss": 1.4784, + "theoretical_loss": 3.981365048957633, + "tokens_seen": 457375744 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043504513540621863, + "loss": 1.5368, + "theoretical_loss": 3.9813031086674173, + "tokens_seen": 457441280 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043503510531594787, + "loss": 1.431, + "theoretical_loss": 3.9812411797348384, + "tokens_seen": 457506816 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043502507522567705, + "loss": 1.4029, + "theoretical_loss": 3.9811792621561874, + "tokens_seen": 457572352 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043501504513540623, + "loss": 1.6303, + "theoretical_loss": 3.981117355927756, + "tokens_seen": 457637888 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004350050150451354, + "loss": 1.3961, + "theoretical_loss": 3.981055461045839, + "tokens_seen": 457703424 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004349949849548646, + "loss": 1.4291, + "theoretical_loss": 3.980993577506733, + "tokens_seen": 457768960 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004349849548645938, + "loss": 1.5943, + "theoretical_loss": 3.980931705306735, + "tokens_seen": 457834496 + }, + { + "epoch": 0.14, + "learning_rate": 0.000434974924774323, + "loss": 1.5803, + "theoretical_loss": 3.9808698444421458, + "tokens_seen": 457900032 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043496489468405214, + "loss": 1.4753, + "theoretical_loss": 3.9808079949092656, + "tokens_seen": 457965568 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043495486459378137, + "loss": 1.5386, + "theoretical_loss": 3.9807461567043987, + "tokens_seen": 458031104 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004349448345035105, + "loss": 1.449, + "theoretical_loss": 3.980684329823849, + "tokens_seen": 458096640 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043493480441323973, + "loss": 1.5056, + "theoretical_loss": 3.980622514263923, + "tokens_seen": 458162176 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004349247743229689, + "loss": 1.4919, + "theoretical_loss": 3.9805607100209297, + "tokens_seen": 458227712 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004349147442326981, + "loss": 1.4047, + "theoretical_loss": 3.980498917091179, + "tokens_seen": 458293248 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004349047141424273, + "loss": 1.3894, + "theoretical_loss": 3.980437135470982, + "tokens_seen": 458358784 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004348946840521565, + "loss": 1.4143, + "theoretical_loss": 3.980375365156653, + "tokens_seen": 458424320 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043488465396188564, + "loss": 1.4929, + "theoretical_loss": 3.9803136061445064, + "tokens_seen": 458489856 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004348746238716149, + "loss": 1.4592, + "theoretical_loss": 3.9802518584308597, + "tokens_seen": 458555392 + }, + { + "epoch": 0.14, + "learning_rate": 0.000434864593781344, + "loss": 1.4465, + "theoretical_loss": 3.980190122012032, + "tokens_seen": 458620928 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043485456369107324, + "loss": 1.441, + "theoretical_loss": 3.980128396884343, + "tokens_seen": 458686464 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 158216, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5084296464920044, + "objective/train/theoretical_loss": 3.9800666830441145, + "objective/train/tokens_used": 458752000, + "theoretical_loss": 3.9800666830441145, + "tokens_seen": 458752000 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004348445336008024, + "loss": 1.5298, + "theoretical_loss": 3.9800666830441145, + "tokens_seen": 458752000 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004348345035105316, + "loss": 1.563, + "theoretical_loss": 3.9800049804876716, + "tokens_seen": 458817536 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004348244734202608, + "loss": 1.4831, + "theoretical_loss": 3.9799432892113384, + "tokens_seen": 458883072 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043481444332998996, + "loss": 1.4584, + "theoretical_loss": 3.9798816092114437, + "tokens_seen": 458948608 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043480441323971914, + "loss": 1.4173, + "theoretical_loss": 3.9798199404843153, + "tokens_seen": 459014144 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004347943831494484, + "loss": 1.4455, + "theoretical_loss": 3.9797582830262845, + "tokens_seen": 459079680 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004347843530591775, + "loss": 1.514, + "theoretical_loss": 3.9796966368336832, + "tokens_seen": 459145216 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043477432296890674, + "loss": 1.464, + "theoretical_loss": 3.979635001902847, + "tokens_seen": 459210752 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043476429287863587, + "loss": 1.5303, + "theoretical_loss": 3.9795733782301097, + "tokens_seen": 459276288 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004347542627883651, + "loss": 1.4733, + "theoretical_loss": 3.979511765811811, + "tokens_seen": 459341824 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004347442326980943, + "loss": 1.5012, + "theoretical_loss": 3.9794501646442884, + "tokens_seen": 459407360 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043473420260782346, + "loss": 1.5925, + "theoretical_loss": 3.9793885747238837, + "tokens_seen": 459472896 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043472417251755265, + "loss": 1.5362, + "theoretical_loss": 3.97932699604694, + "tokens_seen": 459538432 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004347141424272819, + "loss": 1.5328, + "theoretical_loss": 3.979265428609801, + "tokens_seen": 459603968 + }, + { + "epoch": 0.14, + "learning_rate": 0.000434704112337011, + "loss": 1.5314, + "theoretical_loss": 3.9792038724088137, + "tokens_seen": 459669504 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043469408224674024, + "loss": 1.464, + "theoretical_loss": 3.9791423274403255, + "tokens_seen": 459735040 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004346840521564694, + "loss": 1.5826, + "theoretical_loss": 3.9790807937006853, + "tokens_seen": 459800576 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004346740220661986, + "loss": 1.5545, + "theoretical_loss": 3.979019271186246, + "tokens_seen": 459866112 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043466399197592784, + "loss": 1.4954, + "theoretical_loss": 3.9789577598933588, + "tokens_seen": 459931648 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043465396188565697, + "loss": 1.5023, + "theoretical_loss": 3.9788962598183795, + "tokens_seen": 459997184 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004346439317953862, + "loss": 1.407, + "theoretical_loss": 3.978834770957665, + "tokens_seen": 460062720 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043463390170511533, + "loss": 1.4287, + "theoretical_loss": 3.9787732933075715, + "tokens_seen": 460128256 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043462387161484457, + "loss": 1.5268, + "theoretical_loss": 3.97871182686446, + "tokens_seen": 460193792 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043461384152457375, + "loss": 1.4648, + "theoretical_loss": 3.978650371624692, + "tokens_seen": 460259328 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043460381143430293, + "loss": 1.575, + "theoretical_loss": 3.9785889275846307, + "tokens_seen": 460324864 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004345937813440321, + "loss": 1.4655, + "theoretical_loss": 3.978527494740641, + "tokens_seen": 460390400 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004345837512537613, + "loss": 1.5907, + "theoretical_loss": 3.9784660730890895, + "tokens_seen": 460455936 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043457372116349047, + "loss": 1.4682, + "theoretical_loss": 3.978404662626344, + "tokens_seen": 460521472 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004345636910732197, + "loss": 1.4654, + "theoretical_loss": 3.9783432633487745, + "tokens_seen": 460587008 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043455366098294883, + "loss": 1.5192, + "theoretical_loss": 3.9782818752527533, + "tokens_seen": 460652544 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043454363089267807, + "loss": 1.5567, + "theoretical_loss": 3.978220498334654, + "tokens_seen": 460718080 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043453360080240725, + "loss": 1.5145, + "theoretical_loss": 3.97815913259085, + "tokens_seen": 460783616 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043452357071213643, + "loss": 1.5909, + "theoretical_loss": 3.9780977780177205, + "tokens_seen": 460849152 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004345135406218656, + "loss": 1.4263, + "theoretical_loss": 3.978036434611642, + "tokens_seen": 460914688 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004345035105315948, + "loss": 1.4483, + "theoretical_loss": 3.977975102368995, + "tokens_seen": 460980224 + }, + { + "epoch": 0.14, + "learning_rate": 0.000434493480441324, + "loss": 1.3973, + "theoretical_loss": 3.977913781286162, + "tokens_seen": 461045760 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004344834503510532, + "loss": 1.4686, + "theoretical_loss": 3.9778524713595256, + "tokens_seen": 461111296 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043447342026078234, + "loss": 1.4505, + "theoretical_loss": 3.977791172585472, + "tokens_seen": 461176832 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043446339017051157, + "loss": 1.6044, + "theoretical_loss": 3.977729884960387, + "tokens_seen": 461242368 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004344533600802407, + "loss": 1.4872, + "theoretical_loss": 3.9776686084806596, + "tokens_seen": 461307904 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043444332998996993, + "loss": 1.4871, + "theoretical_loss": 3.9776073431426804, + "tokens_seen": 461373440 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004344332998996991, + "loss": 1.4604, + "theoretical_loss": 3.977546088942841, + "tokens_seen": 461438976 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004344232698094283, + "loss": 1.4658, + "theoretical_loss": 3.9774848458775347, + "tokens_seen": 461504512 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004344132397191575, + "loss": 1.5661, + "theoretical_loss": 3.9774236139431576, + "tokens_seen": 461570048 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004344032096288867, + "loss": 1.4014, + "theoretical_loss": 3.977362393136106, + "tokens_seen": 461635584 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043439317953861584, + "loss": 1.5511, + "theoretical_loss": 3.977301183452779, + "tokens_seen": 461701120 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004343831494483451, + "loss": 1.4783, + "theoretical_loss": 3.977239984889576, + "tokens_seen": 461766656 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004343731193580742, + "loss": 1.4919, + "theoretical_loss": 3.9771787974429, + "tokens_seen": 461832192 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043436308926780344, + "loss": 1.4457, + "theoretical_loss": 3.977117621109154, + "tokens_seen": 461897728 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004343530591775326, + "loss": 1.4954, + "theoretical_loss": 3.977056455884744, + "tokens_seen": 461963264 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 159118, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5972529649734497, + "objective/train/theoretical_loss": 3.9769953017660766, + "objective/train/tokens_used": 462028800, + "theoretical_loss": 3.9769953017660766, + "tokens_seen": 462028800 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004343430290872618, + "loss": 1.6019, + "theoretical_loss": 3.9769953017660766, + "tokens_seen": 462028800 + }, + { + "epoch": 0.14, + "learning_rate": 0.000434332998996991, + "loss": 1.5055, + "theoretical_loss": 3.97693415874956, + "tokens_seen": 462094336 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043432296890672016, + "loss": 1.4549, + "theoretical_loss": 3.9768730268316057, + "tokens_seen": 462159872 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043431293881644934, + "loss": 1.3694, + "theoretical_loss": 3.976811906008625, + "tokens_seen": 462225408 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004343029087261786, + "loss": 1.4897, + "theoretical_loss": 3.976750796277031, + "tokens_seen": 462290944 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004342928786359077, + "loss": 1.5361, + "theoretical_loss": 3.9766896976332404, + "tokens_seen": 462356480 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043428284854563694, + "loss": 1.4827, + "theoretical_loss": 3.9766286100736696, + "tokens_seen": 462422016 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043427281845536607, + "loss": 1.4929, + "theoretical_loss": 3.976567533594737, + "tokens_seen": 462487552 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004342627883650953, + "loss": 1.4378, + "theoretical_loss": 3.9765064681928637, + "tokens_seen": 462553088 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004342527582748245, + "loss": 1.3578, + "theoretical_loss": 3.9764454138644703, + "tokens_seen": 462618624 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043424272818455367, + "loss": 1.5441, + "theoretical_loss": 3.976384370605982, + "tokens_seen": 462684160 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043423269809428285, + "loss": 1.6165, + "theoretical_loss": 3.9763233384138235, + "tokens_seen": 462749696 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004342226680040121, + "loss": 1.4394, + "theoretical_loss": 3.976262317284421, + "tokens_seen": 462815232 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004342126379137412, + "loss": 1.4947, + "theoretical_loss": 3.976201307214205, + "tokens_seen": 462880768 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043420260782347044, + "loss": 1.5719, + "theoretical_loss": 3.9761403081996045, + "tokens_seen": 462946304 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043419257773319957, + "loss": 1.4511, + "theoretical_loss": 3.9760793202370524, + "tokens_seen": 463011840 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004341825476429288, + "loss": 1.5794, + "theoretical_loss": 3.976018343322981, + "tokens_seen": 463077376 + }, + { + "epoch": 0.14, + "learning_rate": 0.000434172517552658, + "loss": 1.6406, + "theoretical_loss": 3.975957377453826, + "tokens_seen": 463142912 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043416248746238717, + "loss": 1.4413, + "theoretical_loss": 3.9758964226260254, + "tokens_seen": 463208448 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043415245737211635, + "loss": 1.4395, + "theoretical_loss": 3.975835478836016, + "tokens_seen": 463273984 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043414242728184553, + "loss": 1.5072, + "theoretical_loss": 3.97577454608024, + "tokens_seen": 463339520 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004341323971915747, + "loss": 1.5852, + "theoretical_loss": 3.975713624355139, + "tokens_seen": 463405056 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043412236710130395, + "loss": 1.5448, + "theoretical_loss": 3.975652713657155, + "tokens_seen": 463470592 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004341123370110331, + "loss": 1.5934, + "theoretical_loss": 3.975591813982734, + "tokens_seen": 463536128 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004341023069207623, + "loss": 1.5604, + "theoretical_loss": 3.9755309253283233, + "tokens_seen": 463601664 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043409227683049144, + "loss": 1.5012, + "theoretical_loss": 3.9754700476903713, + "tokens_seen": 463667200 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043408224674022067, + "loss": 1.5358, + "theoretical_loss": 3.975409181065328, + "tokens_seen": 463732736 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043407221664994985, + "loss": 1.5369, + "theoretical_loss": 3.9753483254496444, + "tokens_seen": 463798272 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043406218655967903, + "loss": 1.5046, + "theoretical_loss": 3.9752874808397753, + "tokens_seen": 463863808 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004340521564694082, + "loss": 1.3916, + "theoretical_loss": 3.975226647232175, + "tokens_seen": 463929344 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043404212637913745, + "loss": 1.5931, + "theoretical_loss": 3.9751658246233, + "tokens_seen": 463994880 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004340320962888666, + "loss": 1.4596, + "theoretical_loss": 3.9751050130096095, + "tokens_seen": 464060416 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004340220661985958, + "loss": 1.4663, + "theoretical_loss": 3.9750442123875622, + "tokens_seen": 464125952 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043401203610832494, + "loss": 1.515, + "theoretical_loss": 3.9749834227536214, + "tokens_seen": 464191488 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004340020060180542, + "loss": 1.4279, + "theoretical_loss": 3.9749226441042493, + "tokens_seen": 464257024 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043399197592778336, + "loss": 1.5285, + "theoretical_loss": 3.9748618764359107, + "tokens_seen": 464322560 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043398194583751254, + "loss": 1.4888, + "theoretical_loss": 3.9748011197450728, + "tokens_seen": 464388096 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004339719157472417, + "loss": 1.4675, + "theoretical_loss": 3.974740374028203, + "tokens_seen": 464453632 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004339618856569709, + "loss": 1.4452, + "theoretical_loss": 3.9746796392817716, + "tokens_seen": 464519168 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004339518555667001, + "loss": 1.6557, + "theoretical_loss": 3.9746189155022504, + "tokens_seen": 464584704 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004339418254764293, + "loss": 1.4944, + "theoretical_loss": 3.9745582026861124, + "tokens_seen": 464650240 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004339317953861585, + "loss": 1.4789, + "theoretical_loss": 3.9744975008298313, + "tokens_seen": 464715776 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004339217652958877, + "loss": 1.4651, + "theoretical_loss": 3.974436809929885, + "tokens_seen": 464781312 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004339117352056169, + "loss": 1.4599, + "theoretical_loss": 3.9743761299827494, + "tokens_seen": 464846848 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043390170511534604, + "loss": 1.5209, + "theoretical_loss": 3.974315460984906, + "tokens_seen": 464912384 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004338916750250753, + "loss": 1.4839, + "theoretical_loss": 3.974254802932836, + "tokens_seen": 464977920 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004338816449348044, + "loss": 1.5075, + "theoretical_loss": 3.974194155823021, + "tokens_seen": 465043456 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043387161484453364, + "loss": 1.4776, + "theoretical_loss": 3.9741335196519465, + "tokens_seen": 465108992 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004338615847542628, + "loss": 1.4837, + "theoretical_loss": 3.9740728944160977, + "tokens_seen": 465174528 + }, + { + "epoch": 0.14, + "learning_rate": 0.000433851554663992, + "loss": 1.4964, + "theoretical_loss": 3.974012280111963, + "tokens_seen": 465240064 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 160329, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.444647192955017, + "objective/train/theoretical_loss": 3.973951676736032, + "objective/train/tokens_used": 465305600, + "theoretical_loss": 3.973951676736032, + "tokens_seen": 465305600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004338415245737212, + "loss": 1.5042, + "theoretical_loss": 3.973951676736032, + "tokens_seen": 465305600 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043383149448345036, + "loss": 1.4074, + "theoretical_loss": 3.973891084284795, + "tokens_seen": 465371136 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043382146439317954, + "loss": 1.5261, + "theoretical_loss": 3.973830502754745, + "tokens_seen": 465436672 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004338114343029088, + "loss": 1.4619, + "theoretical_loss": 3.973769932142376, + "tokens_seen": 465502208 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004338014042126379, + "loss": 1.4726, + "theoretical_loss": 3.9737093724441843, + "tokens_seen": 465567744 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043379137412236714, + "loss": 1.4606, + "theoretical_loss": 3.9736488236566667, + "tokens_seen": 465633280 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043378134403209627, + "loss": 1.4856, + "theoretical_loss": 3.973588285776323, + "tokens_seen": 465698816 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004337713139418255, + "loss": 1.5165, + "theoretical_loss": 3.9735277587996527, + "tokens_seen": 465764352 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004337612838515547, + "loss": 1.472, + "theoretical_loss": 3.973467242723159, + "tokens_seen": 465829888 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043375125376128387, + "loss": 1.5737, + "theoretical_loss": 3.9734067375433466, + "tokens_seen": 465895424 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043374122367101305, + "loss": 1.4676, + "theoretical_loss": 3.9733462432567195, + "tokens_seen": 465960960 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004337311935807423, + "loss": 1.4419, + "theoretical_loss": 3.9732857598597855, + "tokens_seen": 466026496 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004337211634904714, + "loss": 1.4983, + "theoretical_loss": 3.9732252873490532, + "tokens_seen": 466092032 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043371113340020064, + "loss": 1.5281, + "theoretical_loss": 3.9731648257210335, + "tokens_seen": 466157568 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043370110330992977, + "loss": 1.5223, + "theoretical_loss": 3.9731043749722375, + "tokens_seen": 466223104 + }, + { + "epoch": 0.14, + "learning_rate": 0.000433691073219659, + "loss": 1.5252, + "theoretical_loss": 3.9730439350991795, + "tokens_seen": 466288640 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004336810431293882, + "loss": 1.5576, + "theoretical_loss": 3.9729835060983745, + "tokens_seen": 466354176 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043367101303911737, + "loss": 1.5102, + "theoretical_loss": 3.972923087966339, + "tokens_seen": 466419712 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043366098294884655, + "loss": 1.5033, + "theoretical_loss": 3.9728626806995915, + "tokens_seen": 466485248 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043365095285857573, + "loss": 1.5223, + "theoretical_loss": 3.9728022842946524, + "tokens_seen": 466550784 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004336409227683049, + "loss": 1.5141, + "theoretical_loss": 3.972741898748043, + "tokens_seen": 466616320 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043363089267803415, + "loss": 1.4884, + "theoretical_loss": 3.972681524056286, + "tokens_seen": 466681856 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004336208625877633, + "loss": 1.5584, + "theoretical_loss": 3.9726211602159074, + "tokens_seen": 466747392 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004336108324974925, + "loss": 1.4741, + "theoretical_loss": 3.9725608072234326, + "tokens_seen": 466812928 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043360080240722164, + "loss": 1.4307, + "theoretical_loss": 3.97250046507539, + "tokens_seen": 466878464 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043359077231695087, + "loss": 1.595, + "theoretical_loss": 3.972440133768309, + "tokens_seen": 466944000 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043358074222668005, + "loss": 1.5028, + "theoretical_loss": 3.972379813298721, + "tokens_seen": 467009536 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043357071213640923, + "loss": 1.4818, + "theoretical_loss": 3.972319503663159, + "tokens_seen": 467075072 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004335606820461384, + "loss": 1.5314, + "theoretical_loss": 3.972259204858157, + "tokens_seen": 467140608 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043355065195586765, + "loss": 1.4413, + "theoretical_loss": 3.9721989168802505, + "tokens_seen": 467206144 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004335406218655968, + "loss": 1.4923, + "theoretical_loss": 3.9721386397259786, + "tokens_seen": 467271680 + }, + { + "epoch": 0.14, + "learning_rate": 0.000433530591775326, + "loss": 1.5856, + "theoretical_loss": 3.972078373391879, + "tokens_seen": 467337216 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043352056168505514, + "loss": 1.5861, + "theoretical_loss": 3.9720181178744935, + "tokens_seen": 467402752 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004335105315947844, + "loss": 1.5039, + "theoretical_loss": 3.971957873170364, + "tokens_seen": 467468288 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043350050150451356, + "loss": 1.5065, + "theoretical_loss": 3.971897639276035, + "tokens_seen": 467533824 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043349047141424274, + "loss": 1.6467, + "theoretical_loss": 3.9718374161880505, + "tokens_seen": 467599360 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004334804413239719, + "loss": 1.4894, + "theoretical_loss": 3.971777203902959, + "tokens_seen": 467664896 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004334704112337011, + "loss": 1.5145, + "theoretical_loss": 3.971717002417309, + "tokens_seen": 467730432 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004334603811434303, + "loss": 1.5124, + "theoretical_loss": 3.9716568117276503, + "tokens_seen": 467795968 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004334503510531595, + "loss": 1.5015, + "theoretical_loss": 3.9715966318305354, + "tokens_seen": 467861504 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043344032096288864, + "loss": 1.4829, + "theoretical_loss": 3.9715364627225176, + "tokens_seen": 467927040 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004334302908726179, + "loss": 1.4824, + "theoretical_loss": 3.9714763044001513, + "tokens_seen": 467992576 + }, + { + "epoch": 0.14, + "learning_rate": 0.000433420260782347, + "loss": 1.5397, + "theoretical_loss": 3.971416156859995, + "tokens_seen": 468058112 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043341023069207624, + "loss": 1.3696, + "theoretical_loss": 3.9713560200986042, + "tokens_seen": 468123648 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004334002006018054, + "loss": 1.3791, + "theoretical_loss": 3.9712958941125405, + "tokens_seen": 468189184 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004333901705115346, + "loss": 1.4487, + "theoretical_loss": 3.971235778898365, + "tokens_seen": 468254720 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004333801404212638, + "loss": 1.4902, + "theoretical_loss": 3.971175674452641, + "tokens_seen": 468320256 + }, + { + "epoch": 0.14, + "learning_rate": 0.000433370110330993, + "loss": 1.4647, + "theoretical_loss": 3.9711155807719316, + "tokens_seen": 468385792 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043336008024072215, + "loss": 1.5229, + "theoretical_loss": 3.971055497852804, + "tokens_seen": 468451328 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004333500501504514, + "loss": 1.4916, + "theoretical_loss": 3.9709954256918265, + "tokens_seen": 468516864 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 161552, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5734035968780518, + "objective/train/theoretical_loss": 3.970935364285567, + "objective/train/tokens_used": 468582400, + "theoretical_loss": 3.970935364285567, + "tokens_seen": 468582400 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004333400200601805, + "loss": 1.5935, + "theoretical_loss": 3.970935364285567, + "tokens_seen": 468582400 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043332998996990974, + "loss": 1.4732, + "theoretical_loss": 3.9708753136305974, + "tokens_seen": 468647936 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004333199598796389, + "loss": 1.4336, + "theoretical_loss": 3.9708152737234887, + "tokens_seen": 468713472 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004333099297893681, + "loss": 1.5202, + "theoretical_loss": 3.9707552445608165, + "tokens_seen": 468779008 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004332998996990973, + "loss": 1.5397, + "theoretical_loss": 3.9706952261391555, + "tokens_seen": 468844544 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043328986960882647, + "loss": 1.5196, + "theoretical_loss": 3.970635218455083, + "tokens_seen": 468910080 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043327983951855565, + "loss": 1.4566, + "theoretical_loss": 3.970575221505178, + "tokens_seen": 468975616 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004332698094282849, + "loss": 1.4972, + "theoretical_loss": 3.97051523528602, + "tokens_seen": 469041152 + }, + { + "epoch": 0.14, + "learning_rate": 0.000433259779338014, + "loss": 1.4775, + "theoretical_loss": 3.970455259794191, + "tokens_seen": 469106688 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043324974924774325, + "loss": 1.5315, + "theoretical_loss": 3.970395295026275, + "tokens_seen": 469172224 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043323971915747243, + "loss": 1.5513, + "theoretical_loss": 3.970335340978856, + "tokens_seen": 469237760 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004332296890672016, + "loss": 1.5979, + "theoretical_loss": 3.970275397648522, + "tokens_seen": 469303296 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004332196589769308, + "loss": 1.5695, + "theoretical_loss": 3.9702154650318597, + "tokens_seen": 469368832 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043320962888665997, + "loss": 1.5271, + "theoretical_loss": 3.9701555431254594, + "tokens_seen": 469434368 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043319959879638915, + "loss": 1.4718, + "theoretical_loss": 3.9700956319259113, + "tokens_seen": 469499904 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004331895687061184, + "loss": 1.4463, + "theoretical_loss": 3.97003573142981, + "tokens_seen": 469565440 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043317953861584757, + "loss": 1.4212, + "theoretical_loss": 3.9699758416337483, + "tokens_seen": 469630976 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043316950852557675, + "loss": 1.4683, + "theoretical_loss": 3.9699159625343223, + "tokens_seen": 469696512 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043315947843530593, + "loss": 1.5468, + "theoretical_loss": 3.96985609412813, + "tokens_seen": 469762048 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004331494483450351, + "loss": 1.4106, + "theoretical_loss": 3.96979623641177, + "tokens_seen": 469827584 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043313941825476435, + "loss": 1.5714, + "theoretical_loss": 3.9697363893818434, + "tokens_seen": 469893120 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004331293881644935, + "loss": 1.4356, + "theoretical_loss": 3.969676553034951, + "tokens_seen": 469958656 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004331193580742227, + "loss": 1.5144, + "theoretical_loss": 3.969616727367698, + "tokens_seen": 470024192 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043310932798395184, + "loss": 1.4395, + "theoretical_loss": 3.9695569123766887, + "tokens_seen": 470089728 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043309929789368107, + "loss": 1.3793, + "theoretical_loss": 3.9694971080585297, + "tokens_seen": 470155264 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043308926780341025, + "loss": 1.4779, + "theoretical_loss": 3.9694373144098307, + "tokens_seen": 470220800 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043307923771313943, + "loss": 1.4722, + "theoretical_loss": 3.9693775314271997, + "tokens_seen": 470286336 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004330692076228686, + "loss": 1.551, + "theoretical_loss": 3.9693177591072493, + "tokens_seen": 470351872 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043305917753259785, + "loss": 1.5278, + "theoretical_loss": 3.9692579974465922, + "tokens_seen": 470417408 + }, + { + "epoch": 0.14, + "learning_rate": 0.000433049147442327, + "loss": 1.5122, + "theoretical_loss": 3.9691982464418434, + "tokens_seen": 470482944 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004330391173520562, + "loss": 1.4317, + "theoretical_loss": 3.9691385060896174, + "tokens_seen": 470548480 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043302908726178534, + "loss": 1.5176, + "theoretical_loss": 3.9690787763865334, + "tokens_seen": 470614016 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004330190571715146, + "loss": 1.5107, + "theoretical_loss": 3.96901905732921, + "tokens_seen": 470679552 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043300902708124376, + "loss": 1.4306, + "theoretical_loss": 3.968959348914268, + "tokens_seen": 470745088 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043299899699097294, + "loss": 1.4437, + "theoretical_loss": 3.9688996511383294, + "tokens_seen": 470810624 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004329889669007021, + "loss": 1.5177, + "theoretical_loss": 3.968839963998018, + "tokens_seen": 470876160 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004329789368104313, + "loss": 1.5029, + "theoretical_loss": 3.9687802874899596, + "tokens_seen": 470941696 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004329689067201605, + "loss": 1.4771, + "theoretical_loss": 3.9687206216107804, + "tokens_seen": 471007232 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004329588766298897, + "loss": 1.4452, + "theoretical_loss": 3.968660966357109, + "tokens_seen": 471072768 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043294884653961884, + "loss": 1.5097, + "theoretical_loss": 3.9686013217255756, + "tokens_seen": 471138304 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004329388164493481, + "loss": 1.5306, + "theoretical_loss": 3.9685416877128112, + "tokens_seen": 471203840 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004329287863590772, + "loss": 1.5416, + "theoretical_loss": 3.9684820643154497, + "tokens_seen": 471269376 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043291875626880644, + "loss": 1.5318, + "theoretical_loss": 3.9684224515301247, + "tokens_seen": 471334912 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004329087261785356, + "loss": 1.54, + "theoretical_loss": 3.968362849353473, + "tokens_seen": 471400448 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004328986960882648, + "loss": 1.4932, + "theoretical_loss": 3.9683032577821313, + "tokens_seen": 471465984 + }, + { + "epoch": 0.14, + "learning_rate": 0.000432888665997994, + "loss": 1.4455, + "theoretical_loss": 3.9682436768127394, + "tokens_seen": 471531520 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004328786359077232, + "loss": 1.4663, + "theoretical_loss": 3.968184106441938, + "tokens_seen": 471597056 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043286860581745235, + "loss": 1.5826, + "theoretical_loss": 3.9681245466663695, + "tokens_seen": 471662592 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004328585757271816, + "loss": 1.5518, + "theoretical_loss": 3.9680649974826774, + "tokens_seen": 471728128 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004328485456369107, + "loss": 1.5456, + "theoretical_loss": 3.9680054588875064, + "tokens_seen": 471793664 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 162412, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.55324387550354, + "objective/train/theoretical_loss": 3.9679459308775042, + "objective/train/tokens_used": 471859200, + "theoretical_loss": 3.9679459308775042, + "tokens_seen": 471859200 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043283851554663994, + "loss": 1.4185, + "theoretical_loss": 3.9679459308775042, + "tokens_seen": 471859200 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004328284854563691, + "loss": 1.5331, + "theoretical_loss": 3.9678864134493184, + "tokens_seen": 471924736 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004328184553660983, + "loss": 1.4231, + "theoretical_loss": 3.9678269065996, + "tokens_seen": 471990272 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004328084252758275, + "loss": 1.4968, + "theoretical_loss": 3.967767410324999, + "tokens_seen": 472055808 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043279839518555667, + "loss": 1.5029, + "theoretical_loss": 3.967707924622169, + "tokens_seen": 472121344 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043278836509528585, + "loss": 1.3986, + "theoretical_loss": 3.967648449487764, + "tokens_seen": 472186880 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004327783350050151, + "loss": 1.4438, + "theoretical_loss": 3.9675889849184407, + "tokens_seen": 472252416 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004327683049147442, + "loss": 1.3879, + "theoretical_loss": 3.9675295309108556, + "tokens_seen": 472317952 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043275827482447345, + "loss": 1.5214, + "theoretical_loss": 3.9674700874616686, + "tokens_seen": 472383488 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043274824473420263, + "loss": 1.536, + "theoretical_loss": 3.9674106545675403, + "tokens_seen": 472449024 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004327382146439318, + "loss": 1.4322, + "theoretical_loss": 3.967351232225132, + "tokens_seen": 472514560 + }, + { + "epoch": 0.14, + "learning_rate": 0.000432728184553661, + "loss": 1.319, + "theoretical_loss": 3.9672918204311074, + "tokens_seen": 472580096 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043271815446339017, + "loss": 1.4978, + "theoretical_loss": 3.9672324191821318, + "tokens_seen": 472645632 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043270812437311935, + "loss": 1.4541, + "theoretical_loss": 3.967173028474871, + "tokens_seen": 472711168 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004326980942828486, + "loss": 1.4352, + "theoretical_loss": 3.967113648305995, + "tokens_seen": 472776704 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004326880641925777, + "loss": 1.507, + "theoretical_loss": 3.9670542786721716, + "tokens_seen": 472842240 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043267803410230695, + "loss": 1.4679, + "theoretical_loss": 3.9669949195700727, + "tokens_seen": 472907776 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004326680040120361, + "loss": 1.5211, + "theoretical_loss": 3.9669355709963705, + "tokens_seen": 472973312 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004326579739217653, + "loss": 1.5491, + "theoretical_loss": 3.9668762329477403, + "tokens_seen": 473038848 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004326479438314945, + "loss": 1.4415, + "theoretical_loss": 3.966816905420856, + "tokens_seen": 473104384 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004326379137412237, + "loss": 1.5106, + "theoretical_loss": 3.966757588412396, + "tokens_seen": 473169920 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043262788365095286, + "loss": 1.5596, + "theoretical_loss": 3.9666982819190384, + "tokens_seen": 473235456 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043261785356068204, + "loss": 1.5715, + "theoretical_loss": 3.9666389859374642, + "tokens_seen": 473300992 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004326078234704112, + "loss": 1.3957, + "theoretical_loss": 3.9665797004643544, + "tokens_seen": 473366528 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043259779338014045, + "loss": 1.3874, + "theoretical_loss": 3.966520425496392, + "tokens_seen": 473432064 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004325877632898696, + "loss": 1.4833, + "theoretical_loss": 3.9664611610302627, + "tokens_seen": 473497600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004325777331995988, + "loss": 1.5608, + "theoretical_loss": 3.966401907062652, + "tokens_seen": 473563136 + }, + { + "epoch": 0.14, + "learning_rate": 0.000432567703109328, + "loss": 1.4578, + "theoretical_loss": 3.9663426635902477, + "tokens_seen": 473628672 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004325576730190572, + "loss": 1.5785, + "theoretical_loss": 3.9662834306097388, + "tokens_seen": 473694208 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043254764292878636, + "loss": 1.4756, + "theoretical_loss": 3.9662242081178167, + "tokens_seen": 473759744 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043253761283851554, + "loss": 1.4905, + "theoretical_loss": 3.9661649961111727, + "tokens_seen": 473825280 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004325275827482447, + "loss": 1.5074, + "theoretical_loss": 3.9661057945865017, + "tokens_seen": 473890816 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043251755265797396, + "loss": 1.5431, + "theoretical_loss": 3.966046603540498, + "tokens_seen": 473956352 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004325075225677031, + "loss": 1.5087, + "theoretical_loss": 3.9659874229698584, + "tokens_seen": 474021888 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004324974924774323, + "loss": 1.5017, + "theoretical_loss": 3.9659282528712816, + "tokens_seen": 474087424 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043248746238716145, + "loss": 1.4804, + "theoretical_loss": 3.965869093241467, + "tokens_seen": 474152960 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004324774322968907, + "loss": 1.3789, + "theoretical_loss": 3.965809944077116, + "tokens_seen": 474218496 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043246740220661986, + "loss": 1.497, + "theoretical_loss": 3.965750805374931, + "tokens_seen": 474284032 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043245737211634904, + "loss": 1.4523, + "theoretical_loss": 3.965691677131617, + "tokens_seen": 474349568 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004324473420260782, + "loss": 1.53, + "theoretical_loss": 3.9656325593438786, + "tokens_seen": 474415104 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004324373119358074, + "loss": 1.5035, + "theoretical_loss": 3.9655734520084236, + "tokens_seen": 474480640 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043242728184553664, + "loss": 1.6323, + "theoretical_loss": 3.965514355121961, + "tokens_seen": 474546176 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004324172517552658, + "loss": 1.4547, + "theoretical_loss": 3.9654552686812004, + "tokens_seen": 474611712 + }, + { + "epoch": 0.14, + "learning_rate": 0.000432407221664995, + "loss": 1.562, + "theoretical_loss": 3.9653961926828534, + "tokens_seen": 474677248 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004323971915747242, + "loss": 1.5377, + "theoretical_loss": 3.9653371271236337, + "tokens_seen": 474742784 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004323871614844534, + "loss": 1.581, + "theoretical_loss": 3.9652780720002556, + "tokens_seen": 474808320 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043237713139418255, + "loss": 1.5035, + "theoretical_loss": 3.965219027309435, + "tokens_seen": 474873856 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004323671013039118, + "loss": 1.4996, + "theoretical_loss": 3.9651599930478905, + "tokens_seen": 474939392 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004323570712136409, + "loss": 1.4637, + "theoretical_loss": 3.9651009692123402, + "tokens_seen": 475004928 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043234704112337014, + "loss": 1.5048, + "theoretical_loss": 3.9650419557995047, + "tokens_seen": 475070464 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 163558, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4912731647491455, + "objective/train/theoretical_loss": 3.964982952806107, + "objective/train/tokens_used": 475136000, + "theoretical_loss": 3.964982952806107, + "tokens_seen": 475136000 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004323370110330993, + "loss": 1.4879, + "theoretical_loss": 3.964982952806107, + "tokens_seen": 475136000 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004323269809428285, + "loss": 1.5633, + "theoretical_loss": 3.9649239602288695, + "tokens_seen": 475201536 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004323169508525577, + "loss": 1.4602, + "theoretical_loss": 3.9648649780645178, + "tokens_seen": 475267072 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043230692076228687, + "loss": 1.509, + "theoretical_loss": 3.9648060063097788, + "tokens_seen": 475332608 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043229689067201605, + "loss": 1.3273, + "theoretical_loss": 3.9647470449613795, + "tokens_seen": 475398144 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004322868605817453, + "loss": 1.4804, + "theoretical_loss": 3.9646880940160507, + "tokens_seen": 475463680 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004322768304914744, + "loss": 1.4277, + "theoretical_loss": 3.9646291534705225, + "tokens_seen": 475529216 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043226680040120365, + "loss": 1.5199, + "theoretical_loss": 3.9645702233215268, + "tokens_seen": 475594752 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043225677031093283, + "loss": 1.4588, + "theoretical_loss": 3.964511303565799, + "tokens_seen": 475660288 + }, + { + "epoch": 0.14, + "learning_rate": 0.000432246740220662, + "loss": 1.3813, + "theoretical_loss": 3.964452394200073, + "tokens_seen": 475725824 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004322367101303912, + "loss": 1.3616, + "theoretical_loss": 3.9643934952210866, + "tokens_seen": 475791360 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043222668004012037, + "loss": 1.5623, + "theoretical_loss": 3.9643346066255782, + "tokens_seen": 475856896 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043221664994984955, + "loss": 1.5867, + "theoretical_loss": 3.964275728410287, + "tokens_seen": 475922432 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004322066198595788, + "loss": 1.5069, + "theoretical_loss": 3.9642168605719545, + "tokens_seen": 475987968 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004321965897693079, + "loss": 1.489, + "theoretical_loss": 3.9641580031073236, + "tokens_seen": 476053504 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043218655967903715, + "loss": 1.5568, + "theoretical_loss": 3.9640991560131384, + "tokens_seen": 476119040 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004321765295887663, + "loss": 1.5371, + "theoretical_loss": 3.964040319286145, + "tokens_seen": 476184576 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004321664994984955, + "loss": 1.5074, + "theoretical_loss": 3.9639814929230894, + "tokens_seen": 476250112 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004321564694082247, + "loss": 1.4769, + "theoretical_loss": 3.9639226769207214, + "tokens_seen": 476315648 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004321464393179539, + "loss": 1.5865, + "theoretical_loss": 3.963863871275791, + "tokens_seen": 476381184 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043213640922768306, + "loss": 1.4366, + "theoretical_loss": 3.963805075985049, + "tokens_seen": 476446720 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043212637913741224, + "loss": 1.6296, + "theoretical_loss": 3.9637462910452497, + "tokens_seen": 476512256 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004321163490471414, + "loss": 1.5039, + "theoretical_loss": 3.963687516453146, + "tokens_seen": 476577792 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043210631895687065, + "loss": 1.4721, + "theoretical_loss": 3.9636287522054947, + "tokens_seen": 476643328 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004320962888665998, + "loss": 1.5191, + "theoretical_loss": 3.9635699982990538, + "tokens_seen": 476708864 + }, + { + "epoch": 0.14, + "learning_rate": 0.000432086258776329, + "loss": 1.4589, + "theoretical_loss": 3.963511254730581, + "tokens_seen": 476774400 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004320762286860582, + "loss": 1.5421, + "theoretical_loss": 3.9634525214968375, + "tokens_seen": 476839936 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004320661985957874, + "loss": 1.4412, + "theoretical_loss": 3.963393798594585, + "tokens_seen": 476905472 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043205616850551656, + "loss": 1.5943, + "theoretical_loss": 3.963335086020586, + "tokens_seen": 476971008 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043204613841524574, + "loss": 1.6246, + "theoretical_loss": 3.9632763837716065, + "tokens_seen": 477036544 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004320361083249749, + "loss": 1.4758, + "theoretical_loss": 3.9632176918444113, + "tokens_seen": 477102080 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043202607823470416, + "loss": 1.5386, + "theoretical_loss": 3.9631590102357697, + "tokens_seen": 477167616 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004320160481444333, + "loss": 1.5016, + "theoretical_loss": 3.963100338942449, + "tokens_seen": 477233152 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004320060180541625, + "loss": 1.4958, + "theoretical_loss": 3.963041677961221, + "tokens_seen": 477298688 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043199598796389165, + "loss": 1.4638, + "theoretical_loss": 3.962983027288857, + "tokens_seen": 477364224 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004319859578736209, + "loss": 1.3932, + "theoretical_loss": 3.9629243869221304, + "tokens_seen": 477429760 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043197592778335006, + "loss": 1.452, + "theoretical_loss": 3.962865756857817, + "tokens_seen": 477495296 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043196589769307924, + "loss": 1.5733, + "theoretical_loss": 3.9628071370926925, + "tokens_seen": 477560832 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004319558676028084, + "loss": 1.5788, + "theoretical_loss": 3.9627485276235346, + "tokens_seen": 477626368 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004319458375125376, + "loss": 1.5059, + "theoretical_loss": 3.962689928447123, + "tokens_seen": 477691904 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004319358074222668, + "loss": 1.4244, + "theoretical_loss": 3.962631339560238, + "tokens_seen": 477757440 + }, + { + "epoch": 0.14, + "learning_rate": 0.000431925777331996, + "loss": 1.4746, + "theoretical_loss": 3.962572760959662, + "tokens_seen": 477822976 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043191574724172515, + "loss": 1.4602, + "theoretical_loss": 3.9625141926421783, + "tokens_seen": 477888512 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004319057171514544, + "loss": 1.5355, + "theoretical_loss": 3.9624556346045727, + "tokens_seen": 477954048 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043189568706118357, + "loss": 1.4573, + "theoretical_loss": 3.962397086843631, + "tokens_seen": 478019584 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043188565697091275, + "loss": 1.5114, + "theoretical_loss": 3.962338549356141, + "tokens_seen": 478085120 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043187562688064193, + "loss": 1.4353, + "theoretical_loss": 3.962280022138893, + "tokens_seen": 478150656 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004318655967903711, + "loss": 1.5003, + "theoretical_loss": 3.962221505188677, + "tokens_seen": 478216192 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004318555667001003, + "loss": 1.4028, + "theoretical_loss": 3.9621629985022855, + "tokens_seen": 478281728 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004318455366098295, + "loss": 1.5137, + "theoretical_loss": 3.9621045020765124, + "tokens_seen": 478347264 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 164567, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5039291381835938, + "objective/train/theoretical_loss": 3.9620460159081525, + "objective/train/tokens_used": 478412800, + "theoretical_loss": 3.9620460159081525, + "tokens_seen": 478412800 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043183550651955865, + "loss": 1.437, + "theoretical_loss": 3.9620460159081525, + "tokens_seen": 478412800 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004318254764292879, + "loss": 1.5041, + "theoretical_loss": 3.9619875399940025, + "tokens_seen": 478478336 + }, + { + "epoch": 0.15, + "learning_rate": 0.000431815446339017, + "loss": 1.4793, + "theoretical_loss": 3.9619290743308606, + "tokens_seen": 478543872 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043180541624874625, + "loss": 1.5897, + "theoretical_loss": 3.961870618915526, + "tokens_seen": 478609408 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043179538615847543, + "loss": 1.4994, + "theoretical_loss": 3.961812173744801, + "tokens_seen": 478674944 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004317853560682046, + "loss": 1.58, + "theoretical_loss": 3.961753738815486, + "tokens_seen": 478740480 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004317753259779338, + "loss": 1.4799, + "theoretical_loss": 3.961695314124385, + "tokens_seen": 478806016 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043176529588766303, + "loss": 1.5929, + "theoretical_loss": 3.961636899668304, + "tokens_seen": 478871552 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043175526579739215, + "loss": 1.4505, + "theoretical_loss": 3.96157849544405, + "tokens_seen": 478937088 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004317452357071214, + "loss": 1.5006, + "theoretical_loss": 3.96152010144843, + "tokens_seen": 479002624 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004317352056168505, + "loss": 1.5197, + "theoretical_loss": 3.961461717678254, + "tokens_seen": 479068160 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043172517552657975, + "loss": 1.5631, + "theoretical_loss": 3.961403344130333, + "tokens_seen": 479133696 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043171514543630893, + "loss": 1.4821, + "theoretical_loss": 3.9613449808014796, + "tokens_seen": 479199232 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004317051153460381, + "loss": 1.4121, + "theoretical_loss": 3.9612866276885073, + "tokens_seen": 479264768 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004316950852557673, + "loss": 1.4697, + "theoretical_loss": 3.961228284788231, + "tokens_seen": 479330304 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004316850551654965, + "loss": 1.4705, + "theoretical_loss": 3.9611699520974684, + "tokens_seen": 479395840 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004316750250752257, + "loss": 1.4808, + "theoretical_loss": 3.961111629613036, + "tokens_seen": 479461376 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004316649949849549, + "loss": 1.4573, + "theoretical_loss": 3.9610533173317553, + "tokens_seen": 479526912 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004316549648946841, + "loss": 1.54, + "theoretical_loss": 3.960995015250446, + "tokens_seen": 479592448 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043164493480441326, + "loss": 1.5397, + "theoretical_loss": 3.96093672336593, + "tokens_seen": 479657984 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043163490471414244, + "loss": 1.519, + "theoretical_loss": 3.9608784416750322, + "tokens_seen": 479723520 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004316248746238716, + "loss": 1.4962, + "theoretical_loss": 3.9608201701745775, + "tokens_seen": 479789056 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043161484453360085, + "loss": 1.4718, + "theoretical_loss": 3.960761908861392, + "tokens_seen": 479854592 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043160481444333, + "loss": 1.3783, + "theoretical_loss": 3.9607036577323047, + "tokens_seen": 479920128 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004315947843530592, + "loss": 1.5716, + "theoretical_loss": 3.9606454167841445, + "tokens_seen": 479985664 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004315847542627884, + "loss": 1.5149, + "theoretical_loss": 3.960587186013742, + "tokens_seen": 480051200 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004315747241725176, + "loss": 1.4788, + "theoretical_loss": 3.96052896541793, + "tokens_seen": 480116736 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043156469408224676, + "loss": 1.4672, + "theoretical_loss": 3.9604707549935423, + "tokens_seen": 480182272 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043155466399197594, + "loss": 1.3887, + "theoretical_loss": 3.9604125547374136, + "tokens_seen": 480247808 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004315446339017051, + "loss": 1.2977, + "theoretical_loss": 3.9603543646463812, + "tokens_seen": 480313344 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043153460381143436, + "loss": 1.5399, + "theoretical_loss": 3.9602961847172824, + "tokens_seen": 480378880 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004315245737211635, + "loss": 1.4211, + "theoretical_loss": 3.960238014946957, + "tokens_seen": 480444416 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004315145436308927, + "loss": 1.5236, + "theoretical_loss": 3.960179855332245, + "tokens_seen": 480509952 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043150451354062185, + "loss": 1.4044, + "theoretical_loss": 3.96012170586999, + "tokens_seen": 480575488 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004314944834503511, + "loss": 1.3495, + "theoretical_loss": 3.9600635665570345, + "tokens_seen": 480641024 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043148445336008026, + "loss": 1.3939, + "theoretical_loss": 3.9600054373902243, + "tokens_seen": 480706560 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043147442326980944, + "loss": 1.4961, + "theoretical_loss": 3.959947318366405, + "tokens_seen": 480772096 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004314643931795386, + "loss": 1.372, + "theoretical_loss": 3.9598892094824256, + "tokens_seen": 480837632 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004314543630892678, + "loss": 1.4515, + "theoretical_loss": 3.9598311107351347, + "tokens_seen": 480903168 + }, + { + "epoch": 0.15, + "learning_rate": 0.000431444332998997, + "loss": 1.5561, + "theoretical_loss": 3.959773022121383, + "tokens_seen": 480968704 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004314343029087262, + "loss": 1.4156, + "theoretical_loss": 3.959714943638023, + "tokens_seen": 481034240 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043142427281845535, + "loss": 1.4173, + "theoretical_loss": 3.9596568752819072, + "tokens_seen": 481099776 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004314142427281846, + "loss": 1.5174, + "theoretical_loss": 3.9595988170498915, + "tokens_seen": 481165312 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043140421263791377, + "loss": 1.602, + "theoretical_loss": 3.959540768938832, + "tokens_seen": 481230848 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043139418254764295, + "loss": 1.5024, + "theoretical_loss": 3.9594827309455862, + "tokens_seen": 481296384 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043138415245737213, + "loss": 1.4819, + "theoretical_loss": 3.9594247030670133, + "tokens_seen": 481361920 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004313741223671013, + "loss": 1.4998, + "theoretical_loss": 3.959366685299974, + "tokens_seen": 481427456 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004313640922768305, + "loss": 1.4936, + "theoretical_loss": 3.9593086776413298, + "tokens_seen": 481492992 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004313540621865597, + "loss": 1.4573, + "theoretical_loss": 3.959250680087945, + "tokens_seen": 481558528 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043134403209628885, + "loss": 1.562, + "theoretical_loss": 3.959192692636683, + "tokens_seen": 481624064 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 165832, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6490261554718018, + "objective/train/theoretical_loss": 3.959134715284411, + "objective/train/tokens_used": 481689600, + "theoretical_loss": 3.959134715284411, + "tokens_seen": 481689600 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004313340020060181, + "loss": 1.512, + "theoretical_loss": 3.959134715284411, + "tokens_seen": 481689600 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004313239719157472, + "loss": 1.5409, + "theoretical_loss": 3.959076748027996, + "tokens_seen": 481755136 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043131394182547645, + "loss": 1.479, + "theoretical_loss": 3.959018790864307, + "tokens_seen": 481820672 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043130391173520563, + "loss": 1.5021, + "theoretical_loss": 3.9589608437902144, + "tokens_seen": 481886208 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004312938816449348, + "loss": 1.3969, + "theoretical_loss": 3.9589029068025896, + "tokens_seen": 481951744 + }, + { + "epoch": 0.15, + "learning_rate": 0.000431283851554664, + "loss": 1.4065, + "theoretical_loss": 3.9588449798983065, + "tokens_seen": 482017280 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043127382146439323, + "loss": 1.3847, + "theoretical_loss": 3.9587870630742383, + "tokens_seen": 482082816 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043126379137412236, + "loss": 1.5656, + "theoretical_loss": 3.958729156327262, + "tokens_seen": 482148352 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004312537612838516, + "loss": 1.4349, + "theoretical_loss": 3.9586712596542553, + "tokens_seen": 482213888 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004312437311935807, + "loss": 1.5112, + "theoretical_loss": 3.958613373052095, + "tokens_seen": 482279424 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043123370110330995, + "loss": 1.5674, + "theoretical_loss": 3.958555496517663, + "tokens_seen": 482344960 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043122367101303913, + "loss": 1.5138, + "theoretical_loss": 3.95849763004784, + "tokens_seen": 482410496 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004312136409227683, + "loss": 1.4739, + "theoretical_loss": 3.958439773639509, + "tokens_seen": 482476032 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004312036108324975, + "loss": 1.5683, + "theoretical_loss": 3.958381927289554, + "tokens_seen": 482541568 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004311935807422267, + "loss": 1.4767, + "theoretical_loss": 3.958324090994861, + "tokens_seen": 482607104 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043118355065195586, + "loss": 1.4856, + "theoretical_loss": 3.958266264752317, + "tokens_seen": 482672640 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004311735205616851, + "loss": 1.5623, + "theoretical_loss": 3.9582084485588096, + "tokens_seen": 482738176 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004311634904714142, + "loss": 1.4553, + "theoretical_loss": 3.95815064241123, + "tokens_seen": 482803712 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043115346038114346, + "loss": 1.4889, + "theoretical_loss": 3.958092846306468, + "tokens_seen": 482869248 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004311434302908726, + "loss": 1.4019, + "theoretical_loss": 3.958035060241417, + "tokens_seen": 482934784 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004311334002006018, + "loss": 1.4984, + "theoretical_loss": 3.957977284212971, + "tokens_seen": 483000320 + }, + { + "epoch": 0.15, + "learning_rate": 0.000431123370110331, + "loss": 1.4393, + "theoretical_loss": 3.9579195182180245, + "tokens_seen": 483065856 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004311133400200602, + "loss": 1.5418, + "theoretical_loss": 3.957861762253475, + "tokens_seen": 483131392 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043110330992978936, + "loss": 1.5627, + "theoretical_loss": 3.9578040163162203, + "tokens_seen": 483196928 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004310932798395186, + "loss": 1.3924, + "theoretical_loss": 3.95774628040316, + "tokens_seen": 483262464 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004310832497492477, + "loss": 1.5125, + "theoretical_loss": 3.9576885545111944, + "tokens_seen": 483328000 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043107321965897696, + "loss": 1.437, + "theoretical_loss": 3.9576308386372263, + "tokens_seen": 483393536 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004310631895687061, + "loss": 1.4087, + "theoretical_loss": 3.957573132778159, + "tokens_seen": 483459072 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004310531594784353, + "loss": 1.5418, + "theoretical_loss": 3.9575154369308985, + "tokens_seen": 483524608 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004310431293881645, + "loss": 1.564, + "theoretical_loss": 3.957457751092349, + "tokens_seen": 483590144 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004310330992978937, + "loss": 1.4745, + "theoretical_loss": 3.9574000752594203, + "tokens_seen": 483655680 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043102306920762286, + "loss": 1.3516, + "theoretical_loss": 3.9573424094290206, + "tokens_seen": 483721216 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043101303911735205, + "loss": 1.4566, + "theoretical_loss": 3.9572847535980604, + "tokens_seen": 483786752 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004310030090270812, + "loss": 1.6107, + "theoretical_loss": 3.9572271077634515, + "tokens_seen": 483852288 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043099297893681046, + "loss": 1.5282, + "theoretical_loss": 3.957169471922107, + "tokens_seen": 483917824 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004309829488465396, + "loss": 1.5747, + "theoretical_loss": 3.9571118460709425, + "tokens_seen": 483983360 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004309729187562688, + "loss": 1.5832, + "theoretical_loss": 3.9570542302068725, + "tokens_seen": 484048896 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043096288866599795, + "loss": 1.5631, + "theoretical_loss": 3.9569966243268153, + "tokens_seen": 484114432 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004309528585757272, + "loss": 1.4147, + "theoretical_loss": 3.9569390284276893, + "tokens_seen": 484179968 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043094282848545637, + "loss": 1.5783, + "theoretical_loss": 3.9568814425064147, + "tokens_seen": 484245504 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043093279839518555, + "loss": 1.5019, + "theoretical_loss": 3.9568238665599127, + "tokens_seen": 484311040 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004309227683049148, + "loss": 1.4161, + "theoretical_loss": 3.9567663005851057, + "tokens_seen": 484376576 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043091273821464397, + "loss": 1.5201, + "theoretical_loss": 3.956708744578919, + "tokens_seen": 484442112 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043090270812437315, + "loss": 1.5152, + "theoretical_loss": 3.9566511985382773, + "tokens_seen": 484507648 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043089267803410233, + "loss": 1.4782, + "theoretical_loss": 3.956593662460108, + "tokens_seen": 484573184 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004308826479438315, + "loss": 1.54, + "theoretical_loss": 3.9565361363413385, + "tokens_seen": 484638720 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004308726178535607, + "loss": 1.4313, + "theoretical_loss": 3.9564786201788995, + "tokens_seen": 484704256 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004308625877632899, + "loss": 1.4913, + "theoretical_loss": 3.956421113969721, + "tokens_seen": 484769792 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043085255767301905, + "loss": 1.4702, + "theoretical_loss": 3.9563636177107364, + "tokens_seen": 484835328 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004308425275827483, + "loss": 1.4779, + "theoretical_loss": 3.956306131398878, + "tokens_seen": 484900864 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 166697, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4811618328094482, + "objective/train/theoretical_loss": 3.9562486550310823, + "objective/train/tokens_used": 484966400, + "theoretical_loss": 3.9562486550310823, + "tokens_seen": 484966400 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004308324974924774, + "loss": 1.5065, + "theoretical_loss": 3.9562486550310823, + "tokens_seen": 484966400 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043082246740220665, + "loss": 1.4773, + "theoretical_loss": 3.9561911886042846, + "tokens_seen": 485031936 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043081243731193583, + "loss": 1.4756, + "theoretical_loss": 3.9561337321154237, + "tokens_seen": 485097472 + }, + { + "epoch": 0.15, + "learning_rate": 0.000430802407221665, + "loss": 1.4294, + "theoretical_loss": 3.956076285561438, + "tokens_seen": 485163008 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004307923771313942, + "loss": 1.479, + "theoretical_loss": 3.9560188489392676, + "tokens_seen": 485228544 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043078234704112343, + "loss": 1.4904, + "theoretical_loss": 3.9559614222458555, + "tokens_seen": 485294080 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043077231695085256, + "loss": 1.3572, + "theoretical_loss": 3.955904005478144, + "tokens_seen": 485359616 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004307622868605818, + "loss": 1.4895, + "theoretical_loss": 3.9558465986330784, + "tokens_seen": 485425152 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004307522567703109, + "loss": 1.3945, + "theoretical_loss": 3.9557892017076037, + "tokens_seen": 485490688 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043074222668004015, + "loss": 1.4852, + "theoretical_loss": 3.9557318146986677, + "tokens_seen": 485556224 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043073219658976933, + "loss": 1.4833, + "theoretical_loss": 3.9556744376032196, + "tokens_seen": 485621760 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004307221664994985, + "loss": 1.4973, + "theoretical_loss": 3.9556170704182083, + "tokens_seen": 485687296 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004307121364092277, + "loss": 1.4489, + "theoretical_loss": 3.955559713140585, + "tokens_seen": 485752832 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004307021063189569, + "loss": 1.387, + "theoretical_loss": 3.9555023657673036, + "tokens_seen": 485818368 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043069207622868606, + "loss": 1.5001, + "theoretical_loss": 3.955445028295317, + "tokens_seen": 485883904 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004306820461384153, + "loss": 1.5022, + "theoretical_loss": 3.955387700721581, + "tokens_seen": 485949440 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004306720160481444, + "loss": 1.3816, + "theoretical_loss": 3.9553303830430524, + "tokens_seen": 486014976 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043066198595787366, + "loss": 1.4187, + "theoretical_loss": 3.955273075256689, + "tokens_seen": 486080512 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004306519558676028, + "loss": 1.4929, + "theoretical_loss": 3.9552157773594496, + "tokens_seen": 486146048 + }, + { + "epoch": 0.15, + "learning_rate": 0.000430641925777332, + "loss": 1.4542, + "theoretical_loss": 3.9551584893482965, + "tokens_seen": 486211584 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004306318956870612, + "loss": 1.4199, + "theoretical_loss": 3.9551012112201906, + "tokens_seen": 486277120 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004306218655967904, + "loss": 1.3337, + "theoretical_loss": 3.9550439429720954, + "tokens_seen": 486342656 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043061183550651956, + "loss": 1.519, + "theoretical_loss": 3.9549866846009762, + "tokens_seen": 486408192 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004306018054162488, + "loss": 1.4846, + "theoretical_loss": 3.9549294361037988, + "tokens_seen": 486473728 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004305917753259779, + "loss": 1.5607, + "theoretical_loss": 3.9548721974775303, + "tokens_seen": 486539264 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043058174523570716, + "loss": 1.4777, + "theoretical_loss": 3.95481496871914, + "tokens_seen": 486604800 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004305717151454363, + "loss": 1.4391, + "theoretical_loss": 3.954757749825598, + "tokens_seen": 486670336 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004305616850551655, + "loss": 1.5345, + "theoretical_loss": 3.9547005407938753, + "tokens_seen": 486735872 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004305516549648947, + "loss": 1.5031, + "theoretical_loss": 3.9546433416209448, + "tokens_seen": 486801408 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004305416248746239, + "loss": 1.5989, + "theoretical_loss": 3.9545861523037815, + "tokens_seen": 486866944 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043053159478435306, + "loss": 1.5244, + "theoretical_loss": 3.95452897283936, + "tokens_seen": 486932480 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043052156469408225, + "loss": 1.4597, + "theoretical_loss": 3.9544718032246573, + "tokens_seen": 486998016 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004305115346038114, + "loss": 1.4721, + "theoretical_loss": 3.9544146434566514, + "tokens_seen": 487063552 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043050150451354066, + "loss": 1.4852, + "theoretical_loss": 3.9543574935323216, + "tokens_seen": 487129088 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004304914744232698, + "loss": 1.2772, + "theoretical_loss": 3.95430035344865, + "tokens_seen": 487194624 + }, + { + "epoch": 0.15, + "learning_rate": 0.000430481444332999, + "loss": 1.5149, + "theoretical_loss": 3.954243223202617, + "tokens_seen": 487260160 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043047141424272815, + "loss": 1.503, + "theoretical_loss": 3.9541861027912066, + "tokens_seen": 487325696 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004304613841524574, + "loss": 1.3803, + "theoretical_loss": 3.9541289922114045, + "tokens_seen": 487391232 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043045135406218657, + "loss": 1.4077, + "theoretical_loss": 3.954071891460196, + "tokens_seen": 487456768 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043044132397191575, + "loss": 1.5017, + "theoretical_loss": 3.954014800534569, + "tokens_seen": 487522304 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043043129388164493, + "loss": 1.4556, + "theoretical_loss": 3.9539577194315116, + "tokens_seen": 487587840 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043042126379137417, + "loss": 1.4325, + "theoretical_loss": 3.953900648148015, + "tokens_seen": 487653376 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004304112337011033, + "loss": 1.5686, + "theoretical_loss": 3.953843586681069, + "tokens_seen": 487718912 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043040120361083253, + "loss": 1.4457, + "theoretical_loss": 3.9537865350276684, + "tokens_seen": 487784448 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043039117352056165, + "loss": 1.4446, + "theoretical_loss": 3.953729493184806, + "tokens_seen": 487849984 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004303811434302909, + "loss": 1.4155, + "theoretical_loss": 3.953672461149477, + "tokens_seen": 487915520 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043037111334002007, + "loss": 1.4617, + "theoretical_loss": 3.953615438918679, + "tokens_seen": 487981056 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043036108324974925, + "loss": 1.4781, + "theoretical_loss": 3.9535584264894092, + "tokens_seen": 488046592 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043035105315947843, + "loss": 1.4591, + "theoretical_loss": 3.9535014238586683, + "tokens_seen": 488112128 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004303410230692076, + "loss": 1.4868, + "theoretical_loss": 3.953444431023456, + "tokens_seen": 488177664 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 167910, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4630789756774902, + "objective/train/theoretical_loss": 3.9533874479807745, + "objective/train/tokens_used": 488243200, + "theoretical_loss": 3.9533874479807745, + "tokens_seen": 488243200 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004303309929789368, + "loss": 1.5076, + "theoretical_loss": 3.9533874479807745, + "tokens_seen": 488243200 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043032096288866603, + "loss": 1.501, + "theoretical_loss": 3.9533304747276263, + "tokens_seen": 488308736 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043031093279839516, + "loss": 1.4002, + "theoretical_loss": 3.953273511261018, + "tokens_seen": 488374272 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004303009027081244, + "loss": 1.5196, + "theoretical_loss": 3.953216557577954, + "tokens_seen": 488439808 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004302908726178535, + "loss": 1.4845, + "theoretical_loss": 3.9531596136754423, + "tokens_seen": 488505344 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043028084252758276, + "loss": 1.4104, + "theoretical_loss": 3.953102679550491, + "tokens_seen": 488570880 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043027081243731194, + "loss": 1.5421, + "theoretical_loss": 3.953045755200111, + "tokens_seen": 488636416 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004302607823470411, + "loss": 1.487, + "theoretical_loss": 3.9529888406213125, + "tokens_seen": 488701952 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004302507522567703, + "loss": 1.4554, + "theoretical_loss": 3.952931935811108, + "tokens_seen": 488767488 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043024072216649953, + "loss": 1.4143, + "theoretical_loss": 3.9528750407665125, + "tokens_seen": 488833024 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043023069207622866, + "loss": 1.3611, + "theoretical_loss": 3.9528181554845405, + "tokens_seen": 488898560 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004302206619859579, + "loss": 1.6642, + "theoretical_loss": 3.9527612799622087, + "tokens_seen": 488964096 + }, + { + "epoch": 0.15, + "learning_rate": 0.000430210631895687, + "loss": 1.3708, + "theoretical_loss": 3.9527044141965346, + "tokens_seen": 489029632 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043020060180541626, + "loss": 1.5251, + "theoretical_loss": 3.9526475581845375, + "tokens_seen": 489095168 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043019057171514544, + "loss": 1.4067, + "theoretical_loss": 3.9525907119232384, + "tokens_seen": 489160704 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004301805416248746, + "loss": 1.4577, + "theoretical_loss": 3.952533875409658, + "tokens_seen": 489226240 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043017051153460386, + "loss": 1.4484, + "theoretical_loss": 3.95247704864082, + "tokens_seen": 489291776 + }, + { + "epoch": 0.15, + "learning_rate": 0.000430160481444333, + "loss": 1.3769, + "theoretical_loss": 3.952420231613748, + "tokens_seen": 489357312 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004301504513540622, + "loss": 1.4081, + "theoretical_loss": 3.952363424325469, + "tokens_seen": 489422848 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004301404212637914, + "loss": 1.4742, + "theoretical_loss": 3.952306626773009, + "tokens_seen": 489488384 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004301303911735206, + "loss": 1.4903, + "theoretical_loss": 3.9522498389533967, + "tokens_seen": 489553920 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043012036108324976, + "loss": 1.5156, + "theoretical_loss": 3.952193060863661, + "tokens_seen": 489619456 + }, + { + "epoch": 0.15, + "learning_rate": 0.000430110330992979, + "loss": 1.4813, + "theoretical_loss": 3.9521362925008336, + "tokens_seen": 489684992 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004301003009027081, + "loss": 1.5447, + "theoretical_loss": 3.952079533861947, + "tokens_seen": 489750528 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043009027081243736, + "loss": 1.4101, + "theoretical_loss": 3.9520227849440337, + "tokens_seen": 489816064 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004300802407221665, + "loss": 1.5186, + "theoretical_loss": 3.951966045744129, + "tokens_seen": 489881600 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004300702106318957, + "loss": 1.461, + "theoretical_loss": 3.9519093162592687, + "tokens_seen": 489947136 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004300601805416249, + "loss": 1.4832, + "theoretical_loss": 3.95185259648649, + "tokens_seen": 490012672 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004300501504513541, + "loss": 1.4505, + "theoretical_loss": 3.951795886422833, + "tokens_seen": 490078208 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043004012036108326, + "loss": 1.445, + "theoretical_loss": 3.9517391860653364, + "tokens_seen": 490143744 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043003009027081245, + "loss": 1.4757, + "theoretical_loss": 3.951682495411041, + "tokens_seen": 490209280 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043002006018054163, + "loss": 1.371, + "theoretical_loss": 3.9516258144569907, + "tokens_seen": 490274816 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043001003009027086, + "loss": 1.4787, + "theoretical_loss": 3.9515691432002296, + "tokens_seen": 490340352 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043, + "loss": 1.5666, + "theoretical_loss": 3.951512481637801, + "tokens_seen": 490405888 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004299899699097292, + "loss": 1.4557, + "theoretical_loss": 3.9514558297667532, + "tokens_seen": 490471424 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042997993981945835, + "loss": 1.3942, + "theoretical_loss": 3.9513991875841334, + "tokens_seen": 490536960 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004299699097291876, + "loss": 1.5378, + "theoretical_loss": 3.95134255508699, + "tokens_seen": 490602496 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042995987963891677, + "loss": 1.5282, + "theoretical_loss": 3.9512859322723743, + "tokens_seen": 490668032 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042994984954864595, + "loss": 1.5093, + "theoretical_loss": 3.9512293191373375, + "tokens_seen": 490733568 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042993981945837513, + "loss": 1.4552, + "theoretical_loss": 3.9511727156789327, + "tokens_seen": 490799104 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042992978936810437, + "loss": 1.4717, + "theoretical_loss": 3.951116121894214, + "tokens_seen": 490864640 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004299197592778335, + "loss": 1.5277, + "theoretical_loss": 3.951059537780237, + "tokens_seen": 490930176 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042990972918756273, + "loss": 1.4782, + "theoretical_loss": 3.951002963334059, + "tokens_seen": 490995712 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042989969909729185, + "loss": 1.4264, + "theoretical_loss": 3.9509463985527375, + "tokens_seen": 491061248 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004298896690070211, + "loss": 1.4711, + "theoretical_loss": 3.9508898434333313, + "tokens_seen": 491126784 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042987963891675027, + "loss": 1.5219, + "theoretical_loss": 3.9508332979729026, + "tokens_seen": 491192320 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042986960882647945, + "loss": 1.498, + "theoretical_loss": 3.950776762168512, + "tokens_seen": 491257856 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042985957873620863, + "loss": 1.4154, + "theoretical_loss": 3.9507202360172236, + "tokens_seen": 491323392 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004298495486459378, + "loss": 1.4874, + "theoretical_loss": 3.9506637195161014, + "tokens_seen": 491388928 + }, + { + "epoch": 0.15, + "learning_rate": 0.000429839518555667, + "loss": 1.5044, + "theoretical_loss": 3.950607212662211, + "tokens_seen": 491454464 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 168795, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6833281517028809, + "objective/train/theoretical_loss": 3.950550715452621, + "objective/train/tokens_used": 491520000, + "theoretical_loss": 3.950550715452621, + "tokens_seen": 491520000 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042982948846539623, + "loss": 1.5753, + "theoretical_loss": 3.950550715452621, + "tokens_seen": 491520000 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042981945837512536, + "loss": 1.3144, + "theoretical_loss": 3.9504942278843984, + "tokens_seen": 491585536 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004298094282848546, + "loss": 1.402, + "theoretical_loss": 3.950437749954613, + "tokens_seen": 491651072 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004297993981945837, + "loss": 1.4679, + "theoretical_loss": 3.950381281660336, + "tokens_seen": 491716608 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042978936810431296, + "loss": 1.4314, + "theoretical_loss": 3.950324822998639, + "tokens_seen": 491782144 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042977933801404214, + "loss": 1.4199, + "theoretical_loss": 3.950268373966597, + "tokens_seen": 491847680 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004297693079237713, + "loss": 1.5075, + "theoretical_loss": 3.950211934561284, + "tokens_seen": 491913216 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004297592778335005, + "loss": 1.4631, + "theoretical_loss": 3.950155504779775, + "tokens_seen": 491978752 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042974924774322973, + "loss": 1.3992, + "theoretical_loss": 3.9500990846191484, + "tokens_seen": 492044288 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042973921765295886, + "loss": 1.5611, + "theoretical_loss": 3.950042674076483, + "tokens_seen": 492109824 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004297291875626881, + "loss": 1.4321, + "theoretical_loss": 3.949986273148858, + "tokens_seen": 492175360 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004297191574724172, + "loss": 1.4692, + "theoretical_loss": 3.9499298818333557, + "tokens_seen": 492240896 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042970912738214646, + "loss": 1.3859, + "theoretical_loss": 3.9498735001270573, + "tokens_seen": 492306432 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042969909729187564, + "loss": 1.4833, + "theoretical_loss": 3.949817128027047, + "tokens_seen": 492371968 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004296890672016048, + "loss": 1.4843, + "theoretical_loss": 3.9497607655304097, + "tokens_seen": 492437504 + }, + { + "epoch": 0.15, + "learning_rate": 0.000429679037111334, + "loss": 1.499, + "theoretical_loss": 3.949704412634232, + "tokens_seen": 492503040 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004296690070210632, + "loss": 1.4853, + "theoretical_loss": 3.949648069335601, + "tokens_seen": 492568576 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042965897693079236, + "loss": 1.4934, + "theoretical_loss": 3.9495917356316057, + "tokens_seen": 492634112 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004296489468405216, + "loss": 1.5697, + "theoretical_loss": 3.9495354115193364, + "tokens_seen": 492699648 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004296389167502507, + "loss": 1.4563, + "theoretical_loss": 3.9494790969958835, + "tokens_seen": 492765184 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042962888665997996, + "loss": 1.4565, + "theoretical_loss": 3.949422792058341, + "tokens_seen": 492830720 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004296188565697091, + "loss": 1.6195, + "theoretical_loss": 3.949366496703802, + "tokens_seen": 492896256 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004296088264794383, + "loss": 1.3894, + "theoretical_loss": 3.9493102109293616, + "tokens_seen": 492961792 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004295987963891675, + "loss": 1.4237, + "theoretical_loss": 3.9492539347321167, + "tokens_seen": 493027328 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004295887662988967, + "loss": 1.4206, + "theoretical_loss": 3.9491976681091643, + "tokens_seen": 493092864 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042957873620862587, + "loss": 1.3939, + "theoretical_loss": 3.949141411057604, + "tokens_seen": 493158400 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004295687061183551, + "loss": 1.4119, + "theoretical_loss": 3.9490851635745354, + "tokens_seen": 493223936 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042955867602808423, + "loss": 1.3593, + "theoretical_loss": 3.9490289256570605, + "tokens_seen": 493289472 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042954864593781347, + "loss": 1.4464, + "theoretical_loss": 3.9489726973022816, + "tokens_seen": 493355008 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004295386158475426, + "loss": 1.5102, + "theoretical_loss": 3.948916478507303, + "tokens_seen": 493420544 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042952858575727183, + "loss": 1.4257, + "theoretical_loss": 3.94886026926923, + "tokens_seen": 493486080 + }, + { + "epoch": 0.15, + "learning_rate": 0.000429518555667001, + "loss": 1.4516, + "theoretical_loss": 3.9488040695851687, + "tokens_seen": 493551616 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004295085255767302, + "loss": 1.526, + "theoretical_loss": 3.9487478794522275, + "tokens_seen": 493617152 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042949849548645937, + "loss": 1.5667, + "theoretical_loss": 3.948691698867515, + "tokens_seen": 493682688 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042948846539618855, + "loss": 1.4941, + "theoretical_loss": 3.9486355278281415, + "tokens_seen": 493748224 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042947843530591773, + "loss": 1.517, + "theoretical_loss": 3.948579366331219, + "tokens_seen": 493813760 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042946840521564697, + "loss": 1.4522, + "theoretical_loss": 3.94852321437386, + "tokens_seen": 493879296 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004294583751253761, + "loss": 1.4981, + "theoretical_loss": 3.9484670719531785, + "tokens_seen": 493944832 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042944834503510533, + "loss": 1.5463, + "theoretical_loss": 3.9484109390662896, + "tokens_seen": 494010368 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004294383149448345, + "loss": 1.387, + "theoretical_loss": 3.94835481571031, + "tokens_seen": 494075904 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004294282848545637, + "loss": 1.3735, + "theoretical_loss": 3.948298701882358, + "tokens_seen": 494141440 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042941825476429293, + "loss": 1.4129, + "theoretical_loss": 3.9482425975795525, + "tokens_seen": 494206976 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042940822467402206, + "loss": 1.4049, + "theoretical_loss": 3.9481865027990137, + "tokens_seen": 494272512 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004293981945837513, + "loss": 1.5257, + "theoretical_loss": 3.9481304175378633, + "tokens_seen": 494338048 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042938816449348047, + "loss": 1.4691, + "theoretical_loss": 3.948074341793224, + "tokens_seen": 494403584 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042937813440320965, + "loss": 1.5061, + "theoretical_loss": 3.94801827556222, + "tokens_seen": 494469120 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042936810431293883, + "loss": 1.5416, + "theoretical_loss": 3.9479622188419765, + "tokens_seen": 494534656 + }, + { + "epoch": 0.15, + "learning_rate": 0.000429358074222668, + "loss": 1.5429, + "theoretical_loss": 3.9479061716296204, + "tokens_seen": 494600192 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004293480441323972, + "loss": 1.4605, + "theoretical_loss": 3.9478501339222793, + "tokens_seen": 494665728 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042933801404212643, + "loss": 1.5031, + "theoretical_loss": 3.9477941057170822, + "tokens_seen": 494731264 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 170097, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.476405143737793, + "objective/train/theoretical_loss": 3.9477380870111602, + "objective/train/tokens_used": 494796800, + "theoretical_loss": 3.9477380870111602, + "tokens_seen": 494796800 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042932798395185556, + "loss": 1.4691, + "theoretical_loss": 3.9477380870111602, + "tokens_seen": 494796800 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004293179538615848, + "loss": 1.4586, + "theoretical_loss": 3.9476820778016437, + "tokens_seen": 494862336 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004293079237713139, + "loss": 1.4076, + "theoretical_loss": 3.9476260780856665, + "tokens_seen": 494927872 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042929789368104316, + "loss": 1.3844, + "theoretical_loss": 3.947570087860362, + "tokens_seen": 494993408 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042928786359077234, + "loss": 1.5439, + "theoretical_loss": 3.947514107122866, + "tokens_seen": 495058944 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004292778335005015, + "loss": 1.5419, + "theoretical_loss": 3.9474581358703147, + "tokens_seen": 495124480 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004292678034102307, + "loss": 1.6219, + "theoretical_loss": 3.947402174099847, + "tokens_seen": 495190016 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042925777331995993, + "loss": 1.3861, + "theoretical_loss": 3.9473462218085995, + "tokens_seen": 495255552 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042924774322968906, + "loss": 1.4992, + "theoretical_loss": 3.947290278993715, + "tokens_seen": 495321088 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004292377131394183, + "loss": 1.4772, + "theoretical_loss": 3.947234345652334, + "tokens_seen": 495386624 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004292276830491474, + "loss": 1.5218, + "theoretical_loss": 3.9471784217815995, + "tokens_seen": 495452160 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042921765295887666, + "loss": 1.4844, + "theoretical_loss": 3.9471225073786553, + "tokens_seen": 495517696 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042920762286860584, + "loss": 1.4864, + "theoretical_loss": 3.9470666024406467, + "tokens_seen": 495583232 + }, + { + "epoch": 0.15, + "learning_rate": 0.000429197592778335, + "loss": 1.4948, + "theoretical_loss": 3.9470107069647202, + "tokens_seen": 495648768 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004291875626880642, + "loss": 1.4634, + "theoretical_loss": 3.946954820948024, + "tokens_seen": 495714304 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004291775325977934, + "loss": 1.4256, + "theoretical_loss": 3.946898944387706, + "tokens_seen": 495779840 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042916750250752256, + "loss": 1.4646, + "theoretical_loss": 3.946843077280917, + "tokens_seen": 495845376 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004291574724172518, + "loss": 1.4191, + "theoretical_loss": 3.946787219624809, + "tokens_seen": 495910912 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004291474423269809, + "loss": 1.391, + "theoretical_loss": 3.9467313714165337, + "tokens_seen": 495976448 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042913741223671016, + "loss": 1.4624, + "theoretical_loss": 3.946675532653246, + "tokens_seen": 496041984 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004291273821464393, + "loss": 1.5425, + "theoretical_loss": 3.9466197033321, + "tokens_seen": 496107520 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004291173520561685, + "loss": 1.3073, + "theoretical_loss": 3.9465638834502528, + "tokens_seen": 496173056 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004291073219658977, + "loss": 1.425, + "theoretical_loss": 3.9465080730048614, + "tokens_seen": 496238592 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004290972918756269, + "loss": 1.5596, + "theoretical_loss": 3.9464522719930857, + "tokens_seen": 496304128 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042908726178535607, + "loss": 1.4764, + "theoretical_loss": 3.9463964804120844, + "tokens_seen": 496369664 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004290772316950853, + "loss": 1.5839, + "theoretical_loss": 3.9463406982590197, + "tokens_seen": 496435200 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042906720160481443, + "loss": 1.5077, + "theoretical_loss": 3.946284925531054, + "tokens_seen": 496500736 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042905717151454367, + "loss": 1.3756, + "theoretical_loss": 3.9462291622253503, + "tokens_seen": 496566272 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004290471414242728, + "loss": 1.4999, + "theoretical_loss": 3.946173408339075, + "tokens_seen": 496631808 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042903711133400203, + "loss": 1.5862, + "theoretical_loss": 3.9461176638693933, + "tokens_seen": 496697344 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004290270812437312, + "loss": 1.5243, + "theoretical_loss": 3.9460619288134726, + "tokens_seen": 496762880 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004290170511534604, + "loss": 1.4738, + "theoretical_loss": 3.9460062031684817, + "tokens_seen": 496828416 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042900702106318957, + "loss": 1.4316, + "theoretical_loss": 3.945950486931591, + "tokens_seen": 496893952 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042899699097291875, + "loss": 1.4736, + "theoretical_loss": 3.9458947800999713, + "tokens_seen": 496959488 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042898696088264793, + "loss": 1.4714, + "theoretical_loss": 3.9458390826707945, + "tokens_seen": 497025024 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042897693079237717, + "loss": 1.4703, + "theoretical_loss": 3.945783394641235, + "tokens_seen": 497090560 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004289669007021063, + "loss": 1.4872, + "theoretical_loss": 3.9457277160084665, + "tokens_seen": 497156096 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042895687061183553, + "loss": 1.5384, + "theoretical_loss": 3.9456720467696655, + "tokens_seen": 497221632 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004289468405215647, + "loss": 1.5285, + "theoretical_loss": 3.9456163869220093, + "tokens_seen": 497287168 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004289368104312939, + "loss": 1.5581, + "theoretical_loss": 3.945560736462677, + "tokens_seen": 497352704 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004289267803410231, + "loss": 1.4857, + "theoretical_loss": 3.945505095388847, + "tokens_seen": 497418240 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042891675025075226, + "loss": 1.5381, + "theoretical_loss": 3.9454494636977007, + "tokens_seen": 497483776 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042890672016048144, + "loss": 1.5886, + "theoretical_loss": 3.9453938413864207, + "tokens_seen": 497549312 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042889669007021067, + "loss": 1.5565, + "theoretical_loss": 3.94533822845219, + "tokens_seen": 497614848 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004288866599799398, + "loss": 1.4506, + "theoretical_loss": 3.9452826248921924, + "tokens_seen": 497680384 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042887662988966903, + "loss": 1.3449, + "theoretical_loss": 3.9452270307036144, + "tokens_seen": 497745920 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042886659979939816, + "loss": 1.497, + "theoretical_loss": 3.945171445883643, + "tokens_seen": 497811456 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004288565697091274, + "loss": 1.5478, + "theoretical_loss": 3.9451158704294658, + "tokens_seen": 497876992 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004288465396188566, + "loss": 1.4984, + "theoretical_loss": 3.945060304338273, + "tokens_seen": 497942528 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042883650952858576, + "loss": 1.5115, + "theoretical_loss": 3.9450047476072547, + "tokens_seen": 498008064 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 171256, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4620888233184814, + "objective/train/theoretical_loss": 3.9449492002336024, + "objective/train/tokens_used": 498073600, + "theoretical_loss": 3.9449492002336024, + "tokens_seen": 498073600 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042882647943831494, + "loss": 1.4244, + "theoretical_loss": 3.9449492002336024, + "tokens_seen": 498073600 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004288164493480441, + "loss": 1.5216, + "theoretical_loss": 3.9448936622145103, + "tokens_seen": 498139136 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004288064192577733, + "loss": 1.4085, + "theoretical_loss": 3.9448381335471714, + "tokens_seen": 498204672 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042879638916750254, + "loss": 1.4746, + "theoretical_loss": 3.9447826142287816, + "tokens_seen": 498270208 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042878635907723166, + "loss": 1.5062, + "theoretical_loss": 3.944727104256537, + "tokens_seen": 498335744 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004287763289869609, + "loss": 1.5229, + "theoretical_loss": 3.9446716036276364, + "tokens_seen": 498401280 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004287662988966901, + "loss": 1.4515, + "theoretical_loss": 3.944616112339279, + "tokens_seen": 498466816 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042875626880641926, + "loss": 1.5167, + "theoretical_loss": 3.944560630388664, + "tokens_seen": 498532352 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042874623871614844, + "loss": 1.4287, + "theoretical_loss": 3.9445051577729937, + "tokens_seen": 498597888 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004287362086258776, + "loss": 1.6114, + "theoretical_loss": 3.944449694489471, + "tokens_seen": 498663424 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004287261785356068, + "loss": 1.5398, + "theoretical_loss": 3.9443942405352987, + "tokens_seen": 498728960 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042871614844533604, + "loss": 1.4021, + "theoretical_loss": 3.9443387959076825, + "tokens_seen": 498794496 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042870611835506517, + "loss": 1.6092, + "theoretical_loss": 3.944283360603829, + "tokens_seen": 498860032 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004286960882647944, + "loss": 1.4255, + "theoretical_loss": 3.9442279346209457, + "tokens_seen": 498925568 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042868605817452353, + "loss": 1.5555, + "theoretical_loss": 3.944172517956241, + "tokens_seen": 498991104 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042867602808425276, + "loss": 1.5326, + "theoretical_loss": 3.944117110606925, + "tokens_seen": 499056640 + }, + { + "epoch": 0.15, + "learning_rate": 0.000428665997993982, + "loss": 1.5142, + "theoretical_loss": 3.9440617125702087, + "tokens_seen": 499122176 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004286559679037111, + "loss": 1.4531, + "theoretical_loss": 3.9440063238433045, + "tokens_seen": 499187712 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042864593781344036, + "loss": 1.557, + "theoretical_loss": 3.9439509444234258, + "tokens_seen": 499253248 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004286359077231695, + "loss": 1.5156, + "theoretical_loss": 3.943895574307788, + "tokens_seen": 499318784 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004286258776328987, + "loss": 1.5016, + "theoretical_loss": 3.9438402134936057, + "tokens_seen": 499384320 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004286158475426279, + "loss": 1.462, + "theoretical_loss": 3.9437848619780973, + "tokens_seen": 499449856 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004286058174523571, + "loss": 1.3531, + "theoretical_loss": 3.94372951975848, + "tokens_seen": 499515392 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042859578736208627, + "loss": 1.4517, + "theoretical_loss": 3.9436741868319745, + "tokens_seen": 499580928 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004285857572718155, + "loss": 1.3645, + "theoretical_loss": 3.943618863195801, + "tokens_seen": 499646464 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042857572718154463, + "loss": 1.4295, + "theoretical_loss": 3.943563548847181, + "tokens_seen": 499712000 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042856569709127387, + "loss": 1.4723, + "theoretical_loss": 3.943508243783338, + "tokens_seen": 499777536 + }, + { + "epoch": 0.15, + "learning_rate": 0.000428555667001003, + "loss": 1.525, + "theoretical_loss": 3.9434529480014966, + "tokens_seen": 499843072 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042854563691073223, + "loss": 1.434, + "theoretical_loss": 3.9433976614988815, + "tokens_seen": 499908608 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004285356068204614, + "loss": 1.55, + "theoretical_loss": 3.9433423842727198, + "tokens_seen": 499974144 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004285255767301906, + "loss": 1.4118, + "theoretical_loss": 3.9432871163202394, + "tokens_seen": 500039680 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042851554663991977, + "loss": 1.3978, + "theoretical_loss": 3.9432318576386693, + "tokens_seen": 500105216 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042850551654964895, + "loss": 1.4219, + "theoretical_loss": 3.94317660822524, + "tokens_seen": 500170752 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042849548645937813, + "loss": 1.4685, + "theoretical_loss": 3.9431213680771826, + "tokens_seen": 500236288 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042848545636910737, + "loss": 1.5187, + "theoretical_loss": 3.94306613719173, + "tokens_seen": 500301824 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004284754262788365, + "loss": 1.5027, + "theoretical_loss": 3.9430109155661164, + "tokens_seen": 500367360 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042846539618856573, + "loss": 1.4873, + "theoretical_loss": 3.9429557031975753, + "tokens_seen": 500432896 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004284553660982949, + "loss": 1.2917, + "theoretical_loss": 3.9429005000833444, + "tokens_seen": 500498432 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004284453360080241, + "loss": 1.5243, + "theoretical_loss": 3.9428453062206605, + "tokens_seen": 500563968 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004284353059177533, + "loss": 1.4382, + "theoretical_loss": 3.942790121606763, + "tokens_seen": 500629504 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042842527582748246, + "loss": 1.4605, + "theoretical_loss": 3.9427349462388905, + "tokens_seen": 500695040 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042841524573721164, + "loss": 1.4473, + "theoretical_loss": 3.942679780114284, + "tokens_seen": 500760576 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042840521564694087, + "loss": 1.5469, + "theoretical_loss": 3.9426246232301865, + "tokens_seen": 500826112 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042839518555667, + "loss": 1.5062, + "theoretical_loss": 3.9425694755838405, + "tokens_seen": 500891648 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042838515546639923, + "loss": 1.4635, + "theoretical_loss": 3.9425143371724913, + "tokens_seen": 500957184 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042837512537612836, + "loss": 1.3996, + "theoretical_loss": 3.942459207993384, + "tokens_seen": 501022720 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004283650952858576, + "loss": 1.5297, + "theoretical_loss": 3.9424040880437654, + "tokens_seen": 501088256 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004283550651955868, + "loss": 1.525, + "theoretical_loss": 3.9423489773208837, + "tokens_seen": 501153792 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042834503510531596, + "loss": 1.449, + "theoretical_loss": 3.9422938758219885, + "tokens_seen": 501219328 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042833500501504514, + "loss": 1.404, + "theoretical_loss": 3.9422387835443296, + "tokens_seen": 501284864 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 172250, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4970343112945557, + "objective/train/theoretical_loss": 3.9421837004851588, + "objective/train/tokens_used": 501350400, + "theoretical_loss": 3.9421837004851588, + "tokens_seen": 501350400 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004283249749247743, + "loss": 1.4588, + "theoretical_loss": 3.9421837004851588, + "tokens_seen": 501350400 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004283149448345035, + "loss": 1.5009, + "theoretical_loss": 3.942128626641729, + "tokens_seen": 501415936 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042830491474423274, + "loss": 1.5769, + "theoretical_loss": 3.942073562011294, + "tokens_seen": 501481472 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042829488465396186, + "loss": 1.4266, + "theoretical_loss": 3.9420185065911086, + "tokens_seen": 501547008 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004282848545636911, + "loss": 1.522, + "theoretical_loss": 3.94196346037843, + "tokens_seen": 501612544 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004282748244734203, + "loss": 1.5545, + "theoretical_loss": 3.941908423370515, + "tokens_seen": 501678080 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042826479438314946, + "loss": 1.538, + "theoretical_loss": 3.941853395564622, + "tokens_seen": 501743616 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042825476429287864, + "loss": 1.5504, + "theoretical_loss": 3.9417983769580114, + "tokens_seen": 501809152 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004282447342026078, + "loss": 1.5151, + "theoretical_loss": 3.941743367547944, + "tokens_seen": 501874688 + }, + { + "epoch": 0.15, + "learning_rate": 0.000428234704112337, + "loss": 1.5027, + "theoretical_loss": 3.9416883673316816, + "tokens_seen": 501940224 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042822467402206624, + "loss": 1.544, + "theoretical_loss": 3.9416333763064886, + "tokens_seen": 502005760 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042821464393179537, + "loss": 1.4798, + "theoretical_loss": 3.941578394469628, + "tokens_seen": 502071296 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004282046138415246, + "loss": 1.5106, + "theoretical_loss": 3.941523421818366, + "tokens_seen": 502136832 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042819458375125373, + "loss": 1.5554, + "theoretical_loss": 3.94146845834997, + "tokens_seen": 502202368 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042818455366098296, + "loss": 1.4305, + "theoretical_loss": 3.9414135040617078, + "tokens_seen": 502267904 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042817452357071215, + "loss": 1.5391, + "theoretical_loss": 3.9413585589508484, + "tokens_seen": 502333440 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004281644934804413, + "loss": 1.4371, + "theoretical_loss": 3.9413036230146625, + "tokens_seen": 502398976 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004281544633901705, + "loss": 1.403, + "theoretical_loss": 3.941248696250421, + "tokens_seen": 502464512 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004281444332998997, + "loss": 1.5403, + "theoretical_loss": 3.941193778655397, + "tokens_seen": 502530048 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042813440320962887, + "loss": 1.4469, + "theoretical_loss": 3.9411388702268644, + "tokens_seen": 502595584 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004281243731193581, + "loss": 1.401, + "theoretical_loss": 3.941083970962098, + "tokens_seen": 502661120 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042811434302908723, + "loss": 1.5611, + "theoretical_loss": 3.9410290808583746, + "tokens_seen": 502726656 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042810431293881647, + "loss": 1.4496, + "theoretical_loss": 3.9409741999129704, + "tokens_seen": 502792192 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042809428284854565, + "loss": 1.4342, + "theoretical_loss": 3.9409193281231647, + "tokens_seen": 502857728 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042808425275827483, + "loss": 1.4463, + "theoretical_loss": 3.9408644654862375, + "tokens_seen": 502923264 + }, + { + "epoch": 0.15, + "learning_rate": 0.000428074222668004, + "loss": 1.4677, + "theoretical_loss": 3.9408096119994687, + "tokens_seen": 502988800 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004280641925777332, + "loss": 1.5006, + "theoretical_loss": 3.940754767660141, + "tokens_seen": 503054336 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004280541624874624, + "loss": 1.4547, + "theoretical_loss": 3.940699932465537, + "tokens_seen": 503119872 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004280441323971916, + "loss": 1.4684, + "theoretical_loss": 3.9406451064129415, + "tokens_seen": 503185408 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042803410230692074, + "loss": 1.4798, + "theoretical_loss": 3.94059028949964, + "tokens_seen": 503250944 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042802407221664997, + "loss": 1.4321, + "theoretical_loss": 3.940535481722919, + "tokens_seen": 503316480 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004280140421263791, + "loss": 1.4905, + "theoretical_loss": 3.9404806830800667, + "tokens_seen": 503382016 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042800401203610833, + "loss": 1.4446, + "theoretical_loss": 3.940425893568371, + "tokens_seen": 503447552 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004279939819458375, + "loss": 1.4432, + "theoretical_loss": 3.9403711131851233, + "tokens_seen": 503513088 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004279839518555667, + "loss": 1.4766, + "theoretical_loss": 3.940316341927614, + "tokens_seen": 503578624 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004279739217652959, + "loss": 1.5185, + "theoretical_loss": 3.9402615797931357, + "tokens_seen": 503644160 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004279638916750251, + "loss": 1.4428, + "theoretical_loss": 3.940206826778982, + "tokens_seen": 503709696 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042795386158475424, + "loss": 1.5034, + "theoretical_loss": 3.9401520828824483, + "tokens_seen": 503775232 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004279438314944835, + "loss": 1.528, + "theoretical_loss": 3.9400973481008297, + "tokens_seen": 503840768 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004279338014042126, + "loss": 1.5426, + "theoretical_loss": 3.9400426224314233, + "tokens_seen": 503906304 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042792377131394184, + "loss": 1.447, + "theoretical_loss": 3.9399879058715275, + "tokens_seen": 503971840 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042791374122367107, + "loss": 1.5134, + "theoretical_loss": 3.9399331984184416, + "tokens_seen": 504037376 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004279037111334002, + "loss": 1.4672, + "theoretical_loss": 3.939878500069466, + "tokens_seen": 504102912 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042789368104312943, + "loss": 1.4899, + "theoretical_loss": 3.939823810821903, + "tokens_seen": 504168448 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042788365095285856, + "loss": 1.5416, + "theoretical_loss": 3.9397691306730547, + "tokens_seen": 504233984 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004278736208625878, + "loss": 1.4578, + "theoretical_loss": 3.9397144596202254, + "tokens_seen": 504299520 + }, + { + "epoch": 0.15, + "learning_rate": 0.000427863590772317, + "loss": 1.2862, + "theoretical_loss": 3.93965979766072, + "tokens_seen": 504365056 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042785356068204616, + "loss": 1.4157, + "theoretical_loss": 3.9396051447918445, + "tokens_seen": 504430592 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042784353059177534, + "loss": 1.4719, + "theoretical_loss": 3.939550501010907, + "tokens_seen": 504496128 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004278335005015045, + "loss": 1.5254, + "theoretical_loss": 3.939495866315216, + "tokens_seen": 504561664 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 173433, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4973053932189941, + "objective/train/theoretical_loss": 3.93944124070208, + "objective/train/tokens_used": 504627200, + "theoretical_loss": 3.93944124070208, + "tokens_seen": 504627200 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004278234704112337, + "loss": 1.4238, + "theoretical_loss": 3.93944124070208, + "tokens_seen": 504627200 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042781344032096294, + "loss": 1.4115, + "theoretical_loss": 3.9393866241688116, + "tokens_seen": 504692736 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042780341023069206, + "loss": 1.5517, + "theoretical_loss": 3.9393320167127213, + "tokens_seen": 504758272 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004277933801404213, + "loss": 1.4928, + "theoretical_loss": 3.939277418331123, + "tokens_seen": 504823808 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004277833500501505, + "loss": 1.5656, + "theoretical_loss": 3.9392228290213316, + "tokens_seen": 504889344 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042777331995987966, + "loss": 1.4917, + "theoretical_loss": 3.939168248780661, + "tokens_seen": 504954880 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042776328986960884, + "loss": 1.5559, + "theoretical_loss": 3.9391136776064286, + "tokens_seen": 505020416 + }, + { + "epoch": 0.15, + "learning_rate": 0.000427753259779338, + "loss": 1.5334, + "theoretical_loss": 3.939059115495952, + "tokens_seen": 505085952 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004277432296890672, + "loss": 1.4496, + "theoretical_loss": 3.93900456244655, + "tokens_seen": 505151488 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042773319959879644, + "loss": 1.4682, + "theoretical_loss": 3.938950018455543, + "tokens_seen": 505217024 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042772316950852557, + "loss": 1.5082, + "theoretical_loss": 3.938895483520252, + "tokens_seen": 505282560 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004277131394182548, + "loss": 1.4831, + "theoretical_loss": 3.938840957637998, + "tokens_seen": 505348096 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042770310932798393, + "loss": 1.4307, + "theoretical_loss": 3.9387864408061066, + "tokens_seen": 505413632 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042769307923771316, + "loss": 1.4434, + "theoretical_loss": 3.9387319330219004, + "tokens_seen": 505479168 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042768304914744235, + "loss": 1.5639, + "theoretical_loss": 3.938677434282706, + "tokens_seen": 505544704 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042767301905717153, + "loss": 1.4028, + "theoretical_loss": 3.93862294458585, + "tokens_seen": 505610240 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004276629889669007, + "loss": 1.5222, + "theoretical_loss": 3.938568463928661, + "tokens_seen": 505675776 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004276529588766299, + "loss": 1.5803, + "theoretical_loss": 3.9385139923084678, + "tokens_seen": 505741312 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042764292878635907, + "loss": 1.4982, + "theoretical_loss": 3.9384595297225995, + "tokens_seen": 505806848 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004276328986960883, + "loss": 1.532, + "theoretical_loss": 3.9384050761683884, + "tokens_seen": 505872384 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042762286860581743, + "loss": 1.5498, + "theoretical_loss": 3.938350631643168, + "tokens_seen": 505937920 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042761283851554667, + "loss": 1.4436, + "theoretical_loss": 3.9382961961442695, + "tokens_seen": 506003456 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042760280842527585, + "loss": 1.523, + "theoretical_loss": 3.93824176966903, + "tokens_seen": 506068992 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042759277833500503, + "loss": 1.5245, + "theoretical_loss": 3.9381873522147837, + "tokens_seen": 506134528 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004275827482447342, + "loss": 1.5387, + "theoretical_loss": 3.938132943778869, + "tokens_seen": 506200064 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004275727181544634, + "loss": 1.5179, + "theoretical_loss": 3.938078544358623, + "tokens_seen": 506265600 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004275626880641926, + "loss": 1.5384, + "theoretical_loss": 3.9380241539513854, + "tokens_seen": 506331136 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004275526579739218, + "loss": 1.4618, + "theoretical_loss": 3.9379697725544967, + "tokens_seen": 506396672 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042754262788365094, + "loss": 1.4705, + "theoretical_loss": 3.9379154001652985, + "tokens_seen": 506462208 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042753259779338017, + "loss": 1.5611, + "theoretical_loss": 3.9378610367811326, + "tokens_seen": 506527744 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004275225677031093, + "loss": 1.4519, + "theoretical_loss": 3.9378066823993443, + "tokens_seen": 506593280 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042751253761283853, + "loss": 1.4241, + "theoretical_loss": 3.937752337017278, + "tokens_seen": 506658816 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004275025075225677, + "loss": 1.495, + "theoretical_loss": 3.9376980006322784, + "tokens_seen": 506724352 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004274924774322969, + "loss": 1.4117, + "theoretical_loss": 3.937643673241695, + "tokens_seen": 506789888 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004274824473420261, + "loss": 1.471, + "theoretical_loss": 3.9375893548428738, + "tokens_seen": 506855424 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004274724172517553, + "loss": 1.5005, + "theoretical_loss": 3.9375350454331657, + "tokens_seen": 506920960 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042746238716148444, + "loss": 1.4998, + "theoretical_loss": 3.937480745009921, + "tokens_seen": 506986496 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004274523570712137, + "loss": 1.4592, + "theoretical_loss": 3.937426453570491, + "tokens_seen": 507052032 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004274423269809428, + "loss": 1.5097, + "theoretical_loss": 3.9373721711122287, + "tokens_seen": 507117568 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042743229689067204, + "loss": 1.3785, + "theoretical_loss": 3.937317897632488, + "tokens_seen": 507183104 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004274222668004012, + "loss": 1.4477, + "theoretical_loss": 3.9372636331286244, + "tokens_seen": 507248640 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004274122367101304, + "loss": 1.4754, + "theoretical_loss": 3.937209377597993, + "tokens_seen": 507314176 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004274022066198596, + "loss": 1.4442, + "theoretical_loss": 3.9371551310379527, + "tokens_seen": 507379712 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042739217652958876, + "loss": 1.4923, + "theoretical_loss": 3.93710089344586, + "tokens_seen": 507445248 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042738214643931794, + "loss": 1.4061, + "theoretical_loss": 3.937046664819076, + "tokens_seen": 507510784 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004273721163490472, + "loss": 1.3683, + "theoretical_loss": 3.93699244515496, + "tokens_seen": 507576320 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004273620862587763, + "loss": 1.4784, + "theoretical_loss": 3.936938234450875, + "tokens_seen": 507641856 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042735205616850554, + "loss": 1.3815, + "theoretical_loss": 3.936884032704183, + "tokens_seen": 507707392 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042734202607823467, + "loss": 1.48, + "theoretical_loss": 3.9368298399122486, + "tokens_seen": 507772928 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004273319959879639, + "loss": 1.4299, + "theoretical_loss": 3.9367756560724363, + "tokens_seen": 507838464 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 174398, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4068242311477661, + "objective/train/theoretical_loss": 3.936721481182113, + "objective/train/tokens_used": 507904000, + "theoretical_loss": 3.936721481182113, + "tokens_seen": 507904000 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004273219658976931, + "loss": 1.5053, + "theoretical_loss": 3.936721481182113, + "tokens_seen": 507904000 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042731193580742226, + "loss": 1.4827, + "theoretical_loss": 3.9366673152386458, + "tokens_seen": 507969536 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042730190571715145, + "loss": 1.5221, + "theoretical_loss": 3.9366131582394024, + "tokens_seen": 508035072 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004272918756268807, + "loss": 1.4639, + "theoretical_loss": 3.936559010181753, + "tokens_seen": 508100608 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004272818455366098, + "loss": 1.3114, + "theoretical_loss": 3.936504871063069, + "tokens_seen": 508166144 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042727181544633904, + "loss": 1.3844, + "theoretical_loss": 3.9364507408807206, + "tokens_seen": 508231680 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042726178535606817, + "loss": 1.5272, + "theoretical_loss": 3.936396619632082, + "tokens_seen": 508297216 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004272517552657974, + "loss": 1.4005, + "theoretical_loss": 3.936342507314527, + "tokens_seen": 508362752 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004272417251755266, + "loss": 1.5265, + "theoretical_loss": 3.9362884039254302, + "tokens_seen": 508428288 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042723169508525577, + "loss": 1.4711, + "theoretical_loss": 3.9362343094621677, + "tokens_seen": 508493824 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042722166499498495, + "loss": 1.5079, + "theoretical_loss": 3.936180223922118, + "tokens_seen": 508559360 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042721163490471413, + "loss": 1.3688, + "theoretical_loss": 3.936126147302658, + "tokens_seen": 508624896 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004272016048144433, + "loss": 1.4135, + "theoretical_loss": 3.9360720796011686, + "tokens_seen": 508690432 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042719157472417255, + "loss": 1.466, + "theoretical_loss": 3.93601802081503, + "tokens_seen": 508755968 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004271815446339017, + "loss": 1.3919, + "theoretical_loss": 3.9359639709416236, + "tokens_seen": 508821504 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004271715145436309, + "loss": 1.5003, + "theoretical_loss": 3.9359099299783327, + "tokens_seen": 508887040 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004271614844533601, + "loss": 1.4972, + "theoretical_loss": 3.935855897922541, + "tokens_seen": 508952576 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042715145436308927, + "loss": 1.4076, + "theoretical_loss": 3.935801874771634, + "tokens_seen": 509018112 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004271414242728185, + "loss": 1.4449, + "theoretical_loss": 3.935747860522997, + "tokens_seen": 509083648 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042713139418254763, + "loss": 1.4176, + "theoretical_loss": 3.9356938551740184, + "tokens_seen": 509149184 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042712136409227687, + "loss": 1.4665, + "theoretical_loss": 3.9356398587220864, + "tokens_seen": 509214720 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042711133400200605, + "loss": 1.3977, + "theoretical_loss": 3.9355858711645895, + "tokens_seen": 509280256 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042710130391173523, + "loss": 1.453, + "theoretical_loss": 3.9355318924989193, + "tokens_seen": 509345792 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004270912738214644, + "loss": 1.4174, + "theoretical_loss": 3.9354779227224674, + "tokens_seen": 509411328 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004270812437311936, + "loss": 1.4806, + "theoretical_loss": 3.935423961832626, + "tokens_seen": 509476864 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004270712136409228, + "loss": 1.4596, + "theoretical_loss": 3.93537000982679, + "tokens_seen": 509542400 + }, + { + "epoch": 0.15, + "learning_rate": 0.000427061183550652, + "loss": 1.4359, + "theoretical_loss": 3.9353160667023532, + "tokens_seen": 509607936 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042705115346038114, + "loss": 1.4665, + "theoretical_loss": 3.935262132456713, + "tokens_seen": 509673472 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042704112337011037, + "loss": 1.5252, + "theoretical_loss": 3.9352082070872654, + "tokens_seen": 509739008 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004270310932798395, + "loss": 1.2973, + "theoretical_loss": 3.9351542905914094, + "tokens_seen": 509804544 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042702106318956873, + "loss": 1.3909, + "theoretical_loss": 3.935100382966544, + "tokens_seen": 509870080 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004270110330992979, + "loss": 1.5289, + "theoretical_loss": 3.93504648421007, + "tokens_seen": 509935616 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004270010030090271, + "loss": 1.5297, + "theoretical_loss": 3.9349925943193886, + "tokens_seen": 510001152 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004269909729187563, + "loss": 1.4409, + "theoretical_loss": 3.934938713291903, + "tokens_seen": 510066688 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004269809428284855, + "loss": 1.5754, + "theoretical_loss": 3.934884841125017, + "tokens_seen": 510132224 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042697091273821464, + "loss": 1.4659, + "theoretical_loss": 3.9348309778161346, + "tokens_seen": 510197760 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004269608826479439, + "loss": 1.4079, + "theoretical_loss": 3.9347771233626627, + "tokens_seen": 510263296 + }, + { + "epoch": 0.15, + "learning_rate": 0.000426950852557673, + "loss": 1.4398, + "theoretical_loss": 3.934723277762008, + "tokens_seen": 510328832 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042694082246740224, + "loss": 1.5611, + "theoretical_loss": 3.934669441011579, + "tokens_seen": 510394368 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004269307923771314, + "loss": 1.3974, + "theoretical_loss": 3.934615613108784, + "tokens_seen": 510459904 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004269207622868606, + "loss": 1.2956, + "theoretical_loss": 3.9345617940510342, + "tokens_seen": 510525440 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004269107321965898, + "loss": 1.3955, + "theoretical_loss": 3.934507983835741, + "tokens_seen": 510590976 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042690070210631896, + "loss": 1.44, + "theoretical_loss": 3.9344541824603163, + "tokens_seen": 510656512 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042689067201604814, + "loss": 1.3618, + "theoretical_loss": 3.934400389922174, + "tokens_seen": 510722048 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004268806419257774, + "loss": 1.4783, + "theoretical_loss": 3.934346606218729, + "tokens_seen": 510787584 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004268706118355065, + "loss": 1.4975, + "theoretical_loss": 3.9342928313473964, + "tokens_seen": 510853120 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042686058174523574, + "loss": 1.4543, + "theoretical_loss": 3.9342390653055945, + "tokens_seen": 510918656 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042685055165496487, + "loss": 1.4959, + "theoretical_loss": 3.93418530809074, + "tokens_seen": 510984192 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004268405215646941, + "loss": 1.4467, + "theoretical_loss": 3.9341315597002517, + "tokens_seen": 511049728 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004268304914744233, + "loss": 1.43, + "theoretical_loss": 3.934077820131551, + "tokens_seen": 511115264 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 175657, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4971660375595093, + "objective/train/theoretical_loss": 3.934024089382058, + "objective/train/tokens_used": 511180800, + "theoretical_loss": 3.934024089382058, + "tokens_seen": 511180800 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042682046138415246, + "loss": 1.4639, + "theoretical_loss": 3.934024089382058, + "tokens_seen": 511180800 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042681043129388165, + "loss": 1.4565, + "theoretical_loss": 3.933970367449195, + "tokens_seen": 511246336 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004268004012036109, + "loss": 1.4844, + "theoretical_loss": 3.9339166543303863, + "tokens_seen": 511311872 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042679037111334, + "loss": 1.4788, + "theoretical_loss": 3.9338629500230553, + "tokens_seen": 511377408 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042678034102306924, + "loss": 1.4746, + "theoretical_loss": 3.9338092545246286, + "tokens_seen": 511442944 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042677031093279837, + "loss": 1.4063, + "theoretical_loss": 3.933755567832532, + "tokens_seen": 511508480 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004267602808425276, + "loss": 1.4541, + "theoretical_loss": 3.933701889944194, + "tokens_seen": 511574016 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004267502507522568, + "loss": 1.4293, + "theoretical_loss": 3.933648220857042, + "tokens_seen": 511639552 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042674022066198597, + "loss": 1.5567, + "theoretical_loss": 3.9335945605685074, + "tokens_seen": 511705088 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042673019057171515, + "loss": 1.5138, + "theoretical_loss": 3.9335409090760196, + "tokens_seen": 511770624 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042672016048144433, + "loss": 1.4455, + "theoretical_loss": 3.933487266377012, + "tokens_seen": 511836160 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004267101303911735, + "loss": 1.5262, + "theoretical_loss": 3.933433632468917, + "tokens_seen": 511901696 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042670010030090275, + "loss": 1.4404, + "theoretical_loss": 3.9333800073491694, + "tokens_seen": 511967232 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004266900702106319, + "loss": 1.4487, + "theoretical_loss": 3.9333263910152034, + "tokens_seen": 512032768 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004266800401203611, + "loss": 1.4404, + "theoretical_loss": 3.9332727834644565, + "tokens_seen": 512098304 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042667001003009024, + "loss": 1.4111, + "theoretical_loss": 3.933219184694365, + "tokens_seen": 512163840 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042665997993981947, + "loss": 1.4921, + "theoretical_loss": 3.933165594702368, + "tokens_seen": 512229376 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042664994984954865, + "loss": 1.4718, + "theoretical_loss": 3.9331120134859052, + "tokens_seen": 512294912 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042663991975927783, + "loss": 1.4565, + "theoretical_loss": 3.9330584410424168, + "tokens_seen": 512360448 + }, + { + "epoch": 0.16, + "learning_rate": 0.000426629889669007, + "loss": 1.5603, + "theoretical_loss": 3.9330048773693447, + "tokens_seen": 512425984 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042661985957873625, + "loss": 1.5566, + "theoretical_loss": 3.932951322464132, + "tokens_seen": 512491520 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004266098294884654, + "loss": 1.3523, + "theoretical_loss": 3.9328977763242223, + "tokens_seen": 512557056 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004265997993981946, + "loss": 1.4835, + "theoretical_loss": 3.93284423894706, + "tokens_seen": 512622592 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042658976930792374, + "loss": 1.4736, + "theoretical_loss": 3.932790710330092, + "tokens_seen": 512688128 + }, + { + "epoch": 0.16, + "learning_rate": 0.000426579739217653, + "loss": 1.513, + "theoretical_loss": 3.932737190470765, + "tokens_seen": 512753664 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042656970912738216, + "loss": 1.4857, + "theoretical_loss": 3.9326836793665265, + "tokens_seen": 512819200 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042655967903711134, + "loss": 1.5444, + "theoretical_loss": 3.9326301770148264, + "tokens_seen": 512884736 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004265496489468405, + "loss": 1.4634, + "theoretical_loss": 3.9325766834131155, + "tokens_seen": 512950272 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004265396188565697, + "loss": 1.4324, + "theoretical_loss": 3.932523198558844, + "tokens_seen": 513015808 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004265295887662989, + "loss": 1.5015, + "theoretical_loss": 3.932469722449464, + "tokens_seen": 513081344 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004265195586760281, + "loss": 1.4949, + "theoretical_loss": 3.932416255082431, + "tokens_seen": 513146880 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042650952858575724, + "loss": 1.4759, + "theoretical_loss": 3.932362796455198, + "tokens_seen": 513212416 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004264994984954865, + "loss": 1.549, + "theoretical_loss": 3.932309346565221, + "tokens_seen": 513277952 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004264894684052156, + "loss": 1.4055, + "theoretical_loss": 3.9322559054099564, + "tokens_seen": 513343488 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042647943831494484, + "loss": 1.497, + "theoretical_loss": 3.932202472986862, + "tokens_seen": 513409024 + }, + { + "epoch": 0.16, + "learning_rate": 0.000426469408224674, + "loss": 1.4568, + "theoretical_loss": 3.932149049293397, + "tokens_seen": 513474560 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004264593781344032, + "loss": 1.3982, + "theoretical_loss": 3.9320956343270206, + "tokens_seen": 513540096 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004264493480441324, + "loss": 1.5503, + "theoretical_loss": 3.9320422280851943, + "tokens_seen": 513605632 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004264393179538616, + "loss": 1.5649, + "theoretical_loss": 3.9319888305653805, + "tokens_seen": 513671168 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042642928786359075, + "loss": 1.4002, + "theoretical_loss": 3.9319354417650416, + "tokens_seen": 513736704 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042641925777332, + "loss": 1.5328, + "theoretical_loss": 3.931882061681641, + "tokens_seen": 513802240 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042640922768304916, + "loss": 1.5008, + "theoretical_loss": 3.9318286903126456, + "tokens_seen": 513867776 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042639919759277834, + "loss": 1.4885, + "theoretical_loss": 3.93177532765552, + "tokens_seen": 513933312 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004263891675025076, + "loss": 1.4703, + "theoretical_loss": 3.9317219737077327, + "tokens_seen": 513998848 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004263791374122367, + "loss": 1.468, + "theoretical_loss": 3.9316686284667517, + "tokens_seen": 514064384 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042636910732196594, + "loss": 1.4505, + "theoretical_loss": 3.9316152919300458, + "tokens_seen": 514129920 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042635907723169507, + "loss": 1.4414, + "theoretical_loss": 3.9315619640950863, + "tokens_seen": 514195456 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004263490471414243, + "loss": 1.5672, + "theoretical_loss": 3.9315086449593446, + "tokens_seen": 514260992 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004263390170511535, + "loss": 1.4595, + "theoretical_loss": 3.931455334520293, + "tokens_seen": 514326528 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042632898696088266, + "loss": 1.4093, + "theoretical_loss": 3.931402032775405, + "tokens_seen": 514392064 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 176665, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3710360527038574, + "objective/train/theoretical_loss": 3.9313487397221554, + "objective/train/tokens_used": 514457600, + "theoretical_loss": 3.9313487397221554, + "tokens_seen": 514457600 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042631895687061185, + "loss": 1.529, + "theoretical_loss": 3.9313487397221554, + "tokens_seen": 514457600 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004263089267803411, + "loss": 1.5396, + "theoretical_loss": 3.9312954553580206, + "tokens_seen": 514523136 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004262988966900702, + "loss": 1.3503, + "theoretical_loss": 3.931242179680477, + "tokens_seen": 514588672 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042628886659979944, + "loss": 1.5062, + "theoretical_loss": 3.931188912687002, + "tokens_seen": 514654208 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042627883650952857, + "loss": 1.4691, + "theoretical_loss": 3.931135654375075, + "tokens_seen": 514719744 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004262688064192578, + "loss": 1.5404, + "theoretical_loss": 3.9310824047421757, + "tokens_seen": 514785280 + }, + { + "epoch": 0.16, + "learning_rate": 0.000426258776328987, + "loss": 1.4454, + "theoretical_loss": 3.9310291637857855, + "tokens_seen": 514850816 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042624874623871617, + "loss": 1.5152, + "theoretical_loss": 3.930975931503386, + "tokens_seen": 514916352 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042623871614844535, + "loss": 1.4366, + "theoretical_loss": 3.930922707892461, + "tokens_seen": 514981888 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042622868605817453, + "loss": 1.4005, + "theoretical_loss": 3.930869492950494, + "tokens_seen": 515047424 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004262186559679037, + "loss": 1.4043, + "theoretical_loss": 3.9308162866749705, + "tokens_seen": 515112960 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042620862587763295, + "loss": 1.5106, + "theoretical_loss": 3.930763089063377, + "tokens_seen": 515178496 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004261985957873621, + "loss": 1.4419, + "theoretical_loss": 3.9307099001132, + "tokens_seen": 515244032 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004261885656970913, + "loss": 1.5983, + "theoretical_loss": 3.930656719821929, + "tokens_seen": 515309568 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042617853560682044, + "loss": 1.574, + "theoretical_loss": 3.9306035481870527, + "tokens_seen": 515375104 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042616850551654967, + "loss": 1.456, + "theoretical_loss": 3.9305503852060615, + "tokens_seen": 515440640 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042615847542627885, + "loss": 1.5701, + "theoretical_loss": 3.9304972308764476, + "tokens_seen": 515506176 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042614844533600803, + "loss": 1.4533, + "theoretical_loss": 3.9304440851957025, + "tokens_seen": 515571712 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004261384152457372, + "loss": 1.4677, + "theoretical_loss": 3.930390948161321, + "tokens_seen": 515637248 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042612838515546645, + "loss": 1.5041, + "theoretical_loss": 3.9303378197707968, + "tokens_seen": 515702784 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004261183550651956, + "loss": 1.3963, + "theoretical_loss": 3.9302847000216263, + "tokens_seen": 515768320 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004261083249749248, + "loss": 1.4592, + "theoretical_loss": 3.9302315889113055, + "tokens_seen": 515833856 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042609829488465394, + "loss": 1.3411, + "theoretical_loss": 3.930178486437333, + "tokens_seen": 515899392 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004260882647943832, + "loss": 1.5275, + "theoretical_loss": 3.930125392597207, + "tokens_seen": 515964928 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042607823470411236, + "loss": 1.5032, + "theoretical_loss": 3.9300723073884276, + "tokens_seen": 516030464 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042606820461384154, + "loss": 1.5437, + "theoretical_loss": 3.9300192308084956, + "tokens_seen": 516096000 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004260581745235707, + "loss": 1.5416, + "theoretical_loss": 3.929966162854913, + "tokens_seen": 516161536 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004260481444332999, + "loss": 1.5028, + "theoretical_loss": 3.929913103525183, + "tokens_seen": 516227072 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004260381143430291, + "loss": 1.4876, + "theoretical_loss": 3.929860052816809, + "tokens_seen": 516292608 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004260280842527583, + "loss": 1.4015, + "theoretical_loss": 3.9298070107272967, + "tokens_seen": 516358144 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042601805416248744, + "loss": 1.4075, + "theoretical_loss": 3.929753977254152, + "tokens_seen": 516423680 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004260080240722167, + "loss": 1.4145, + "theoretical_loss": 3.929700952394882, + "tokens_seen": 516489216 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004259979939819458, + "loss": 1.3287, + "theoretical_loss": 3.9296479361469947, + "tokens_seen": 516554752 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042598796389167504, + "loss": 1.3649, + "theoretical_loss": 3.929594928508, + "tokens_seen": 516620288 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004259779338014042, + "loss": 1.4403, + "theoretical_loss": 3.9295419294754077, + "tokens_seen": 516685824 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004259679037111334, + "loss": 1.4767, + "theoretical_loss": 3.9294889390467285, + "tokens_seen": 516751360 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004259578736208626, + "loss": 1.4461, + "theoretical_loss": 3.929435957219476, + "tokens_seen": 516816896 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004259478435305918, + "loss": 1.627, + "theoretical_loss": 3.9293829839911623, + "tokens_seen": 516882432 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042593781344032095, + "loss": 1.3755, + "theoretical_loss": 3.9293300193593024, + "tokens_seen": 516947968 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004259277833500502, + "loss": 1.4698, + "theoretical_loss": 3.929277063321412, + "tokens_seen": 517013504 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004259177532597793, + "loss": 1.4983, + "theoretical_loss": 3.929224115875007, + "tokens_seen": 517079040 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042590772316950854, + "loss": 1.4756, + "theoretical_loss": 3.929171177017605, + "tokens_seen": 517144576 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004258976930792377, + "loss": 1.5008, + "theoretical_loss": 3.9291182467467247, + "tokens_seen": 517210112 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004258876629889669, + "loss": 1.3461, + "theoretical_loss": 3.9290653250598857, + "tokens_seen": 517275648 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004258776328986961, + "loss": 1.5438, + "theoretical_loss": 3.9290124119546084, + "tokens_seen": 517341184 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042586760280842527, + "loss": 1.4454, + "theoretical_loss": 3.9289595074284143, + "tokens_seen": 517406720 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042585757271815445, + "loss": 1.5483, + "theoretical_loss": 3.9289066114788267, + "tokens_seen": 517472256 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004258475426278837, + "loss": 1.4947, + "theoretical_loss": 3.9288537241033685, + "tokens_seen": 517537792 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004258375125376128, + "loss": 1.4899, + "theoretical_loss": 3.9288008452995644, + "tokens_seen": 517603328 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042582748244734205, + "loss": 1.499, + "theoretical_loss": 3.9287479750649412, + "tokens_seen": 517668864 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 177884, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5477997064590454, + "objective/train/theoretical_loss": 3.9286951133970245, + "objective/train/tokens_used": 517734400, + "theoretical_loss": 3.9286951133970245, + "tokens_seen": 517734400 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004258174523570712, + "loss": 1.4739, + "theoretical_loss": 3.9286951133970245, + "tokens_seen": 517734400 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004258074222668004, + "loss": 1.5289, + "theoretical_loss": 3.9286422602933424, + "tokens_seen": 517799936 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004257973921765296, + "loss": 1.5009, + "theoretical_loss": 3.928589415751424, + "tokens_seen": 517865472 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042578736208625877, + "loss": 1.4031, + "theoretical_loss": 3.9285365797687986, + "tokens_seen": 517931008 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042577733199598795, + "loss": 1.4634, + "theoretical_loss": 3.9284837523429976, + "tokens_seen": 517996544 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004257673019057172, + "loss": 1.4946, + "theoretical_loss": 3.9284309334715526, + "tokens_seen": 518062080 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004257572718154463, + "loss": 1.5113, + "theoretical_loss": 3.9283781231519965, + "tokens_seen": 518127616 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042574724172517555, + "loss": 1.4307, + "theoretical_loss": 3.9283253213818634, + "tokens_seen": 518193152 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004257372116349047, + "loss": 1.4629, + "theoretical_loss": 3.9282725281586885, + "tokens_seen": 518258688 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004257271815446339, + "loss": 1.4736, + "theoretical_loss": 3.928219743480007, + "tokens_seen": 518324224 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004257171514543631, + "loss": 1.4501, + "theoretical_loss": 3.9281669673433566, + "tokens_seen": 518389760 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004257071213640923, + "loss": 1.4918, + "theoretical_loss": 3.9281141997462754, + "tokens_seen": 518455296 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042569709127382145, + "loss": 1.5014, + "theoretical_loss": 3.9280614406863013, + "tokens_seen": 518520832 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042568706118355064, + "loss": 1.5419, + "theoretical_loss": 3.9280086901609756, + "tokens_seen": 518586368 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004256770310932798, + "loss": 1.4655, + "theoretical_loss": 3.927955948167839, + "tokens_seen": 518651904 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042566700100300905, + "loss": 1.6034, + "theoretical_loss": 3.9279032147044335, + "tokens_seen": 518717440 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042565697091273823, + "loss": 1.5666, + "theoretical_loss": 3.9278504897683018, + "tokens_seen": 518782976 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004256469408224674, + "loss": 1.5053, + "theoretical_loss": 3.927797773356989, + "tokens_seen": 518848512 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042563691073219665, + "loss": 1.544, + "theoretical_loss": 3.9277450654680397, + "tokens_seen": 518914048 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004256268806419258, + "loss": 1.5494, + "theoretical_loss": 3.9276923660990004, + "tokens_seen": 518979584 + }, + { + "epoch": 0.16, + "learning_rate": 0.000425616850551655, + "loss": 1.4362, + "theoretical_loss": 3.9276396752474176, + "tokens_seen": 519045120 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042560682046138414, + "loss": 1.4832, + "theoretical_loss": 3.9275869929108405, + "tokens_seen": 519110656 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004255967903711134, + "loss": 1.4638, + "theoretical_loss": 3.927534319086817, + "tokens_seen": 519176192 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042558676028084256, + "loss": 1.5458, + "theoretical_loss": 3.9274816537728983, + "tokens_seen": 519241728 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042557673019057174, + "loss": 1.4705, + "theoretical_loss": 3.9274289969666354, + "tokens_seen": 519307264 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004255667001003009, + "loss": 1.4991, + "theoretical_loss": 3.9273763486655806, + "tokens_seen": 519372800 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004255566700100301, + "loss": 1.4606, + "theoretical_loss": 3.9273237088672874, + "tokens_seen": 519438336 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004255466399197593, + "loss": 1.5372, + "theoretical_loss": 3.92727107756931, + "tokens_seen": 519503872 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004255366098294885, + "loss": 1.474, + "theoretical_loss": 3.927218454769203, + "tokens_seen": 519569408 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042552657973921764, + "loss": 1.5217, + "theoretical_loss": 3.9271658404645233, + "tokens_seen": 519634944 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004255165496489469, + "loss": 1.4172, + "theoretical_loss": 3.9271132346528286, + "tokens_seen": 519700480 + }, + { + "epoch": 0.16, + "learning_rate": 0.000425506519558676, + "loss": 1.4527, + "theoretical_loss": 3.9270606373316763, + "tokens_seen": 519766016 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042549648946840524, + "loss": 1.4735, + "theoretical_loss": 3.9270080484986267, + "tokens_seen": 519831552 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004254864593781344, + "loss": 1.4755, + "theoretical_loss": 3.9269554681512395, + "tokens_seen": 519897088 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004254764292878636, + "loss": 1.4849, + "theoretical_loss": 3.926902896287076, + "tokens_seen": 519962624 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004254663991975928, + "loss": 1.5695, + "theoretical_loss": 3.9268503329036992, + "tokens_seen": 520028160 + }, + { + "epoch": 0.16, + "learning_rate": 0.000425456369107322, + "loss": 1.3956, + "theoretical_loss": 3.926797777998672, + "tokens_seen": 520093696 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042544633901705115, + "loss": 1.3641, + "theoretical_loss": 3.9267452315695586, + "tokens_seen": 520159232 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004254363089267804, + "loss": 1.4952, + "theoretical_loss": 3.9266926936139246, + "tokens_seen": 520224768 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004254262788365095, + "loss": 1.4401, + "theoretical_loss": 3.9266401641293367, + "tokens_seen": 520290304 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042541624874623874, + "loss": 1.3831, + "theoretical_loss": 3.926587643113362, + "tokens_seen": 520355840 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004254062186559679, + "loss": 1.4556, + "theoretical_loss": 3.9265351305635687, + "tokens_seen": 520421376 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004253961885656971, + "loss": 1.4621, + "theoretical_loss": 3.9264826264775268, + "tokens_seen": 520486912 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004253861584754263, + "loss": 1.4356, + "theoretical_loss": 3.9264301308528067, + "tokens_seen": 520552448 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042537612838515547, + "loss": 1.4265, + "theoretical_loss": 3.926377643686979, + "tokens_seen": 520617984 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042536609829488465, + "loss": 1.5098, + "theoretical_loss": 3.926325164977616, + "tokens_seen": 520683520 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004253560682046139, + "loss": 1.4906, + "theoretical_loss": 3.9262726947222926, + "tokens_seen": 520749056 + }, + { + "epoch": 0.16, + "learning_rate": 0.000425346038114343, + "loss": 1.453, + "theoretical_loss": 3.926220232918582, + "tokens_seen": 520814592 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042533600802407225, + "loss": 1.4303, + "theoretical_loss": 3.92616777956406, + "tokens_seen": 520880128 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004253259779338014, + "loss": 1.3688, + "theoretical_loss": 3.926115334656303, + "tokens_seen": 520945664 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 179053, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5376003980636597, + "objective/train/theoretical_loss": 3.926062898192888, + "objective/train/tokens_used": 521011200, + "theoretical_loss": 3.926062898192888, + "tokens_seen": 521011200 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004253159478435306, + "loss": 1.5032, + "theoretical_loss": 3.926062898192888, + "tokens_seen": 521011200 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004253059177532598, + "loss": 1.5308, + "theoretical_loss": 3.926010470171394, + "tokens_seen": 521076736 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042529588766298897, + "loss": 1.5577, + "theoretical_loss": 3.9259580505894007, + "tokens_seen": 521142272 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042528585757271815, + "loss": 1.5005, + "theoretical_loss": 3.925905639444488, + "tokens_seen": 521207808 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004252758274824474, + "loss": 1.5245, + "theoretical_loss": 3.9258532367342367, + "tokens_seen": 521273344 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004252657973921765, + "loss": 1.4814, + "theoretical_loss": 3.9258008424562307, + "tokens_seen": 521338880 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042525576730190575, + "loss": 1.5331, + "theoretical_loss": 3.9257484566080523, + "tokens_seen": 521404416 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004252457372116349, + "loss": 1.4366, + "theoretical_loss": 3.9256960791872855, + "tokens_seen": 521469952 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004252357071213641, + "loss": 1.4675, + "theoretical_loss": 3.925643710191517, + "tokens_seen": 521535488 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004252256770310933, + "loss": 1.3384, + "theoretical_loss": 3.9255913496183332, + "tokens_seen": 521601024 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004252156469408225, + "loss": 1.3877, + "theoretical_loss": 3.9255389974653196, + "tokens_seen": 521666560 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042520561685055165, + "loss": 1.4827, + "theoretical_loss": 3.9254866537300668, + "tokens_seen": 521732096 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042519558676028084, + "loss": 1.5247, + "theoretical_loss": 3.925434318410163, + "tokens_seen": 521797632 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042518555667001, + "loss": 1.5475, + "theoretical_loss": 3.9253819915031993, + "tokens_seen": 521863168 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042517552657973925, + "loss": 1.4727, + "theoretical_loss": 3.9253296730067664, + "tokens_seen": 521928704 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004251654964894684, + "loss": 1.4358, + "theoretical_loss": 3.9252773629184565, + "tokens_seen": 521994240 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004251554663991976, + "loss": 1.4433, + "theoretical_loss": 3.925225061235863, + "tokens_seen": 522059776 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004251454363089268, + "loss": 1.4185, + "theoretical_loss": 3.925172767956581, + "tokens_seen": 522125312 + }, + { + "epoch": 0.16, + "learning_rate": 0.000425135406218656, + "loss": 1.4218, + "theoretical_loss": 3.9251204830782056, + "tokens_seen": 522190848 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042512537612838516, + "loss": 1.438, + "theoretical_loss": 3.925068206598332, + "tokens_seen": 522256384 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042511534603811434, + "loss": 1.5, + "theoretical_loss": 3.9250159385145595, + "tokens_seen": 522321920 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004251053159478435, + "loss": 1.5386, + "theoretical_loss": 3.9249636788244846, + "tokens_seen": 522387456 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042509528585757276, + "loss": 1.5408, + "theoretical_loss": 3.924911427525708, + "tokens_seen": 522452992 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004250852557673019, + "loss": 1.5265, + "theoretical_loss": 3.9248591846158285, + "tokens_seen": 522518528 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004250752256770311, + "loss": 1.4071, + "theoretical_loss": 3.9248069500924485, + "tokens_seen": 522584064 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042506519558676024, + "loss": 1.4519, + "theoretical_loss": 3.9247547239531695, + "tokens_seen": 522649600 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004250551654964895, + "loss": 1.6396, + "theoretical_loss": 3.924702506195595, + "tokens_seen": 522715136 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042504513540621866, + "loss": 1.5353, + "theoretical_loss": 3.9246502968173296, + "tokens_seen": 522780672 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042503510531594784, + "loss": 1.488, + "theoretical_loss": 3.9245980958159787, + "tokens_seen": 522846208 + }, + { + "epoch": 0.16, + "learning_rate": 0.000425025075225677, + "loss": 1.4779, + "theoretical_loss": 3.924545903189147, + "tokens_seen": 522911744 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004250150451354062, + "loss": 1.444, + "theoretical_loss": 3.9244937189344435, + "tokens_seen": 522977280 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004250050150451354, + "loss": 1.4976, + "theoretical_loss": 3.9244415430494755, + "tokens_seen": 523042816 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004249949849548646, + "loss": 1.4548, + "theoretical_loss": 3.9243893755318515, + "tokens_seen": 523108352 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042498495486459375, + "loss": 1.3799, + "theoretical_loss": 3.924337216379183, + "tokens_seen": 523173888 + }, + { + "epoch": 0.16, + "learning_rate": 0.000424974924774323, + "loss": 1.5618, + "theoretical_loss": 3.92428506558908, + "tokens_seen": 523239424 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042496489468405216, + "loss": 1.4193, + "theoretical_loss": 3.924232923159155, + "tokens_seen": 523304960 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042495486459378135, + "loss": 1.5005, + "theoretical_loss": 3.9241807890870213, + "tokens_seen": 523370496 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004249448345035105, + "loss": 1.465, + "theoretical_loss": 3.9241286633702925, + "tokens_seen": 523436032 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004249348044132397, + "loss": 1.4316, + "theoretical_loss": 3.9240765460065834, + "tokens_seen": 523501568 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042492477432296894, + "loss": 1.4222, + "theoretical_loss": 3.924024436993511, + "tokens_seen": 523567104 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004249147442326981, + "loss": 1.3812, + "theoretical_loss": 3.9239723363286916, + "tokens_seen": 523632640 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004249047141424273, + "loss": 1.3639, + "theoretical_loss": 3.923920244009743, + "tokens_seen": 523698176 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004248946840521565, + "loss": 1.4965, + "theoretical_loss": 3.9238681600342846, + "tokens_seen": 523763712 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042488465396188567, + "loss": 1.5323, + "theoretical_loss": 3.9238160843999363, + "tokens_seen": 523829248 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042487462387161485, + "loss": 1.3982, + "theoretical_loss": 3.923764017104318, + "tokens_seen": 523894784 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004248645937813441, + "loss": 1.4535, + "theoretical_loss": 3.9237119581450535, + "tokens_seen": 523960320 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004248545636910732, + "loss": 1.3963, + "theoretical_loss": 3.923659907519764, + "tokens_seen": 524025856 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042484453360080245, + "loss": 1.4633, + "theoretical_loss": 3.9236078652260735, + "tokens_seen": 524091392 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004248345035105316, + "loss": 1.3963, + "theoretical_loss": 3.9235558312616075, + "tokens_seen": 524156928 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004248244734202608, + "loss": 1.4596, + "theoretical_loss": 3.9235038056239917, + "tokens_seen": 524222464 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 179979, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.488440990447998, + "objective/train/theoretical_loss": 3.9234517883108517, + "objective/train/tokens_used": 524288000, + "theoretical_loss": 3.9234517883108517, + "tokens_seen": 524288000 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042481444332999, + "loss": 1.4425, + "theoretical_loss": 3.9234517883108517, + "tokens_seen": 524288000 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042480441323971917, + "loss": 1.4791, + "theoretical_loss": 3.9233997793198165, + "tokens_seen": 524353536 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042479438314944835, + "loss": 1.4292, + "theoretical_loss": 3.9233477786485143, + "tokens_seen": 524419072 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004247843530591776, + "loss": 1.458, + "theoretical_loss": 3.923295786294575, + "tokens_seen": 524484608 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004247743229689067, + "loss": 1.5088, + "theoretical_loss": 3.9232438022556293, + "tokens_seen": 524550144 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042476429287863595, + "loss": 1.517, + "theoretical_loss": 3.923191826529308, + "tokens_seen": 524615680 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004247542627883651, + "loss": 1.4437, + "theoretical_loss": 3.9231398591132445, + "tokens_seen": 524681216 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004247442326980943, + "loss": 1.3896, + "theoretical_loss": 3.9230879000050716, + "tokens_seen": 524746752 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004247342026078235, + "loss": 1.4239, + "theoretical_loss": 3.9230359492024247, + "tokens_seen": 524812288 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004247241725175527, + "loss": 1.3813, + "theoretical_loss": 3.9229840067029382, + "tokens_seen": 524877824 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042471414242728186, + "loss": 1.4488, + "theoretical_loss": 3.92293207250425, + "tokens_seen": 524943360 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042470411233701104, + "loss": 1.4089, + "theoretical_loss": 3.9228801466039958, + "tokens_seen": 525008896 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004246940822467402, + "loss": 1.4595, + "theoretical_loss": 3.9228282289998155, + "tokens_seen": 525074432 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042468405215646945, + "loss": 1.3173, + "theoretical_loss": 3.9227763196893477, + "tokens_seen": 525139968 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004246740220661986, + "loss": 1.5575, + "theoretical_loss": 3.9227244186702332, + "tokens_seen": 525205504 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004246639919759278, + "loss": 1.474, + "theoretical_loss": 3.9226725259401123, + "tokens_seen": 525271040 + }, + { + "epoch": 0.16, + "learning_rate": 0.000424653961885657, + "loss": 1.4396, + "theoretical_loss": 3.9226206414966285, + "tokens_seen": 525336576 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004246439317953862, + "loss": 1.4276, + "theoretical_loss": 3.9225687653374237, + "tokens_seen": 525402112 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042463390170511536, + "loss": 1.4928, + "theoretical_loss": 3.922516897460143, + "tokens_seen": 525467648 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042462387161484454, + "loss": 1.5599, + "theoretical_loss": 3.9224650378624313, + "tokens_seen": 525533184 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004246138415245737, + "loss": 1.3533, + "theoretical_loss": 3.9224131865419345, + "tokens_seen": 525598720 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042460381143430296, + "loss": 1.3876, + "theoretical_loss": 3.9223613434963003, + "tokens_seen": 525664256 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004245937813440321, + "loss": 1.4543, + "theoretical_loss": 3.922309508723176, + "tokens_seen": 525729792 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004245837512537613, + "loss": 1.4126, + "theoretical_loss": 3.9222576822202107, + "tokens_seen": 525795328 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042457372116349044, + "loss": 1.4957, + "theoretical_loss": 3.922205863985055, + "tokens_seen": 525860864 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004245636910732197, + "loss": 1.4546, + "theoretical_loss": 3.9221540540153588, + "tokens_seen": 525926400 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042455366098294886, + "loss": 1.3849, + "theoretical_loss": 3.922102252308775, + "tokens_seen": 525991936 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042454363089267804, + "loss": 1.3569, + "theoretical_loss": 3.9220504588629552, + "tokens_seen": 526057472 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004245336008024072, + "loss": 1.4285, + "theoretical_loss": 3.9219986736755548, + "tokens_seen": 526123008 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004245235707121364, + "loss": 1.5887, + "theoretical_loss": 3.921946896744227, + "tokens_seen": 526188544 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004245135406218656, + "loss": 1.47, + "theoretical_loss": 3.9218951280666285, + "tokens_seen": 526254080 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004245035105315948, + "loss": 1.4843, + "theoretical_loss": 3.9218433676404154, + "tokens_seen": 526319616 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042449348044132395, + "loss": 1.4139, + "theoretical_loss": 3.9217916154632455, + "tokens_seen": 526385152 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004244834503510532, + "loss": 1.4562, + "theoretical_loss": 3.9217398715327776, + "tokens_seen": 526450688 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042447342026078236, + "loss": 1.5292, + "theoretical_loss": 3.9216881358466713, + "tokens_seen": 526516224 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042446339017051155, + "loss": 1.4503, + "theoretical_loss": 3.9216364084025868, + "tokens_seen": 526581760 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004244533600802407, + "loss": 1.5318, + "theoretical_loss": 3.921584689198185, + "tokens_seen": 526647296 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004244433299899699, + "loss": 1.406, + "theoretical_loss": 3.921532978231129, + "tokens_seen": 526712832 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004244332998996991, + "loss": 1.5636, + "theoretical_loss": 3.921481275499082, + "tokens_seen": 526778368 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004244232698094283, + "loss": 1.4093, + "theoretical_loss": 3.921429580999709, + "tokens_seen": 526843904 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042441323971915745, + "loss": 1.4766, + "theoretical_loss": 3.921377894730674, + "tokens_seen": 526909440 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004244032096288867, + "loss": 1.4165, + "theoretical_loss": 3.9213262166896445, + "tokens_seen": 526974976 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004243931795386158, + "loss": 1.4864, + "theoretical_loss": 3.9212745468742867, + "tokens_seen": 527040512 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042438314944834505, + "loss": 1.5659, + "theoretical_loss": 3.921222885282268, + "tokens_seen": 527106048 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042437311935807423, + "loss": 1.4701, + "theoretical_loss": 3.9211712319112597, + "tokens_seen": 527171584 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004243630892678034, + "loss": 1.3574, + "theoretical_loss": 3.92111958675893, + "tokens_seen": 527237120 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004243530591775326, + "loss": 1.4388, + "theoretical_loss": 3.9210679498229504, + "tokens_seen": 527302656 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004243430290872618, + "loss": 1.419, + "theoretical_loss": 3.9210163211009927, + "tokens_seen": 527368192 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042433299899699095, + "loss": 1.5271, + "theoretical_loss": 3.92096470059073, + "tokens_seen": 527433728 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004243229689067202, + "loss": 1.5947, + "theoretical_loss": 3.9209130882898364, + "tokens_seen": 527499264 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 181170, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4174742698669434, + "objective/train/theoretical_loss": 3.920861484195986, + "objective/train/tokens_used": 527564800, + "theoretical_loss": 3.920861484195986, + "tokens_seen": 527564800 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004243129388164493, + "loss": 1.4364, + "theoretical_loss": 3.920861484195986, + "tokens_seen": 527564800 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042430290872617855, + "loss": 1.436, + "theoretical_loss": 3.920809888306854, + "tokens_seen": 527630336 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042429287863590773, + "loss": 1.4885, + "theoretical_loss": 3.9207583006201183, + "tokens_seen": 527695872 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004242828485456369, + "loss": 1.5653, + "theoretical_loss": 3.920706721133456, + "tokens_seen": 527761408 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004242728184553661, + "loss": 1.4205, + "theoretical_loss": 3.9206551498445465, + "tokens_seen": 527826944 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004242627883650953, + "loss": 1.4502, + "theoretical_loss": 3.920603586751068, + "tokens_seen": 527892480 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042425275827482446, + "loss": 1.3903, + "theoretical_loss": 3.9205520318507006, + "tokens_seen": 527958016 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004242427281845537, + "loss": 1.4697, + "theoretical_loss": 3.9205004851411274, + "tokens_seen": 528023552 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004242326980942828, + "loss": 1.4916, + "theoretical_loss": 3.920448946620029, + "tokens_seen": 528089088 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042422266800401206, + "loss": 1.4662, + "theoretical_loss": 3.9203974162850903, + "tokens_seen": 528154624 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004242126379137412, + "loss": 1.4551, + "theoretical_loss": 3.920345894133994, + "tokens_seen": 528220160 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004242026078234704, + "loss": 1.4221, + "theoretical_loss": 3.9202943801644263, + "tokens_seen": 528285696 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004241925777331996, + "loss": 1.3546, + "theoretical_loss": 3.9202428743740727, + "tokens_seen": 528351232 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004241825476429288, + "loss": 1.5052, + "theoretical_loss": 3.92019137676062, + "tokens_seen": 528416768 + }, + { + "epoch": 0.16, + "learning_rate": 0.000424172517552658, + "loss": 1.4992, + "theoretical_loss": 3.920139887321757, + "tokens_seen": 528482304 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004241624874623872, + "loss": 1.4087, + "theoretical_loss": 3.9200884060551724, + "tokens_seen": 528547840 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004241524573721164, + "loss": 1.3584, + "theoretical_loss": 3.9200369329585563, + "tokens_seen": 528613376 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042414242728184556, + "loss": 1.5055, + "theoretical_loss": 3.9199854680295982, + "tokens_seen": 528678912 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042413239719157474, + "loss": 1.4719, + "theoretical_loss": 3.9199340112659913, + "tokens_seen": 528744448 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004241223671013039, + "loss": 1.5262, + "theoretical_loss": 3.9198825626654275, + "tokens_seen": 528809984 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042411233701103316, + "loss": 1.5593, + "theoretical_loss": 3.9198311222256, + "tokens_seen": 528875520 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004241023069207623, + "loss": 1.4928, + "theoretical_loss": 3.919779689944205, + "tokens_seen": 528941056 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004240922768304915, + "loss": 1.4252, + "theoretical_loss": 3.9197282658189363, + "tokens_seen": 529006592 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042408224674022065, + "loss": 1.3778, + "theoretical_loss": 3.919676849847491, + "tokens_seen": 529072128 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004240722166499499, + "loss": 1.4844, + "theoretical_loss": 3.9196254420275665, + "tokens_seen": 529137664 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042406218655967906, + "loss": 1.4163, + "theoretical_loss": 3.919574042356861, + "tokens_seen": 529203200 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042405215646940824, + "loss": 1.4436, + "theoretical_loss": 3.9195226508330734, + "tokens_seen": 529268736 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004240421263791374, + "loss": 1.4448, + "theoretical_loss": 3.919471267453905, + "tokens_seen": 529334272 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004240320962888666, + "loss": 1.5236, + "theoretical_loss": 3.919419892217055, + "tokens_seen": 529399808 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004240220661985958, + "loss": 1.4619, + "theoretical_loss": 3.919368525120227, + "tokens_seen": 529465344 + }, + { + "epoch": 0.16, + "learning_rate": 0.000424012036108325, + "loss": 1.4782, + "theoretical_loss": 3.9193171661611235, + "tokens_seen": 529530880 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042400200601805415, + "loss": 1.4909, + "theoretical_loss": 3.9192658153374484, + "tokens_seen": 529596416 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004239919759277834, + "loss": 1.5124, + "theoretical_loss": 3.9192144726469063, + "tokens_seen": 529661952 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042398194583751256, + "loss": 1.3853, + "theoretical_loss": 3.919163138087203, + "tokens_seen": 529727488 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042397191574724175, + "loss": 1.3788, + "theoretical_loss": 3.919111811656046, + "tokens_seen": 529793024 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004239618856569709, + "loss": 1.4678, + "theoretical_loss": 3.9190604933511417, + "tokens_seen": 529858560 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004239518555667001, + "loss": 1.466, + "theoretical_loss": 3.9190091831701994, + "tokens_seen": 529924096 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004239418254764293, + "loss": 1.4567, + "theoretical_loss": 3.918957881110928, + "tokens_seen": 529989632 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004239317953861585, + "loss": 1.4229, + "theoretical_loss": 3.9189065871710387, + "tokens_seen": 530055168 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042392176529588765, + "loss": 1.37, + "theoretical_loss": 3.918855301348242, + "tokens_seen": 530120704 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004239117352056169, + "loss": 1.4835, + "theoretical_loss": 3.9188040236402513, + "tokens_seen": 530186240 + }, + { + "epoch": 0.16, + "learning_rate": 0.000423901705115346, + "loss": 1.4008, + "theoretical_loss": 3.918752754044778, + "tokens_seen": 530251776 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042389167502507525, + "loss": 1.4868, + "theoretical_loss": 3.9187014925595385, + "tokens_seen": 530317312 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042388164493480443, + "loss": 1.4929, + "theoretical_loss": 3.918650239182246, + "tokens_seen": 530382848 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004238716148445336, + "loss": 1.4275, + "theoretical_loss": 3.918598993910617, + "tokens_seen": 530448384 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004238615847542628, + "loss": 1.4083, + "theoretical_loss": 3.9185477567423685, + "tokens_seen": 530513920 + }, + { + "epoch": 0.16, + "learning_rate": 0.000423851554663992, + "loss": 1.4674, + "theoretical_loss": 3.9184965276752184, + "tokens_seen": 530579456 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042384152457372115, + "loss": 1.4482, + "theoretical_loss": 3.9184453067068854, + "tokens_seen": 530644992 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004238314944834504, + "loss": 1.4656, + "theoretical_loss": 3.918394093835089, + "tokens_seen": 530710528 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004238214643931795, + "loss": 1.409, + "theoretical_loss": 3.91834288905755, + "tokens_seen": 530776064 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 182071, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4908123016357422, + "objective/train/theoretical_loss": 3.91829169237199, + "objective/train/tokens_used": 530841600, + "theoretical_loss": 3.91829169237199, + "tokens_seen": 530841600 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042381143430290875, + "loss": 1.503, + "theoretical_loss": 3.91829169237199, + "tokens_seen": 530841600 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042380140421263793, + "loss": 1.4631, + "theoretical_loss": 3.918240503776131, + "tokens_seen": 530907136 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004237913741223671, + "loss": 1.448, + "theoretical_loss": 3.9181893232676965, + "tokens_seen": 530972672 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004237813440320963, + "loss": 1.4021, + "theoretical_loss": 3.9181381508444106, + "tokens_seen": 531038208 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004237713139418255, + "loss": 1.4089, + "theoretical_loss": 3.918086986504, + "tokens_seen": 531103744 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042376128385155466, + "loss": 1.4157, + "theoretical_loss": 3.9180358302441887, + "tokens_seen": 531169280 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004237512537612839, + "loss": 1.4104, + "theoretical_loss": 3.9179846820627047, + "tokens_seen": 531234816 + }, + { + "epoch": 0.16, + "learning_rate": 0.000423741223671013, + "loss": 1.4718, + "theoretical_loss": 3.917933541957276, + "tokens_seen": 531300352 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042373119358074226, + "loss": 1.4816, + "theoretical_loss": 3.9178824099256317, + "tokens_seen": 531365888 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004237211634904714, + "loss": 1.4202, + "theoretical_loss": 3.9178312859655016, + "tokens_seen": 531431424 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004237111334002006, + "loss": 1.4612, + "theoretical_loss": 3.9177801700746153, + "tokens_seen": 531496960 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004237011033099298, + "loss": 1.5011, + "theoretical_loss": 3.9177290622507055, + "tokens_seen": 531562496 + }, + { + "epoch": 0.16, + "learning_rate": 0.000423691073219659, + "loss": 1.4986, + "theoretical_loss": 3.9176779624915055, + "tokens_seen": 531628032 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042368104312938816, + "loss": 1.5055, + "theoretical_loss": 3.917626870794747, + "tokens_seen": 531693568 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004236710130391174, + "loss": 1.4064, + "theoretical_loss": 3.9175757871581647, + "tokens_seen": 531759104 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004236609829488465, + "loss": 1.4256, + "theoretical_loss": 3.917524711579495, + "tokens_seen": 531824640 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042365095285857576, + "loss": 1.4156, + "theoretical_loss": 3.9174736440564732, + "tokens_seen": 531890176 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004236409227683049, + "loss": 1.5763, + "theoretical_loss": 3.917422584586837, + "tokens_seen": 531955712 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004236308926780341, + "loss": 1.5061, + "theoretical_loss": 3.9173715331683243, + "tokens_seen": 532021248 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004236208625877633, + "loss": 1.4596, + "theoretical_loss": 3.9173204897986738, + "tokens_seen": 532086784 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004236108324974925, + "loss": 1.4717, + "theoretical_loss": 3.9172694544756252, + "tokens_seen": 532152320 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042360080240722166, + "loss": 1.462, + "theoretical_loss": 3.91721842719692, + "tokens_seen": 532217856 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042359077231695085, + "loss": 1.56, + "theoretical_loss": 3.9171674079603, + "tokens_seen": 532283392 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042358074222668, + "loss": 1.4203, + "theoretical_loss": 3.9171163967635065, + "tokens_seen": 532348928 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042357071213640926, + "loss": 1.4941, + "theoretical_loss": 3.9170653936042843, + "tokens_seen": 532414464 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004235606820461384, + "loss": 1.5073, + "theoretical_loss": 3.917014398480377, + "tokens_seen": 532480000 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004235506519558676, + "loss": 1.4835, + "theoretical_loss": 3.9169634113895304, + "tokens_seen": 532545536 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042354062186559675, + "loss": 1.4475, + "theoretical_loss": 3.9169124323294913, + "tokens_seen": 532611072 + }, + { + "epoch": 0.16, + "learning_rate": 0.000423530591775326, + "loss": 1.5419, + "theoretical_loss": 3.916861461298005, + "tokens_seen": 532676608 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042352056168505517, + "loss": 1.4371, + "theoretical_loss": 3.916810498292822, + "tokens_seen": 532742144 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042351053159478435, + "loss": 1.3857, + "theoretical_loss": 3.9167595433116897, + "tokens_seen": 532807680 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042350050150451353, + "loss": 1.491, + "theoretical_loss": 3.9167085963523585, + "tokens_seen": 532873216 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042349047141424276, + "loss": 1.4578, + "theoretical_loss": 3.916657657412579, + "tokens_seen": 532938752 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004234804413239719, + "loss": 1.4829, + "theoretical_loss": 3.9166067264901026, + "tokens_seen": 533004288 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042347041123370113, + "loss": 1.5234, + "theoretical_loss": 3.916555803582683, + "tokens_seen": 533069824 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042346038114343025, + "loss": 1.3891, + "theoretical_loss": 3.9165048886880722, + "tokens_seen": 533135360 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004234503510531595, + "loss": 1.3184, + "theoretical_loss": 3.916453981804026, + "tokens_seen": 533200896 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042344032096288867, + "loss": 1.4017, + "theoretical_loss": 3.916403082928299, + "tokens_seen": 533266432 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042343029087261785, + "loss": 1.4529, + "theoretical_loss": 3.916352192058648, + "tokens_seen": 533331968 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004234202607823471, + "loss": 1.4146, + "theoretical_loss": 3.9163013091928294, + "tokens_seen": 533397504 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004234102306920762, + "loss": 1.4317, + "theoretical_loss": 3.9162504343286013, + "tokens_seen": 533463040 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042340020060180545, + "loss": 1.357, + "theoretical_loss": 3.9161995674637233, + "tokens_seen": 533528576 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042339017051153463, + "loss": 1.4182, + "theoretical_loss": 3.916148708595955, + "tokens_seen": 533594112 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004233801404212638, + "loss": 1.563, + "theoretical_loss": 3.9160978577230567, + "tokens_seen": 533659648 + }, + { + "epoch": 0.16, + "learning_rate": 0.000423370110330993, + "loss": 1.581, + "theoretical_loss": 3.91604701484279, + "tokens_seen": 533725184 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004233600802407222, + "loss": 1.5752, + "theoretical_loss": 3.915996179952918, + "tokens_seen": 533790720 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042335005015045135, + "loss": 1.3818, + "theoretical_loss": 3.915945353051205, + "tokens_seen": 533856256 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004233400200601806, + "loss": 1.4445, + "theoretical_loss": 3.9158945341354134, + "tokens_seen": 533921792 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004233299899699097, + "loss": 1.5261, + "theoretical_loss": 3.9158437232033094, + "tokens_seen": 533987328 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042331995987963895, + "loss": 1.3319, + "theoretical_loss": 3.9157929202526596, + "tokens_seen": 534052864 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 183305, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5211052894592285, + "objective/train/theoretical_loss": 3.9157421252812306, + "objective/train/tokens_used": 534118400, + "theoretical_loss": 3.9157421252812306, + "tokens_seen": 534118400 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042330992978936813, + "loss": 1.5048, + "theoretical_loss": 3.9157421252812306, + "tokens_seen": 534118400 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004232998996990973, + "loss": 1.3442, + "theoretical_loss": 3.9156913382867895, + "tokens_seen": 534183936 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004232898696088265, + "loss": 1.4196, + "theoretical_loss": 3.915640559267107, + "tokens_seen": 534249472 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004232798395185557, + "loss": 1.3705, + "theoretical_loss": 3.9155897882199513, + "tokens_seen": 534315008 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042326980942828486, + "loss": 1.5363, + "theoretical_loss": 3.9155390251430937, + "tokens_seen": 534380544 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004232597793380141, + "loss": 1.4179, + "theoretical_loss": 3.915488270034306, + "tokens_seen": 534446080 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004232497492477432, + "loss": 1.5639, + "theoretical_loss": 3.9154375228913594, + "tokens_seen": 534511616 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042323971915747246, + "loss": 1.5394, + "theoretical_loss": 3.9153867837120284, + "tokens_seen": 534577152 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004232296890672016, + "loss": 1.4783, + "theoretical_loss": 3.9153360524940872, + "tokens_seen": 534642688 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004232196589769308, + "loss": 1.459, + "theoretical_loss": 3.915285329235311, + "tokens_seen": 534708224 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042320962888666, + "loss": 1.4081, + "theoretical_loss": 3.915234613933475, + "tokens_seen": 534773760 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004231995987963892, + "loss": 1.5265, + "theoretical_loss": 3.915183906586357, + "tokens_seen": 534839296 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042318956870611836, + "loss": 1.4511, + "theoretical_loss": 3.915133207191734, + "tokens_seen": 534904832 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004231795386158476, + "loss": 1.4135, + "theoretical_loss": 3.915082515747385, + "tokens_seen": 534970368 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004231695085255767, + "loss": 1.4927, + "theoretical_loss": 3.9150318322510893, + "tokens_seen": 535035904 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042315947843530596, + "loss": 1.3046, + "theoretical_loss": 3.9149811567006285, + "tokens_seen": 535101440 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004231494483450351, + "loss": 1.462, + "theoretical_loss": 3.9149304890937833, + "tokens_seen": 535166976 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004231394182547643, + "loss": 1.4653, + "theoretical_loss": 3.9148798294283353, + "tokens_seen": 535232512 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004231293881644935, + "loss": 1.5137, + "theoretical_loss": 3.9148291777020683, + "tokens_seen": 535298048 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004231193580742227, + "loss": 1.5336, + "theoretical_loss": 3.9147785339127665, + "tokens_seen": 535363584 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042310932798395186, + "loss": 1.46, + "theoretical_loss": 3.914727898058215, + "tokens_seen": 535429120 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042309929789368105, + "loss": 1.3614, + "theoretical_loss": 3.914677270136199, + "tokens_seen": 535494656 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004230892678034102, + "loss": 1.4975, + "theoretical_loss": 3.914626650144505, + "tokens_seen": 535560192 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042307923771313946, + "loss": 1.6551, + "theoretical_loss": 3.914576038080922, + "tokens_seen": 535625728 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004230692076228686, + "loss": 1.4999, + "theoretical_loss": 3.9145254339432367, + "tokens_seen": 535691264 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004230591775325978, + "loss": 1.5854, + "theoretical_loss": 3.9144748377292395, + "tokens_seen": 535756800 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042304914744232695, + "loss": 1.3148, + "theoretical_loss": 3.9144242494367205, + "tokens_seen": 535822336 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004230391173520562, + "loss": 1.4144, + "theoretical_loss": 3.914373669063471, + "tokens_seen": 535887872 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042302908726178537, + "loss": 1.5131, + "theoretical_loss": 3.914323096607283, + "tokens_seen": 535953408 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042301905717151455, + "loss": 1.4195, + "theoretical_loss": 3.914272532065949, + "tokens_seen": 536018944 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042300902708124373, + "loss": 1.538, + "theoretical_loss": 3.914221975437264, + "tokens_seen": 536084480 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042299899699097297, + "loss": 1.4685, + "theoretical_loss": 3.914171426719021, + "tokens_seen": 536150016 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004229889669007021, + "loss": 1.5366, + "theoretical_loss": 3.9141208859090164, + "tokens_seen": 536215552 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042297893681043133, + "loss": 1.4064, + "theoretical_loss": 3.9140703530050467, + "tokens_seen": 536281088 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042296890672016045, + "loss": 1.4681, + "theoretical_loss": 3.914019828004909, + "tokens_seen": 536346624 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004229588766298897, + "loss": 1.4021, + "theoretical_loss": 3.9139693109064018, + "tokens_seen": 536412160 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042294884653961887, + "loss": 1.4506, + "theoretical_loss": 3.9139188017073243, + "tokens_seen": 536477696 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042293881644934805, + "loss": 1.3739, + "theoretical_loss": 3.9138683004054764, + "tokens_seen": 536543232 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042292878635907723, + "loss": 1.3936, + "theoretical_loss": 3.913817806998659, + "tokens_seen": 536608768 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004229187562688064, + "loss": 1.4368, + "theoretical_loss": 3.9137673214846735, + "tokens_seen": 536674304 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004229087261785356, + "loss": 1.4503, + "theoretical_loss": 3.913716843861323, + "tokens_seen": 536739840 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042289869608826483, + "loss": 1.5074, + "theoretical_loss": 3.91366637412641, + "tokens_seen": 536805376 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042288866599799396, + "loss": 1.4008, + "theoretical_loss": 3.9136159122777405, + "tokens_seen": 536870912 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004228786359077232, + "loss": 1.3333, + "theoretical_loss": 3.9135654583131183, + "tokens_seen": 536936448 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004228686058174523, + "loss": 1.5045, + "theoretical_loss": 3.9135150122303504, + "tokens_seen": 537001984 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042285857572718155, + "loss": 1.413, + "theoretical_loss": 3.913464574027244, + "tokens_seen": 537067520 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042284854563691074, + "loss": 1.476, + "theoretical_loss": 3.913414143701606, + "tokens_seen": 537133056 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004228385155466399, + "loss": 1.4713, + "theoretical_loss": 3.913363721251246, + "tokens_seen": 537198592 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004228284854563691, + "loss": 1.4846, + "theoretical_loss": 3.913313306673974, + "tokens_seen": 537264128 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042281845536609833, + "loss": 1.4371, + "theoretical_loss": 3.913262899967599, + "tokens_seen": 537329664 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 184222, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.484065294265747, + "objective/train/theoretical_loss": 3.9132125011299337, + "objective/train/tokens_used": 537395200, + "theoretical_loss": 3.9132125011299337, + "tokens_seen": 537395200 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042280842527582746, + "loss": 1.4531, + "theoretical_loss": 3.9132125011299337, + "tokens_seen": 537395200 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004227983951855567, + "loss": 1.5002, + "theoretical_loss": 3.9131621101587903, + "tokens_seen": 537460736 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004227883650952858, + "loss": 1.4227, + "theoretical_loss": 3.913111727051982, + "tokens_seen": 537526272 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042277833500501506, + "loss": 1.4697, + "theoretical_loss": 3.9130613518073227, + "tokens_seen": 537591808 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042276830491474424, + "loss": 1.4722, + "theoretical_loss": 3.913010984422627, + "tokens_seen": 537657344 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004227582748244734, + "loss": 1.4676, + "theoretical_loss": 3.9129606248957103, + "tokens_seen": 537722880 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004227482447342026, + "loss": 1.3821, + "theoretical_loss": 3.91291027322439, + "tokens_seen": 537788416 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004227382146439318, + "loss": 1.4978, + "theoretical_loss": 3.9128599294064834, + "tokens_seen": 537853952 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042272818455366096, + "loss": 1.4632, + "theoretical_loss": 3.9128095934398095, + "tokens_seen": 537919488 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004227181544633902, + "loss": 1.5509, + "theoretical_loss": 3.9127592653221868, + "tokens_seen": 537985024 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004227081243731193, + "loss": 1.4357, + "theoretical_loss": 3.912708945051435, + "tokens_seen": 538050560 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042269809428284856, + "loss": 1.3978, + "theoretical_loss": 3.9126586326253765, + "tokens_seen": 538116096 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004226880641925777, + "loss": 1.5196, + "theoretical_loss": 3.9126083280418325, + "tokens_seen": 538181632 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004226780341023069, + "loss": 1.4793, + "theoretical_loss": 3.9125580312986257, + "tokens_seen": 538247168 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042266800401203616, + "loss": 1.4522, + "theoretical_loss": 3.9125077423935792, + "tokens_seen": 538312704 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004226579739217653, + "loss": 1.3907, + "theoretical_loss": 3.912457461324519, + "tokens_seen": 538378240 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004226479438314945, + "loss": 1.6468, + "theoretical_loss": 3.9124071880892686, + "tokens_seen": 538443776 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004226379137412237, + "loss": 1.416, + "theoretical_loss": 3.912356922685655, + "tokens_seen": 538509312 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004226278836509529, + "loss": 1.3703, + "theoretical_loss": 3.912306665111506, + "tokens_seen": 538574848 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042261785356068206, + "loss": 1.4133, + "theoretical_loss": 3.9122564153646486, + "tokens_seen": 538640384 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042260782347041125, + "loss": 1.4411, + "theoretical_loss": 3.9122061734429128, + "tokens_seen": 538705920 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004225977933801404, + "loss": 1.5693, + "theoretical_loss": 3.9121559393441268, + "tokens_seen": 538771456 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042258776328986966, + "loss": 1.4146, + "theoretical_loss": 3.912105713066122, + "tokens_seen": 538836992 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004225777331995988, + "loss": 1.4723, + "theoretical_loss": 3.91205549460673, + "tokens_seen": 538902528 + }, + { + "epoch": 0.16, + "learning_rate": 0.000422567703109328, + "loss": 1.4058, + "theoretical_loss": 3.9120052839637824, + "tokens_seen": 538968064 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042255767301905715, + "loss": 1.6231, + "theoretical_loss": 3.9119550811351127, + "tokens_seen": 539033600 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004225476429287864, + "loss": 1.4724, + "theoretical_loss": 3.9119048861185552, + "tokens_seen": 539099136 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042253761283851557, + "loss": 1.4408, + "theoretical_loss": 3.9118546989119447, + "tokens_seen": 539164672 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042252758274824475, + "loss": 1.3392, + "theoretical_loss": 3.9118045195131166, + "tokens_seen": 539230208 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042251755265797393, + "loss": 1.5097, + "theoretical_loss": 3.911754347919908, + "tokens_seen": 539295744 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042250752256770317, + "loss": 1.3965, + "theoretical_loss": 3.911704184130156, + "tokens_seen": 539361280 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004224974924774323, + "loss": 1.4918, + "theoretical_loss": 3.911654028141699, + "tokens_seen": 539426816 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042248746238716153, + "loss": 1.4706, + "theoretical_loss": 3.9116038799523762, + "tokens_seen": 539492352 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042247743229689065, + "loss": 1.4796, + "theoretical_loss": 3.911553739560028, + "tokens_seen": 539557888 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004224674022066199, + "loss": 1.4471, + "theoretical_loss": 3.9115036069624947, + "tokens_seen": 539623424 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042245737211634907, + "loss": 1.5329, + "theoretical_loss": 3.9114534821576186, + "tokens_seen": 539688960 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042244734202607825, + "loss": 1.4322, + "theoretical_loss": 3.9114033651432414, + "tokens_seen": 539754496 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042243731193580743, + "loss": 1.5565, + "theoretical_loss": 3.9113532559172084, + "tokens_seen": 539820032 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004224272818455366, + "loss": 1.4497, + "theoretical_loss": 3.9113031544773618, + "tokens_seen": 539885568 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004224172517552658, + "loss": 1.4031, + "theoretical_loss": 3.9112530608215486, + "tokens_seen": 539951104 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042240722166499503, + "loss": 1.4924, + "theoretical_loss": 3.9112029749476145, + "tokens_seen": 540016640 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042239719157472416, + "loss": 1.4986, + "theoretical_loss": 3.9111528968534053, + "tokens_seen": 540082176 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004223871614844534, + "loss": 1.4631, + "theoretical_loss": 3.91110282653677, + "tokens_seen": 540147712 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004223771313941825, + "loss": 1.5415, + "theoretical_loss": 3.9110527639955563, + "tokens_seen": 540213248 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042236710130391176, + "loss": 1.569, + "theoretical_loss": 3.911002709227615, + "tokens_seen": 540278784 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042235707121364094, + "loss": 1.5188, + "theoretical_loss": 3.9109526622307955, + "tokens_seen": 540344320 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004223470411233701, + "loss": 1.509, + "theoretical_loss": 3.910902623002949, + "tokens_seen": 540409856 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004223370110330993, + "loss": 1.4352, + "theoretical_loss": 3.9108525915419277, + "tokens_seen": 540475392 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042232698094282853, + "loss": 1.4378, + "theoretical_loss": 3.9108025678455847, + "tokens_seen": 540540928 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042231695085255766, + "loss": 1.4153, + "theoretical_loss": 3.9107525519117736, + "tokens_seen": 540606464 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 185518, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4521079063415527, + "objective/train/theoretical_loss": 3.910702543738349, + "objective/train/tokens_used": 540672000, + "theoretical_loss": 3.910702543738349, + "tokens_seen": 540672000 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004223069207622869, + "loss": 1.3849, + "theoretical_loss": 3.910702543738349, + "tokens_seen": 540672000 + }, + { + "epoch": 0.16, + "learning_rate": 0.000422296890672016, + "loss": 1.4125, + "theoretical_loss": 3.9106525433231663, + "tokens_seen": 540737536 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042228686058174526, + "loss": 1.3622, + "theoretical_loss": 3.9106025506640822, + "tokens_seen": 540803072 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042227683049147444, + "loss": 1.4254, + "theoretical_loss": 3.9105525657589535, + "tokens_seen": 540868608 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004222668004012036, + "loss": 1.4098, + "theoretical_loss": 3.910502588605638, + "tokens_seen": 540934144 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004222567703109328, + "loss": 1.374, + "theoretical_loss": 3.9104526192019957, + "tokens_seen": 540999680 + }, + { + "epoch": 0.16, + "learning_rate": 0.000422246740220662, + "loss": 1.5545, + "theoretical_loss": 3.9104026575458857, + "tokens_seen": 541065216 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042223671013039116, + "loss": 1.3911, + "theoretical_loss": 3.9103527036351684, + "tokens_seen": 541130752 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004222266800401204, + "loss": 1.4406, + "theoretical_loss": 3.910302757467705, + "tokens_seen": 541196288 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004222166499498495, + "loss": 1.4532, + "theoretical_loss": 3.910252819041358, + "tokens_seen": 541261824 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042220661985957876, + "loss": 1.525, + "theoretical_loss": 3.910202888353991, + "tokens_seen": 541327360 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004221965897693079, + "loss": 1.4174, + "theoretical_loss": 3.9101529654034675, + "tokens_seen": 541392896 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004221865596790371, + "loss": 1.596, + "theoretical_loss": 3.9101030501876526, + "tokens_seen": 541458432 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004221765295887663, + "loss": 1.4288, + "theoretical_loss": 3.9100531427044114, + "tokens_seen": 541523968 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004221664994984955, + "loss": 1.4187, + "theoretical_loss": 3.9100032429516114, + "tokens_seen": 541589504 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042215646940822467, + "loss": 1.4238, + "theoretical_loss": 3.909953350927119, + "tokens_seen": 541655040 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004221464393179539, + "loss": 1.4274, + "theoretical_loss": 3.909903466628803, + "tokens_seen": 541720576 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042213640922768303, + "loss": 1.4631, + "theoretical_loss": 3.9098535900545324, + "tokens_seen": 541786112 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042212637913741226, + "loss": 1.4231, + "theoretical_loss": 3.909803721202177, + "tokens_seen": 541851648 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004221163490471414, + "loss": 1.5587, + "theoretical_loss": 3.909753860069608, + "tokens_seen": 541917184 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004221063189568706, + "loss": 1.5061, + "theoretical_loss": 3.909704006654695, + "tokens_seen": 541982720 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004220962888665998, + "loss": 1.4981, + "theoretical_loss": 3.9096541609553137, + "tokens_seen": 542048256 + }, + { + "epoch": 0.16, + "learning_rate": 0.000422086258776329, + "loss": 1.5048, + "theoretical_loss": 3.909604322969335, + "tokens_seen": 542113792 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042207622868605817, + "loss": 1.4674, + "theoretical_loss": 3.9095544926946344, + "tokens_seen": 542179328 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042206619859578735, + "loss": 1.5528, + "theoretical_loss": 3.909504670129085, + "tokens_seen": 542244864 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042205616850551653, + "loss": 1.4203, + "theoretical_loss": 3.909454855270565, + "tokens_seen": 542310400 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042204613841524577, + "loss": 1.3973, + "theoretical_loss": 3.909405048116949, + "tokens_seen": 542375936 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004220361083249749, + "loss": 1.482, + "theoretical_loss": 3.909355248666116, + "tokens_seen": 542441472 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042202607823470413, + "loss": 1.4604, + "theoretical_loss": 3.9093054569159436, + "tokens_seen": 542507008 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004220160481444333, + "loss": 1.5183, + "theoretical_loss": 3.909255672864311, + "tokens_seen": 542572544 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004220060180541625, + "loss": 1.3709, + "theoretical_loss": 3.909205896509098, + "tokens_seen": 542638080 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004219959879638917, + "loss": 1.478, + "theoretical_loss": 3.9091561278481866, + "tokens_seen": 542703616 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042198595787362085, + "loss": 1.4043, + "theoretical_loss": 3.909106366879457, + "tokens_seen": 542769152 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042197592778335004, + "loss": 1.3794, + "theoretical_loss": 3.909056613600793, + "tokens_seen": 542834688 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042196589769307927, + "loss": 1.525, + "theoretical_loss": 3.9090068680100765, + "tokens_seen": 542900224 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004219558676028084, + "loss": 1.5333, + "theoretical_loss": 3.9089571301051933, + "tokens_seen": 542965760 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042194583751253763, + "loss": 1.3719, + "theoretical_loss": 3.9089073998840274, + "tokens_seen": 543031296 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042193580742226676, + "loss": 1.4402, + "theoretical_loss": 3.908857677344465, + "tokens_seen": 543096832 + }, + { + "epoch": 0.16, + "learning_rate": 0.000421925777331996, + "loss": 1.4038, + "theoretical_loss": 3.908807962484393, + "tokens_seen": 543162368 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042191574724172523, + "loss": 1.4176, + "theoretical_loss": 3.908758255301699, + "tokens_seen": 543227904 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042190571715145436, + "loss": 1.529, + "theoretical_loss": 3.9087085557942713, + "tokens_seen": 543293440 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004218956870611836, + "loss": 1.531, + "theoretical_loss": 3.908658863959999, + "tokens_seen": 543358976 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004218856569709127, + "loss": 1.4616, + "theoretical_loss": 3.9086091797967724, + "tokens_seen": 543424512 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042187562688064196, + "loss": 1.3588, + "theoretical_loss": 3.908559503302482, + "tokens_seen": 543490048 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042186559679037114, + "loss": 1.5527, + "theoretical_loss": 3.9085098344750193, + "tokens_seen": 543555584 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004218555667001003, + "loss": 1.4699, + "theoretical_loss": 3.908460173312278, + "tokens_seen": 543621120 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004218455366098295, + "loss": 1.4309, + "theoretical_loss": 3.9084105198121506, + "tokens_seen": 543686656 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042183550651955873, + "loss": 1.4711, + "theoretical_loss": 3.9083608739725317, + "tokens_seen": 543752192 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042182547642928786, + "loss": 1.554, + "theoretical_loss": 3.9083112357913166, + "tokens_seen": 543817728 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004218154463390171, + "loss": 1.5287, + "theoretical_loss": 3.9082616052664, + "tokens_seen": 543883264 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 186793, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2986621856689453, + "objective/train/theoretical_loss": 3.90821198239568, + "objective/train/tokens_used": 543948800, + "theoretical_loss": 3.90821198239568, + "tokens_seen": 543948800 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004218054162487462, + "loss": 1.4624, + "theoretical_loss": 3.90821198239568, + "tokens_seen": 543948800 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042179538615847546, + "loss": 1.5475, + "theoretical_loss": 3.9081623671770536, + "tokens_seen": 544014336 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042178535606820464, + "loss": 1.4413, + "theoretical_loss": 3.9081127596084193, + "tokens_seen": 544079872 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004217753259779338, + "loss": 1.4139, + "theoretical_loss": 3.908063159687676, + "tokens_seen": 544145408 + }, + { + "epoch": 0.16, + "learning_rate": 0.000421765295887663, + "loss": 1.4304, + "theoretical_loss": 3.9080135674127243, + "tokens_seen": 544210944 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004217552657973922, + "loss": 1.3651, + "theoretical_loss": 3.9079639827814647, + "tokens_seen": 544276480 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042174523570712136, + "loss": 1.5152, + "theoretical_loss": 3.907914405791799, + "tokens_seen": 544342016 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004217352056168506, + "loss": 1.5249, + "theoretical_loss": 3.9078648364416297, + "tokens_seen": 544407552 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004217251755265797, + "loss": 1.5305, + "theoretical_loss": 3.9078152747288604, + "tokens_seen": 544473088 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042171514543630896, + "loss": 1.5357, + "theoretical_loss": 3.907765720651395, + "tokens_seen": 544538624 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004217051153460381, + "loss": 1.4532, + "theoretical_loss": 3.907716174207139, + "tokens_seen": 544604160 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004216950852557673, + "loss": 1.4906, + "theoretical_loss": 3.9076666353939977, + "tokens_seen": 544669696 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004216850551654965, + "loss": 1.5192, + "theoretical_loss": 3.9076171042098777, + "tokens_seen": 544735232 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004216750250752257, + "loss": 1.4484, + "theoretical_loss": 3.907567580652687, + "tokens_seen": 544800768 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042166499498495487, + "loss": 1.4868, + "theoretical_loss": 3.9075180647203336, + "tokens_seen": 544866304 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004216549648946841, + "loss": 1.4944, + "theoretical_loss": 3.907468556410727, + "tokens_seen": 544931840 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042164493480441323, + "loss": 1.4984, + "theoretical_loss": 3.907419055721777, + "tokens_seen": 544997376 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042163490471414246, + "loss": 1.391, + "theoretical_loss": 3.907369562651394, + "tokens_seen": 545062912 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004216248746238716, + "loss": 1.4991, + "theoretical_loss": 3.9073200771974905, + "tokens_seen": 545128448 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004216148445336008, + "loss": 1.4742, + "theoretical_loss": 3.907270599357978, + "tokens_seen": 545193984 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042160481444333, + "loss": 1.4402, + "theoretical_loss": 3.9072211291307704, + "tokens_seen": 545259520 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004215947843530592, + "loss": 1.5315, + "theoretical_loss": 3.907171666513781, + "tokens_seen": 545325056 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042158475426278837, + "loss": 1.6173, + "theoretical_loss": 3.907122211504926, + "tokens_seen": 545390592 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042157472417251755, + "loss": 1.5591, + "theoretical_loss": 3.9070727641021206, + "tokens_seen": 545456128 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042156469408224673, + "loss": 1.4994, + "theoretical_loss": 3.907023324303281, + "tokens_seen": 545521664 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042155466399197597, + "loss": 1.4577, + "theoretical_loss": 3.9069738921063246, + "tokens_seen": 545587200 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004215446339017051, + "loss": 1.4589, + "theoretical_loss": 3.9069244675091697, + "tokens_seen": 545652736 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042153460381143433, + "loss": 1.5015, + "theoretical_loss": 3.9068750505097354, + "tokens_seen": 545718272 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004215245737211635, + "loss": 1.5326, + "theoretical_loss": 3.906825641105942, + "tokens_seen": 545783808 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004215145436308927, + "loss": 1.5921, + "theoretical_loss": 3.9067762392957093, + "tokens_seen": 545849344 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004215045135406219, + "loss": 1.4922, + "theoretical_loss": 3.90672684507696, + "tokens_seen": 545914880 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042149448345035105, + "loss": 1.5289, + "theoretical_loss": 3.9066774584476147, + "tokens_seen": 545980416 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042148445336008024, + "loss": 1.4455, + "theoretical_loss": 3.9066280794055976, + "tokens_seen": 546045952 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042147442326980947, + "loss": 1.559, + "theoretical_loss": 3.9065787079488326, + "tokens_seen": 546111488 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004214643931795386, + "loss": 1.4018, + "theoretical_loss": 3.9065293440752447, + "tokens_seen": 546177024 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042145436308926783, + "loss": 1.5548, + "theoretical_loss": 3.9064799877827587, + "tokens_seen": 546242560 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042144433299899696, + "loss": 1.4608, + "theoretical_loss": 3.9064306390693018, + "tokens_seen": 546308096 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004214343029087262, + "loss": 1.475, + "theoretical_loss": 3.9063812979328, + "tokens_seen": 546373632 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004214242728184554, + "loss": 1.5408, + "theoretical_loss": 3.9063319643711827, + "tokens_seen": 546439168 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042141424272818456, + "loss": 1.5833, + "theoretical_loss": 3.906282638382377, + "tokens_seen": 546504704 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042140421263791374, + "loss": 1.429, + "theoretical_loss": 3.906233319964315, + "tokens_seen": 546570240 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004213941825476429, + "loss": 1.4002, + "theoretical_loss": 3.9061840091149254, + "tokens_seen": 546635776 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004213841524573721, + "loss": 1.5216, + "theoretical_loss": 3.90613470583214, + "tokens_seen": 546701312 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042137412236710134, + "loss": 1.4988, + "theoretical_loss": 3.9060854101138904, + "tokens_seen": 546766848 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042136409227683046, + "loss": 1.5389, + "theoretical_loss": 3.90603612195811, + "tokens_seen": 546832384 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004213540621865597, + "loss": 1.4621, + "theoretical_loss": 3.9059868413627328, + "tokens_seen": 546897920 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004213440320962889, + "loss": 1.4701, + "theoretical_loss": 3.905937568325692, + "tokens_seen": 546963456 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042133400200601806, + "loss": 1.4683, + "theoretical_loss": 3.9058883028449247, + "tokens_seen": 547028992 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042132397191574724, + "loss": 1.3808, + "theoretical_loss": 3.905839044918366, + "tokens_seen": 547094528 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004213139418254764, + "loss": 1.4076, + "theoretical_loss": 3.905789794543953, + "tokens_seen": 547160064 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 187579, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3113932609558105, + "objective/train/theoretical_loss": 3.9057405517196235, + "objective/train/tokens_used": 547225600, + "theoretical_loss": 3.9057405517196235, + "tokens_seen": 547225600 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004213039117352056, + "loss": 1.4151, + "theoretical_loss": 3.9057405517196235, + "tokens_seen": 547225600 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042129388164493484, + "loss": 1.5847, + "theoretical_loss": 3.905691316443316, + "tokens_seen": 547291136 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042128385155466397, + "loss": 1.4115, + "theoretical_loss": 3.90564208871297, + "tokens_seen": 547356672 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004212738214643932, + "loss": 1.4677, + "theoretical_loss": 3.9055928685265258, + "tokens_seen": 547422208 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042126379137412233, + "loss": 1.3947, + "theoretical_loss": 3.9055436558819245, + "tokens_seen": 547487744 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042125376128385156, + "loss": 1.4913, + "theoretical_loss": 3.905494450777107, + "tokens_seen": 547553280 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042124373119358075, + "loss": 1.4571, + "theoretical_loss": 3.9054452532100177, + "tokens_seen": 547618816 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004212337011033099, + "loss": 1.4878, + "theoretical_loss": 3.905396063178598, + "tokens_seen": 547684352 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004212236710130391, + "loss": 1.3654, + "theoretical_loss": 3.905346880680794, + "tokens_seen": 547749888 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004212136409227683, + "loss": 1.5229, + "theoretical_loss": 3.905297705714549, + "tokens_seen": 547815424 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042120361083249747, + "loss": 1.4995, + "theoretical_loss": 3.9052485382778097, + "tokens_seen": 547880960 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004211935807422267, + "loss": 1.3549, + "theoretical_loss": 3.905199378368523, + "tokens_seen": 547946496 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042118355065195583, + "loss": 1.4927, + "theoretical_loss": 3.9051502259846362, + "tokens_seen": 548012032 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042117352056168507, + "loss": 1.459, + "theoretical_loss": 3.9051010811240974, + "tokens_seen": 548077568 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004211634904714143, + "loss": 1.4346, + "theoretical_loss": 3.905051943784856, + "tokens_seen": 548143104 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042115346038114343, + "loss": 1.4114, + "theoretical_loss": 3.905002813964861, + "tokens_seen": 548208640 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042114343029087266, + "loss": 1.4132, + "theoretical_loss": 3.904953691662064, + "tokens_seen": 548274176 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004211334002006018, + "loss": 1.4064, + "theoretical_loss": 3.904904576874416, + "tokens_seen": 548339712 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042112337011033103, + "loss": 1.4648, + "theoretical_loss": 3.90485546959987, + "tokens_seen": 548405248 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004211133400200602, + "loss": 1.4759, + "theoretical_loss": 3.904806369836378, + "tokens_seen": 548470784 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004211033099297894, + "loss": 1.4363, + "theoretical_loss": 3.904757277581895, + "tokens_seen": 548536320 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042109327983951857, + "loss": 1.4878, + "theoretical_loss": 3.9047081928343745, + "tokens_seen": 548601856 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042108324974924775, + "loss": 1.4872, + "theoretical_loss": 3.904659115591773, + "tokens_seen": 548667392 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042107321965897693, + "loss": 1.5398, + "theoretical_loss": 3.904610045852046, + "tokens_seen": 548732928 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042106318956870617, + "loss": 1.3987, + "theoretical_loss": 3.904560983613152, + "tokens_seen": 548798464 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004210531594784353, + "loss": 1.5644, + "theoretical_loss": 3.9045119288730468, + "tokens_seen": 548864000 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042104312938816453, + "loss": 1.4924, + "theoretical_loss": 3.9044628816296907, + "tokens_seen": 548929536 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004210330992978937, + "loss": 1.4567, + "theoretical_loss": 3.9044138418810426, + "tokens_seen": 548995072 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004210230692076229, + "loss": 1.3177, + "theoretical_loss": 3.904364809625063, + "tokens_seen": 549060608 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004210130391173521, + "loss": 1.4632, + "theoretical_loss": 3.9043157848597128, + "tokens_seen": 549126144 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042100300902708125, + "loss": 1.3819, + "theoretical_loss": 3.904266767582954, + "tokens_seen": 549191680 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042099297893681044, + "loss": 1.5235, + "theoretical_loss": 3.9042177577927495, + "tokens_seen": 549257216 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042098294884653967, + "loss": 1.4773, + "theoretical_loss": 3.904168755487062, + "tokens_seen": 549322752 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004209729187562688, + "loss": 1.4826, + "theoretical_loss": 3.9041197606638565, + "tokens_seen": 549388288 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042096288866599803, + "loss": 1.5519, + "theoretical_loss": 3.9040707733210978, + "tokens_seen": 549453824 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042095285857572716, + "loss": 1.5799, + "theoretical_loss": 3.9040217934567525, + "tokens_seen": 549519360 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004209428284854564, + "loss": 1.3962, + "theoretical_loss": 3.903972821068786, + "tokens_seen": 549584896 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004209327983951856, + "loss": 1.432, + "theoretical_loss": 3.903923856155167, + "tokens_seen": 549650432 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042092276830491476, + "loss": 1.4674, + "theoretical_loss": 3.9038748987138625, + "tokens_seen": 549715968 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042091273821464394, + "loss": 1.5013, + "theoretical_loss": 3.9038259487428424, + "tokens_seen": 549781504 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004209027081243731, + "loss": 1.4659, + "theoretical_loss": 3.9037770062400767, + "tokens_seen": 549847040 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004208926780341023, + "loss": 1.5036, + "theoretical_loss": 3.903728071203535, + "tokens_seen": 549912576 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042088264794383154, + "loss": 1.3779, + "theoretical_loss": 3.90367914363119, + "tokens_seen": 549978112 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042087261785356066, + "loss": 1.3536, + "theoretical_loss": 3.903630223521013, + "tokens_seen": 550043648 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004208625877632899, + "loss": 1.4681, + "theoretical_loss": 3.903581310870978, + "tokens_seen": 550109184 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004208525576730191, + "loss": 1.3732, + "theoretical_loss": 3.9035324056790577, + "tokens_seen": 550174720 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042084252758274826, + "loss": 1.483, + "theoretical_loss": 3.9034835079432275, + "tokens_seen": 550240256 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042083249749247744, + "loss": 1.4608, + "theoretical_loss": 3.9034346176614623, + "tokens_seen": 550305792 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004208224674022066, + "loss": 1.3676, + "theoretical_loss": 3.9033857348317387, + "tokens_seen": 550371328 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004208124373119358, + "loss": 1.5206, + "theoretical_loss": 3.9033368594520335, + "tokens_seen": 550436864 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 188908, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.361107349395752, + "objective/train/theoretical_loss": 3.903287991520324, + "objective/train/tokens_used": 550502400, + "theoretical_loss": 3.903287991520324, + "tokens_seen": 550502400 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042080240722166504, + "loss": 1.4074, + "theoretical_loss": 3.903287991520324, + "tokens_seen": 550502400 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042079237713139417, + "loss": 1.4959, + "theoretical_loss": 3.903239131034589, + "tokens_seen": 550567936 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004207823470411234, + "loss": 1.5497, + "theoretical_loss": 3.9031902779928087, + "tokens_seen": 550633472 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042077231695085253, + "loss": 1.5658, + "theoretical_loss": 3.903141432392962, + "tokens_seen": 550699008 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042076228686058176, + "loss": 1.4212, + "theoretical_loss": 3.903092594233031, + "tokens_seen": 550764544 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042075225677031095, + "loss": 1.3944, + "theoretical_loss": 3.903043763510996, + "tokens_seen": 550830080 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004207422266800401, + "loss": 1.4846, + "theoretical_loss": 3.9029949402248407, + "tokens_seen": 550895616 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004207321965897693, + "loss": 1.4781, + "theoretical_loss": 3.902946124372548, + "tokens_seen": 550961152 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004207221664994985, + "loss": 1.3531, + "theoretical_loss": 3.902897315952102, + "tokens_seen": 551026688 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042071213640922767, + "loss": 1.3857, + "theoretical_loss": 3.9028485149614873, + "tokens_seen": 551092224 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004207021063189569, + "loss": 1.556, + "theoretical_loss": 3.90279972139869, + "tokens_seen": 551157760 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042069207622868603, + "loss": 1.3875, + "theoretical_loss": 3.902750935261696, + "tokens_seen": 551223296 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042068204613841527, + "loss": 1.4983, + "theoretical_loss": 3.902702156548493, + "tokens_seen": 551288832 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042067201604814445, + "loss": 1.4342, + "theoretical_loss": 3.9026533852570684, + "tokens_seen": 551354368 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042066198595787363, + "loss": 1.4082, + "theoretical_loss": 3.9026046213854118, + "tokens_seen": 551419904 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004206519558676028, + "loss": 1.4786, + "theoretical_loss": 3.9025558649315117, + "tokens_seen": 551485440 + }, + { + "epoch": 0.17, + "learning_rate": 0.000420641925777332, + "loss": 1.4584, + "theoretical_loss": 3.9025071158933597, + "tokens_seen": 551550976 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004206318956870612, + "loss": 1.428, + "theoretical_loss": 3.9024583742689463, + "tokens_seen": 551616512 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004206218655967904, + "loss": 1.4751, + "theoretical_loss": 3.902409640056263, + "tokens_seen": 551682048 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042061183550651954, + "loss": 1.3895, + "theoretical_loss": 3.902360913253303, + "tokens_seen": 551747584 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042060180541624877, + "loss": 1.4464, + "theoretical_loss": 3.9023121938580605, + "tokens_seen": 551813120 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004205917753259779, + "loss": 1.4759, + "theoretical_loss": 3.9022634818685282, + "tokens_seen": 551878656 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042058174523570713, + "loss": 1.4253, + "theoretical_loss": 3.902214777282702, + "tokens_seen": 551944192 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004205717151454363, + "loss": 1.4929, + "theoretical_loss": 3.902166080098578, + "tokens_seen": 552009728 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004205616850551655, + "loss": 1.4098, + "theoretical_loss": 3.902117390314152, + "tokens_seen": 552075264 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004205516549648947, + "loss": 1.5464, + "theoretical_loss": 3.902068707927422, + "tokens_seen": 552140800 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004205416248746239, + "loss": 1.5079, + "theoretical_loss": 3.9020200329363863, + "tokens_seen": 552206336 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042053159478435304, + "loss": 1.4577, + "theoretical_loss": 3.901971365339043, + "tokens_seen": 552271872 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004205215646940823, + "loss": 1.4789, + "theoretical_loss": 3.9019227051333925, + "tokens_seen": 552337408 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004205115346038114, + "loss": 1.3816, + "theoretical_loss": 3.901874052317435, + "tokens_seen": 552402944 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042050150451354064, + "loss": 1.4727, + "theoretical_loss": 3.9018254068891727, + "tokens_seen": 552468480 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004204914744232698, + "loss": 1.4408, + "theoretical_loss": 3.901776768846606, + "tokens_seen": 552534016 + }, + { + "epoch": 0.17, + "learning_rate": 0.000420481444332999, + "loss": 1.5269, + "theoretical_loss": 3.901728138187739, + "tokens_seen": 552599552 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004204714142427282, + "loss": 1.3871, + "theoretical_loss": 3.9016795149105747, + "tokens_seen": 552665088 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042046138415245736, + "loss": 1.4314, + "theoretical_loss": 3.901630899013118, + "tokens_seen": 552730624 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042045135406218654, + "loss": 1.46, + "theoretical_loss": 3.901582290493374, + "tokens_seen": 552796160 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004204413239719158, + "loss": 1.3308, + "theoretical_loss": 3.901533689349348, + "tokens_seen": 552861696 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004204312938816449, + "loss": 1.4933, + "theoretical_loss": 3.9014850955790474, + "tokens_seen": 552927232 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042042126379137414, + "loss": 1.5384, + "theoretical_loss": 3.9014365091804795, + "tokens_seen": 552992768 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004204112337011033, + "loss": 1.3987, + "theoretical_loss": 3.9013879301516523, + "tokens_seen": 553058304 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004204012036108325, + "loss": 1.3822, + "theoretical_loss": 3.901339358490575, + "tokens_seen": 553123840 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042039117352056174, + "loss": 1.4458, + "theoretical_loss": 3.9012907941952575, + "tokens_seen": 553189376 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042038114343029086, + "loss": 1.4088, + "theoretical_loss": 3.9012422372637103, + "tokens_seen": 553254912 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004203711133400201, + "loss": 1.4589, + "theoretical_loss": 3.901193687693945, + "tokens_seen": 553320448 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004203610832497493, + "loss": 1.4452, + "theoretical_loss": 3.9011451454839734, + "tokens_seen": 553385984 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042035105315947846, + "loss": 1.5018, + "theoretical_loss": 3.901096610631808, + "tokens_seen": 553451520 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042034102306920764, + "loss": 1.3901, + "theoretical_loss": 3.901048083135464, + "tokens_seen": 553517056 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004203309929789368, + "loss": 1.5487, + "theoretical_loss": 3.900999562992954, + "tokens_seen": 553582592 + }, + { + "epoch": 0.17, + "learning_rate": 0.000420320962888666, + "loss": 1.5134, + "theoretical_loss": 3.9009510502022944, + "tokens_seen": 553648128 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042031093279839524, + "loss": 1.5124, + "theoretical_loss": 3.9009025447615002, + "tokens_seen": 553713664 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 189802, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3590831756591797, + "objective/train/theoretical_loss": 3.9008540466685897, + "objective/train/tokens_used": 553779200, + "theoretical_loss": 3.9008540466685897, + "tokens_seen": 553779200 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042030090270812437, + "loss": 1.4928, + "theoretical_loss": 3.9008540466685897, + "tokens_seen": 553779200 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004202908726178536, + "loss": 1.4232, + "theoretical_loss": 3.900805555921579, + "tokens_seen": 553844736 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042028084252758273, + "loss": 1.4822, + "theoretical_loss": 3.9007570725184872, + "tokens_seen": 553910272 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042027081243731196, + "loss": 1.4364, + "theoretical_loss": 3.900708596457333, + "tokens_seen": 553975808 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042026078234704115, + "loss": 1.4353, + "theoretical_loss": 3.900660127736136, + "tokens_seen": 554041344 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004202507522567703, + "loss": 1.5116, + "theoretical_loss": 3.9006116663529173, + "tokens_seen": 554106880 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004202407221664995, + "loss": 1.5408, + "theoretical_loss": 3.9005632123056984, + "tokens_seen": 554172416 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004202306920762287, + "loss": 1.4899, + "theoretical_loss": 3.9005147655925008, + "tokens_seen": 554237952 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042022066198595787, + "loss": 1.4288, + "theoretical_loss": 3.9004663262113475, + "tokens_seen": 554303488 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004202106318956871, + "loss": 1.5067, + "theoretical_loss": 3.900417894160263, + "tokens_seen": 554369024 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042020060180541623, + "loss": 1.467, + "theoretical_loss": 3.900369469437271, + "tokens_seen": 554434560 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042019057171514547, + "loss": 1.5458, + "theoretical_loss": 3.900321052040397, + "tokens_seen": 554500096 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042018054162487465, + "loss": 1.4663, + "theoretical_loss": 3.900272641967666, + "tokens_seen": 554565632 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042017051153460383, + "loss": 1.5399, + "theoretical_loss": 3.900224239217106, + "tokens_seen": 554631168 + }, + { + "epoch": 0.17, + "learning_rate": 0.000420160481444333, + "loss": 1.5447, + "theoretical_loss": 3.9001758437867435, + "tokens_seen": 554696704 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004201504513540622, + "loss": 1.4918, + "theoretical_loss": 3.900127455674608, + "tokens_seen": 554762240 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004201404212637914, + "loss": 1.5805, + "theoretical_loss": 3.9000790748787275, + "tokens_seen": 554827776 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004201303911735206, + "loss": 1.5652, + "theoretical_loss": 3.9000307013971325, + "tokens_seen": 554893312 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042012036108324974, + "loss": 1.364, + "theoretical_loss": 3.899982335227852, + "tokens_seen": 554958848 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042011033099297897, + "loss": 1.4156, + "theoretical_loss": 3.89993397636892, + "tokens_seen": 555024384 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004201003009027081, + "loss": 1.4637, + "theoretical_loss": 3.899885624818366, + "tokens_seen": 555089920 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042009027081243733, + "loss": 1.4162, + "theoretical_loss": 3.8998372805742245, + "tokens_seen": 555155456 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004200802407221665, + "loss": 1.4441, + "theoretical_loss": 3.8997889436345283, + "tokens_seen": 555220992 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004200702106318957, + "loss": 1.3919, + "theoretical_loss": 3.899740613997312, + "tokens_seen": 555286528 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004200601805416249, + "loss": 1.3719, + "theoretical_loss": 3.8996922916606103, + "tokens_seen": 555352064 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004200501504513541, + "loss": 1.382, + "theoretical_loss": 3.89964397662246, + "tokens_seen": 555417600 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042004012036108324, + "loss": 1.4304, + "theoretical_loss": 3.899595668880897, + "tokens_seen": 555483136 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004200300902708125, + "loss": 1.4654, + "theoretical_loss": 3.899547368433959, + "tokens_seen": 555548672 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004200200601805416, + "loss": 1.496, + "theoretical_loss": 3.899499075279684, + "tokens_seen": 555614208 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042001003009027084, + "loss": 1.4807, + "theoretical_loss": 3.899450789416111, + "tokens_seen": 555679744 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042, + "loss": 1.4342, + "theoretical_loss": 3.8994025108412793, + "tokens_seen": 555745280 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004199899699097292, + "loss": 1.3671, + "theoretical_loss": 3.8993542395532304, + "tokens_seen": 555810816 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004199799398194584, + "loss": 1.5283, + "theoretical_loss": 3.8993059755500044, + "tokens_seen": 555876352 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041996990972918756, + "loss": 1.39, + "theoretical_loss": 3.8992577188296433, + "tokens_seen": 555941888 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041995987963891674, + "loss": 1.4494, + "theoretical_loss": 3.8992094693901906, + "tokens_seen": 556007424 + }, + { + "epoch": 0.17, + "learning_rate": 0.000419949849548646, + "loss": 1.444, + "theoretical_loss": 3.8991612272296896, + "tokens_seen": 556072960 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004199398194583751, + "loss": 1.4474, + "theoretical_loss": 3.899112992346184, + "tokens_seen": 556138496 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041992978936810434, + "loss": 1.4256, + "theoretical_loss": 3.899064764737719, + "tokens_seen": 556204032 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041991975927783347, + "loss": 1.4984, + "theoretical_loss": 3.8990165444023406, + "tokens_seen": 556269568 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004199097291875627, + "loss": 1.381, + "theoretical_loss": 3.8989683313380947, + "tokens_seen": 556335104 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004198996990972919, + "loss": 1.3897, + "theoretical_loss": 3.8989201255430292, + "tokens_seen": 556400640 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041988966900702106, + "loss": 1.4931, + "theoretical_loss": 3.8988719270151924, + "tokens_seen": 556466176 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041987963891675025, + "loss": 1.5214, + "theoretical_loss": 3.8988237357526314, + "tokens_seen": 556531712 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004198696088264795, + "loss": 1.4516, + "theoretical_loss": 3.8987755517533973, + "tokens_seen": 556597248 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004198595787362086, + "loss": 1.4749, + "theoretical_loss": 3.8987273750155396, + "tokens_seen": 556662784 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041984954864593784, + "loss": 1.4254, + "theoretical_loss": 3.89867920553711, + "tokens_seen": 556728320 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041983951855566697, + "loss": 1.5404, + "theoretical_loss": 3.8986310433161595, + "tokens_seen": 556793856 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004198294884653962, + "loss": 1.4248, + "theoretical_loss": 3.898582888350741, + "tokens_seen": 556859392 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004198194583751254, + "loss": 1.4614, + "theoretical_loss": 3.8985347406389077, + "tokens_seen": 556924928 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041980942828485457, + "loss": 1.5843, + "theoretical_loss": 3.898486600178714, + "tokens_seen": 556990464 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 190997, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.521323561668396, + "objective/train/theoretical_loss": 3.898438466968214, + "objective/train/tokens_used": 557056000, + "theoretical_loss": 3.898438466968214, + "tokens_seen": 557056000 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041979939819458375, + "loss": 1.5035, + "theoretical_loss": 3.898438466968214, + "tokens_seen": 557056000 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041978936810431293, + "loss": 1.4621, + "theoretical_loss": 3.8983903410054634, + "tokens_seen": 557121536 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004197793380140421, + "loss": 1.4883, + "theoretical_loss": 3.8983422222885187, + "tokens_seen": 557187072 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041976930792377135, + "loss": 1.4448, + "theoretical_loss": 3.8982941108154368, + "tokens_seen": 557252608 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041975927783350047, + "loss": 1.4647, + "theoretical_loss": 3.8982460065842757, + "tokens_seen": 557318144 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004197492477432297, + "loss": 1.4935, + "theoretical_loss": 3.8981979095930934, + "tokens_seen": 557383680 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041973921765295883, + "loss": 1.4477, + "theoretical_loss": 3.89814981983995, + "tokens_seen": 557449216 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041972918756268807, + "loss": 1.4762, + "theoretical_loss": 3.8981017373229045, + "tokens_seen": 557514752 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041971915747241725, + "loss": 1.5055, + "theoretical_loss": 3.8980536620400184, + "tokens_seen": 557580288 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041970912738214643, + "loss": 1.394, + "theoretical_loss": 3.8980055939893523, + "tokens_seen": 557645824 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004196990972918756, + "loss": 1.3915, + "theoretical_loss": 3.89795753316897, + "tokens_seen": 557711360 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041968906720160485, + "loss": 1.3778, + "theoretical_loss": 3.8979094795769336, + "tokens_seen": 557776896 + }, + { + "epoch": 0.17, + "learning_rate": 0.000419679037111334, + "loss": 1.4665, + "theoretical_loss": 3.8978614332113066, + "tokens_seen": 557842432 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004196690070210632, + "loss": 1.4023, + "theoretical_loss": 3.8978133940701536, + "tokens_seen": 557907968 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004196589769307924, + "loss": 1.4548, + "theoretical_loss": 3.8977653621515405, + "tokens_seen": 557973504 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004196489468405216, + "loss": 1.3504, + "theoretical_loss": 3.8977173374535328, + "tokens_seen": 558039040 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004196389167502508, + "loss": 1.5248, + "theoretical_loss": 3.8976693199741974, + "tokens_seen": 558104576 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041962888665997994, + "loss": 1.4346, + "theoretical_loss": 3.8976213097116013, + "tokens_seen": 558170112 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041961885656970917, + "loss": 1.4368, + "theoretical_loss": 3.8975733066638134, + "tokens_seen": 558235648 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004196088264794383, + "loss": 1.5135, + "theoretical_loss": 3.897525310828902, + "tokens_seen": 558301184 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041959879638916753, + "loss": 1.4877, + "theoretical_loss": 3.8974773222049377, + "tokens_seen": 558366720 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004195887662988967, + "loss": 1.5823, + "theoretical_loss": 3.89742934078999, + "tokens_seen": 558432256 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004195787362086259, + "loss": 1.3955, + "theoretical_loss": 3.8973813665821306, + "tokens_seen": 558497792 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004195687061183551, + "loss": 1.4099, + "theoretical_loss": 3.8973333995794315, + "tokens_seen": 558563328 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004195586760280843, + "loss": 1.4929, + "theoretical_loss": 3.897285439779965, + "tokens_seen": 558628864 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041954864593781344, + "loss": 1.4124, + "theoretical_loss": 3.8972374871818047, + "tokens_seen": 558694400 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004195386158475427, + "loss": 1.3908, + "theoretical_loss": 3.8971895417830247, + "tokens_seen": 558759936 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004195285857572718, + "loss": 1.3882, + "theoretical_loss": 3.8971416035817006, + "tokens_seen": 558825472 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041951855566700104, + "loss": 1.4259, + "theoretical_loss": 3.8970936725759073, + "tokens_seen": 558891008 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004195085255767302, + "loss": 1.5123, + "theoretical_loss": 3.8970457487637216, + "tokens_seen": 558956544 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004194984954864594, + "loss": 1.4624, + "theoretical_loss": 3.89699783214322, + "tokens_seen": 559022080 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004194884653961886, + "loss": 1.4567, + "theoretical_loss": 3.8969499227124804, + "tokens_seen": 559087616 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041947843530591776, + "loss": 1.4804, + "theoretical_loss": 3.8969020204695823, + "tokens_seen": 559153152 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041946840521564694, + "loss": 1.5391, + "theoretical_loss": 3.8968541254126046, + "tokens_seen": 559218688 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004194583751253762, + "loss": 1.5096, + "theoretical_loss": 3.896806237539627, + "tokens_seen": 559284224 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004194483450351053, + "loss": 1.4875, + "theoretical_loss": 3.8967583568487307, + "tokens_seen": 559349760 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041943831494483454, + "loss": 1.4157, + "theoretical_loss": 3.8967104833379973, + "tokens_seen": 559415296 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041942828485456367, + "loss": 1.3982, + "theoretical_loss": 3.8966626170055085, + "tokens_seen": 559480832 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004194182547642929, + "loss": 1.5438, + "theoretical_loss": 3.896614757849348, + "tokens_seen": 559546368 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004194082246740221, + "loss": 1.4495, + "theoretical_loss": 3.8965669058675996, + "tokens_seen": 559611904 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041939819458375126, + "loss": 1.4759, + "theoretical_loss": 3.8965190610583473, + "tokens_seen": 559677440 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041938816449348045, + "loss": 1.576, + "theoretical_loss": 3.896471223419677, + "tokens_seen": 559742976 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004193781344032097, + "loss": 1.521, + "theoretical_loss": 3.896423392949674, + "tokens_seen": 559808512 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004193681043129388, + "loss": 1.4284, + "theoretical_loss": 3.896375569646425, + "tokens_seen": 559874048 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041935807422266804, + "loss": 1.5355, + "theoretical_loss": 3.896327753508018, + "tokens_seen": 559939584 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041934804413239717, + "loss": 1.4855, + "theoretical_loss": 3.896279944532541, + "tokens_seen": 560005120 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004193380140421264, + "loss": 1.2959, + "theoretical_loss": 3.8962321427180826, + "tokens_seen": 560070656 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004193279839518556, + "loss": 1.4464, + "theoretical_loss": 3.8961843480627323, + "tokens_seen": 560136192 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041931795386158477, + "loss": 1.4211, + "theoretical_loss": 3.8961365605645817, + "tokens_seen": 560201728 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041930792377131395, + "loss": 1.451, + "theoretical_loss": 3.89608878022172, + "tokens_seen": 560267264 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 191892, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5365924835205078, + "objective/train/theoretical_loss": 3.896041007032241, + "objective/train/tokens_used": 560332800, + "theoretical_loss": 3.896041007032241, + "tokens_seen": 560332800 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041929789368104313, + "loss": 1.4245, + "theoretical_loss": 3.896041007032241, + "tokens_seen": 560332800 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004192878635907723, + "loss": 1.4961, + "theoretical_loss": 3.895993240994236, + "tokens_seen": 560398336 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041927783350050155, + "loss": 1.4472, + "theoretical_loss": 3.895945482105798, + "tokens_seen": 560463872 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041926780341023067, + "loss": 1.4747, + "theoretical_loss": 3.8958977303650224, + "tokens_seen": 560529408 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004192577733199599, + "loss": 1.4148, + "theoretical_loss": 3.8958499857700035, + "tokens_seen": 560594944 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041924774322968904, + "loss": 1.4764, + "theoretical_loss": 3.8958022483188364, + "tokens_seen": 560660480 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041923771313941827, + "loss": 1.3992, + "theoretical_loss": 3.895754518009617, + "tokens_seen": 560726016 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041922768304914745, + "loss": 1.5068, + "theoretical_loss": 3.8957067948404434, + "tokens_seen": 560791552 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041921765295887663, + "loss": 1.433, + "theoretical_loss": 3.895659078809412, + "tokens_seen": 560857088 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004192076228686058, + "loss": 1.4053, + "theoretical_loss": 3.895611369914622, + "tokens_seen": 560922624 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041919759277833505, + "loss": 1.4003, + "theoretical_loss": 3.895563668154173, + "tokens_seen": 560988160 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004191875626880642, + "loss": 1.4824, + "theoretical_loss": 3.8955159735261637, + "tokens_seen": 561053696 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004191775325977934, + "loss": 1.43, + "theoretical_loss": 3.895468286028695, + "tokens_seen": 561119232 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041916750250752254, + "loss": 1.2923, + "theoretical_loss": 3.8954206056598695, + "tokens_seen": 561184768 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004191574724172518, + "loss": 1.5176, + "theoretical_loss": 3.8953729324177875, + "tokens_seen": 561250304 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041914744232698095, + "loss": 1.4086, + "theoretical_loss": 3.895325266300553, + "tokens_seen": 561315840 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041913741223671014, + "loss": 1.4542, + "theoretical_loss": 3.8952776073062685, + "tokens_seen": 561381376 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004191273821464393, + "loss": 1.4824, + "theoretical_loss": 3.895229955433039, + "tokens_seen": 561446912 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004191173520561685, + "loss": 1.4965, + "theoretical_loss": 3.8951823106789694, + "tokens_seen": 561512448 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004191073219658977, + "loss": 1.4254, + "theoretical_loss": 3.895134673042165, + "tokens_seen": 561577984 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004190972918756269, + "loss": 1.4911, + "theoretical_loss": 3.895087042520733, + "tokens_seen": 561643520 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041908726178535604, + "loss": 1.4439, + "theoretical_loss": 3.895039419112779, + "tokens_seen": 561709056 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004190772316950853, + "loss": 1.4659, + "theoretical_loss": 3.8949918028164126, + "tokens_seen": 561774592 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004190672016048144, + "loss": 1.476, + "theoretical_loss": 3.8949441936297413, + "tokens_seen": 561840128 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041905717151454364, + "loss": 1.4737, + "theoretical_loss": 3.894896591550875, + "tokens_seen": 561905664 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004190471414242728, + "loss": 1.5186, + "theoretical_loss": 3.894848996577924, + "tokens_seen": 561971200 + }, + { + "epoch": 0.17, + "learning_rate": 0.000419037111334002, + "loss": 1.4554, + "theoretical_loss": 3.8948014087089975, + "tokens_seen": 562036736 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004190270812437312, + "loss": 1.4601, + "theoretical_loss": 3.8947538279422083, + "tokens_seen": 562102272 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004190170511534604, + "loss": 1.3405, + "theoretical_loss": 3.8947062542756683, + "tokens_seen": 562167808 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041900702106318954, + "loss": 1.5528, + "theoretical_loss": 3.8946586877074907, + "tokens_seen": 562233344 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004189969909729188, + "loss": 1.3123, + "theoretical_loss": 3.894611128235789, + "tokens_seen": 562298880 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004189869608826479, + "loss": 1.505, + "theoretical_loss": 3.894563575858677, + "tokens_seen": 562364416 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041897693079237714, + "loss": 1.5115, + "theoretical_loss": 3.8945160305742705, + "tokens_seen": 562429952 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004189669007021063, + "loss": 1.4485, + "theoretical_loss": 3.8944684923806845, + "tokens_seen": 562495488 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004189568706118355, + "loss": 1.4873, + "theoretical_loss": 3.8944209612760368, + "tokens_seen": 562561024 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004189468405215647, + "loss": 1.465, + "theoretical_loss": 3.894373437258443, + "tokens_seen": 562626560 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041893681043129387, + "loss": 1.4858, + "theoretical_loss": 3.8943259203260228, + "tokens_seen": 562692096 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041892678034102305, + "loss": 1.4097, + "theoretical_loss": 3.8942784104768937, + "tokens_seen": 562757632 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004189167502507523, + "loss": 1.3253, + "theoretical_loss": 3.8942309077091757, + "tokens_seen": 562823168 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041890672016048146, + "loss": 1.4257, + "theoretical_loss": 3.894183412020988, + "tokens_seen": 562888704 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041889669007021065, + "loss": 1.4381, + "theoretical_loss": 3.894135923410453, + "tokens_seen": 562954240 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004188866599799399, + "loss": 1.4311, + "theoretical_loss": 3.8940884418756907, + "tokens_seen": 563019776 + }, + { + "epoch": 0.17, + "learning_rate": 0.000418876629889669, + "loss": 1.5842, + "theoretical_loss": 3.894040967414824, + "tokens_seen": 563085312 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041886659979939824, + "loss": 1.4686, + "theoretical_loss": 3.893993500025976, + "tokens_seen": 563150848 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041885656970912737, + "loss": 1.4599, + "theoretical_loss": 3.893946039707271, + "tokens_seen": 563216384 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004188465396188566, + "loss": 1.4697, + "theoretical_loss": 3.893898586456832, + "tokens_seen": 563281920 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004188365095285858, + "loss": 1.2958, + "theoretical_loss": 3.893851140272785, + "tokens_seen": 563347456 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041882647943831497, + "loss": 1.3972, + "theoretical_loss": 3.8938037011532556, + "tokens_seen": 563412992 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041881644934804415, + "loss": 1.3016, + "theoretical_loss": 3.893756269096371, + "tokens_seen": 563478528 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041880641925777333, + "loss": 1.4171, + "theoretical_loss": 3.8937088441002574, + "tokens_seen": 563544064 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 193079, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3910335302352905, + "objective/train/theoretical_loss": 3.893661426163044, + "objective/train/tokens_used": 563609600, + "theoretical_loss": 3.893661426163044, + "tokens_seen": 563609600 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004187963891675025, + "loss": 1.4646, + "theoretical_loss": 3.893661426163044, + "tokens_seen": 563609600 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041878635907723175, + "loss": 1.4666, + "theoretical_loss": 3.8936140152828584, + "tokens_seen": 563675136 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004187763289869609, + "loss": 1.4295, + "theoretical_loss": 3.8935666114578305, + "tokens_seen": 563740672 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004187662988966901, + "loss": 1.5348, + "theoretical_loss": 3.8935192146860906, + "tokens_seen": 563806208 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041875626880641924, + "loss": 1.4774, + "theoretical_loss": 3.8934718249657694, + "tokens_seen": 563871744 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041874623871614847, + "loss": 1.3944, + "theoretical_loss": 3.8934244422949984, + "tokens_seen": 563937280 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041873620862587765, + "loss": 1.369, + "theoretical_loss": 3.89337706667191, + "tokens_seen": 564002816 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041872617853560683, + "loss": 1.5223, + "theoretical_loss": 3.8933296980946372, + "tokens_seen": 564068352 + }, + { + "epoch": 0.17, + "learning_rate": 0.000418716148445336, + "loss": 1.4797, + "theoretical_loss": 3.8932823365613136, + "tokens_seen": 564133888 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041870611835506525, + "loss": 1.3879, + "theoretical_loss": 3.893234982070074, + "tokens_seen": 564199424 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004186960882647944, + "loss": 1.3277, + "theoretical_loss": 3.8931876346190517, + "tokens_seen": 564264960 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004186860581745236, + "loss": 1.5044, + "theoretical_loss": 3.8931402942063853, + "tokens_seen": 564330496 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041867602808425274, + "loss": 1.3862, + "theoretical_loss": 3.89309296083021, + "tokens_seen": 564396032 + }, + { + "epoch": 0.17, + "learning_rate": 0.000418665997993982, + "loss": 1.5632, + "theoretical_loss": 3.8930456344886624, + "tokens_seen": 564461568 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041865596790371115, + "loss": 1.3904, + "theoretical_loss": 3.8929983151798817, + "tokens_seen": 564527104 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041864593781344034, + "loss": 1.4826, + "theoretical_loss": 3.892951002902006, + "tokens_seen": 564592640 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004186359077231695, + "loss": 1.4306, + "theoretical_loss": 3.892903697653174, + "tokens_seen": 564658176 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004186258776328987, + "loss": 1.4147, + "theoretical_loss": 3.8928563994315266, + "tokens_seen": 564723712 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004186158475426279, + "loss": 1.4349, + "theoretical_loss": 3.8928091082352045, + "tokens_seen": 564789248 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004186058174523571, + "loss": 1.3669, + "theoretical_loss": 3.8927618240623496, + "tokens_seen": 564854784 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041859578736208624, + "loss": 1.3145, + "theoretical_loss": 3.8927145469111033, + "tokens_seen": 564920320 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004185857572718155, + "loss": 1.5239, + "theoretical_loss": 3.892667276779609, + "tokens_seen": 564985856 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004185757271815446, + "loss": 1.3883, + "theoretical_loss": 3.89262001366601, + "tokens_seen": 565051392 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041856569709127384, + "loss": 1.4531, + "theoretical_loss": 3.892572757568451, + "tokens_seen": 565116928 + }, + { + "epoch": 0.17, + "learning_rate": 0.000418555667001003, + "loss": 1.575, + "theoretical_loss": 3.892525508485077, + "tokens_seen": 565182464 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004185456369107322, + "loss": 1.4814, + "theoretical_loss": 3.892478266414033, + "tokens_seen": 565248000 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004185356068204614, + "loss": 1.342, + "theoretical_loss": 3.8924310313534667, + "tokens_seen": 565313536 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004185255767301906, + "loss": 1.4146, + "theoretical_loss": 3.892383803301524, + "tokens_seen": 565379072 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041851554663991974, + "loss": 1.4158, + "theoretical_loss": 3.8923365822563536, + "tokens_seen": 565444608 + }, + { + "epoch": 0.17, + "learning_rate": 0.000418505516549649, + "loss": 1.496, + "theoretical_loss": 3.8922893682161037, + "tokens_seen": 565510144 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004184954864593781, + "loss": 1.4547, + "theoretical_loss": 3.892242161178924, + "tokens_seen": 565575680 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041848545636910734, + "loss": 1.4994, + "theoretical_loss": 3.892194961142964, + "tokens_seen": 565641216 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004184754262788365, + "loss": 1.4802, + "theoretical_loss": 3.8921477681063736, + "tokens_seen": 565706752 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004184653961885657, + "loss": 1.5611, + "theoretical_loss": 3.892100582067306, + "tokens_seen": 565772288 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004184553660982949, + "loss": 1.3912, + "theoretical_loss": 3.8920534030239122, + "tokens_seen": 565837824 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041844533600802407, + "loss": 1.3394, + "theoretical_loss": 3.892006230974345, + "tokens_seen": 565903360 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041843530591775325, + "loss": 1.5138, + "theoretical_loss": 3.8919590659167578, + "tokens_seen": 565968896 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004184252758274825, + "loss": 1.5266, + "theoretical_loss": 3.8919119078493045, + "tokens_seen": 566034432 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004184152457372116, + "loss": 1.472, + "theoretical_loss": 3.8918647567701417, + "tokens_seen": 566099968 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041840521564694085, + "loss": 1.4513, + "theoretical_loss": 3.8918176126774227, + "tokens_seen": 566165504 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041839518555666997, + "loss": 1.5502, + "theoretical_loss": 3.891770475569305, + "tokens_seen": 566231040 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004183851554663992, + "loss": 1.5728, + "theoretical_loss": 3.8917233454439453, + "tokens_seen": 566296576 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004183751253761284, + "loss": 1.519, + "theoretical_loss": 3.8916762222995014, + "tokens_seen": 566362112 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041836509528585757, + "loss": 1.3648, + "theoretical_loss": 3.8916291061341317, + "tokens_seen": 566427648 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041835506519558675, + "loss": 1.487, + "theoretical_loss": 3.8915819969459946, + "tokens_seen": 566493184 + }, + { + "epoch": 0.17, + "learning_rate": 0.000418345035105316, + "loss": 1.4861, + "theoretical_loss": 3.891534894733251, + "tokens_seen": 566558720 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004183350050150451, + "loss": 1.4165, + "theoretical_loss": 3.891487799494061, + "tokens_seen": 566624256 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041832497492477435, + "loss": 1.4031, + "theoretical_loss": 3.891440711226585, + "tokens_seen": 566689792 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004183149448345035, + "loss": 1.4752, + "theoretical_loss": 3.891393629928986, + "tokens_seen": 566755328 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004183049147442327, + "loss": 1.3867, + "theoretical_loss": 3.8913465555994255, + "tokens_seen": 566820864 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 193993, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5188324451446533, + "objective/train/theoretical_loss": 3.8912994882360676, + "objective/train/tokens_used": 566886400, + "theoretical_loss": 3.8912994882360676, + "tokens_seen": 566886400 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004182948846539619, + "loss": 1.4217, + "theoretical_loss": 3.8912994882360676, + "tokens_seen": 566886400 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004182848545636911, + "loss": 1.4623, + "theoretical_loss": 3.891252427837076, + "tokens_seen": 566951936 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041827482447342025, + "loss": 1.4365, + "theoretical_loss": 3.8912053744006156, + "tokens_seen": 567017472 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041826479438314944, + "loss": 1.3639, + "theoretical_loss": 3.8911583279248507, + "tokens_seen": 567083008 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004182547642928786, + "loss": 1.5982, + "theoretical_loss": 3.8911112884079486, + "tokens_seen": 567148544 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041824473420260785, + "loss": 1.5736, + "theoretical_loss": 3.8910642558480752, + "tokens_seen": 567214080 + }, + { + "epoch": 0.17, + "learning_rate": 0.000418234704112337, + "loss": 1.3947, + "theoretical_loss": 3.8910172302433987, + "tokens_seen": 567279616 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004182246740220662, + "loss": 1.5154, + "theoretical_loss": 3.8909702115920863, + "tokens_seen": 567345152 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004182146439317954, + "loss": 1.5076, + "theoretical_loss": 3.890923199892308, + "tokens_seen": 567410688 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004182046138415246, + "loss": 1.4271, + "theoretical_loss": 3.890876195142232, + "tokens_seen": 567476224 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041819458375125376, + "loss": 1.4721, + "theoretical_loss": 3.8908291973400297, + "tokens_seen": 567541760 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041818455366098294, + "loss": 1.4991, + "theoretical_loss": 3.890782206483871, + "tokens_seen": 567607296 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004181745235707121, + "loss": 1.4156, + "theoretical_loss": 3.8907352225719283, + "tokens_seen": 567672832 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041816449348044136, + "loss": 1.352, + "theoretical_loss": 3.890688245602373, + "tokens_seen": 567738368 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041815446339017054, + "loss": 1.4601, + "theoretical_loss": 3.890641275573379, + "tokens_seen": 567803904 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004181444332998997, + "loss": 1.4399, + "theoretical_loss": 3.8905943124831195, + "tokens_seen": 567869440 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004181344032096289, + "loss": 1.3953, + "theoretical_loss": 3.890547356329769, + "tokens_seen": 567934976 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004181243731193581, + "loss": 1.5142, + "theoretical_loss": 3.890500407111502, + "tokens_seen": 568000512 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004181143430290873, + "loss": 1.5073, + "theoretical_loss": 3.8904534648264955, + "tokens_seen": 568066048 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041810431293881644, + "loss": 1.3426, + "theoretical_loss": 3.8904065294729246, + "tokens_seen": 568131584 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004180942828485457, + "loss": 1.4532, + "theoretical_loss": 3.890359601048967, + "tokens_seen": 568197120 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004180842527582748, + "loss": 1.4426, + "theoretical_loss": 3.8903126795528014, + "tokens_seen": 568262656 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041807422266800404, + "loss": 1.3762, + "theoretical_loss": 3.8902657649826047, + "tokens_seen": 568328192 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004180641925777332, + "loss": 1.4356, + "theoretical_loss": 3.890218857336557, + "tokens_seen": 568393728 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004180541624874624, + "loss": 1.4972, + "theoretical_loss": 3.890171956612838, + "tokens_seen": 568459264 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004180441323971916, + "loss": 1.3311, + "theoretical_loss": 3.890125062809628, + "tokens_seen": 568524800 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004180341023069208, + "loss": 1.4939, + "theoretical_loss": 3.8900781759251086, + "tokens_seen": 568590336 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041802407221664994, + "loss": 1.4495, + "theoretical_loss": 3.8900312959574617, + "tokens_seen": 568655872 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004180140421263792, + "loss": 1.451, + "theoretical_loss": 3.88998442290487, + "tokens_seen": 568721408 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004180040120361083, + "loss": 1.3728, + "theoretical_loss": 3.889937556765517, + "tokens_seen": 568786944 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041799398194583754, + "loss": 1.3386, + "theoretical_loss": 3.8898906975375858, + "tokens_seen": 568852480 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004179839518555667, + "loss": 1.3639, + "theoretical_loss": 3.8898438452192616, + "tokens_seen": 568918016 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004179739217652959, + "loss": 1.3633, + "theoretical_loss": 3.8897969998087305, + "tokens_seen": 568983552 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004179638916750251, + "loss": 1.4225, + "theoretical_loss": 3.8897501613041783, + "tokens_seen": 569049088 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041795386158475427, + "loss": 1.5082, + "theoretical_loss": 3.8897033297037904, + "tokens_seen": 569114624 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041794383149448345, + "loss": 1.4961, + "theoretical_loss": 3.889656505005756, + "tokens_seen": 569180160 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004179338014042127, + "loss": 1.4274, + "theoretical_loss": 3.889609687208262, + "tokens_seen": 569245696 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004179237713139418, + "loss": 1.4734, + "theoretical_loss": 3.8895628763094976, + "tokens_seen": 569311232 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041791374122367105, + "loss": 1.5066, + "theoretical_loss": 3.8895160723076527, + "tokens_seen": 569376768 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041790371113340017, + "loss": 1.3257, + "theoretical_loss": 3.889469275200917, + "tokens_seen": 569442304 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004178936810431294, + "loss": 1.4416, + "theoretical_loss": 3.889422484987481, + "tokens_seen": 569507840 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004178836509528586, + "loss": 1.3851, + "theoretical_loss": 3.889375701665537, + "tokens_seen": 569573376 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041787362086258777, + "loss": 1.3607, + "theoretical_loss": 3.8893289252332766, + "tokens_seen": 569638912 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041786359077231695, + "loss": 1.4712, + "theoretical_loss": 3.8892821556888935, + "tokens_seen": 569704448 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004178535606820462, + "loss": 1.3418, + "theoretical_loss": 3.8892353930305807, + "tokens_seen": 569769984 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004178435305917753, + "loss": 1.3118, + "theoretical_loss": 3.8891886372565327, + "tokens_seen": 569835520 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041783350050150455, + "loss": 1.4396, + "theoretical_loss": 3.889141888364944, + "tokens_seen": 569901056 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004178234704112337, + "loss": 1.3573, + "theoretical_loss": 3.8890951463540104, + "tokens_seen": 569966592 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004178134403209629, + "loss": 1.5023, + "theoretical_loss": 3.8890484112219283, + "tokens_seen": 570032128 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004178034102306921, + "loss": 1.4827, + "theoretical_loss": 3.889001682966895, + "tokens_seen": 570097664 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 195305, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4108757972717285, + "objective/train/theoretical_loss": 3.8889549615871077, + "objective/train/tokens_used": 570163200, + "theoretical_loss": 3.8889549615871077, + "tokens_seen": 570163200 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004177933801404213, + "loss": 1.4429, + "theoretical_loss": 3.8889549615871077, + "tokens_seen": 570163200 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041778335005015045, + "loss": 1.4074, + "theoretical_loss": 3.8889082470807645, + "tokens_seen": 570228736 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041777331995987964, + "loss": 1.4414, + "theoretical_loss": 3.888861539446066, + "tokens_seen": 570294272 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004177632898696088, + "loss": 1.3048, + "theoretical_loss": 3.88881483868121, + "tokens_seen": 570359808 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041775325977933805, + "loss": 1.4334, + "theoretical_loss": 3.8887681447843976, + "tokens_seen": 570425344 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004177432296890672, + "loss": 1.4488, + "theoretical_loss": 3.88872145775383, + "tokens_seen": 570490880 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004177331995987964, + "loss": 1.3664, + "theoretical_loss": 3.8886747775877084, + "tokens_seen": 570556416 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004177231695085256, + "loss": 1.3981, + "theoretical_loss": 3.888628104284236, + "tokens_seen": 570621952 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004177131394182548, + "loss": 1.4461, + "theoretical_loss": 3.888581437841615, + "tokens_seen": 570687488 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041770310932798396, + "loss": 1.2581, + "theoretical_loss": 3.8885347782580504, + "tokens_seen": 570753024 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041769307923771314, + "loss": 1.4039, + "theoretical_loss": 3.8884881255317456, + "tokens_seen": 570818560 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004176830491474423, + "loss": 1.3824, + "theoretical_loss": 3.888441479660906, + "tokens_seen": 570884096 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041767301905717156, + "loss": 1.3595, + "theoretical_loss": 3.8883948406437376, + "tokens_seen": 570949632 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004176629889669007, + "loss": 1.4725, + "theoretical_loss": 3.8883482084784466, + "tokens_seen": 571015168 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004176529588766299, + "loss": 1.4592, + "theoretical_loss": 3.8883015831632406, + "tokens_seen": 571080704 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041764292878635904, + "loss": 1.4905, + "theoretical_loss": 3.8882549646963263, + "tokens_seen": 571146240 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004176328986960883, + "loss": 1.4841, + "theoretical_loss": 3.888208353075914, + "tokens_seen": 571211776 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041762286860581746, + "loss": 1.4777, + "theoretical_loss": 3.8881617483002113, + "tokens_seen": 571277312 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041761283851554664, + "loss": 1.4181, + "theoretical_loss": 3.8881151503674283, + "tokens_seen": 571342848 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004176028084252758, + "loss": 1.3196, + "theoretical_loss": 3.888068559275776, + "tokens_seen": 571408384 + }, + { + "epoch": 0.17, + "learning_rate": 0.000417592778335005, + "loss": 1.3955, + "theoretical_loss": 3.8880219750234657, + "tokens_seen": 571473920 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004175827482447342, + "loss": 1.3787, + "theoretical_loss": 3.8879753976087086, + "tokens_seen": 571539456 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004175727181544634, + "loss": 1.4912, + "theoretical_loss": 3.8879288270297176, + "tokens_seen": 571604992 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041756268806419255, + "loss": 1.4167, + "theoretical_loss": 3.8878822632847063, + "tokens_seen": 571670528 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004175526579739218, + "loss": 1.4655, + "theoretical_loss": 3.887835706371888, + "tokens_seen": 571736064 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041754262788365096, + "loss": 1.4703, + "theoretical_loss": 3.887789156289477, + "tokens_seen": 571801600 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041753259779338015, + "loss": 1.4144, + "theoretical_loss": 3.887742613035689, + "tokens_seen": 571867136 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004175225677031093, + "loss": 1.5024, + "theoretical_loss": 3.8876960766087407, + "tokens_seen": 571932672 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004175125376128385, + "loss": 1.4499, + "theoretical_loss": 3.8876495470068475, + "tokens_seen": 571998208 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004175025075225677, + "loss": 1.3464, + "theoretical_loss": 3.8876030242282265, + "tokens_seen": 572063744 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004174924774322969, + "loss": 1.5011, + "theoretical_loss": 3.8875565082710963, + "tokens_seen": 572129280 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041748244734202605, + "loss": 1.414, + "theoretical_loss": 3.887509999133675, + "tokens_seen": 572194816 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004174724172517553, + "loss": 1.4704, + "theoretical_loss": 3.887463496814182, + "tokens_seen": 572260352 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004174623871614844, + "loss": 1.4058, + "theoretical_loss": 3.887417001310838, + "tokens_seen": 572325888 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041745235707121365, + "loss": 1.4398, + "theoretical_loss": 3.887370512621862, + "tokens_seen": 572391424 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041744232698094283, + "loss": 1.4352, + "theoretical_loss": 3.887324030745477, + "tokens_seen": 572456960 + }, + { + "epoch": 0.17, + "learning_rate": 0.000417432296890672, + "loss": 1.4225, + "theoretical_loss": 3.887277555679903, + "tokens_seen": 572522496 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004174222668004012, + "loss": 1.3508, + "theoretical_loss": 3.8872310874233644, + "tokens_seen": 572588032 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041741223671013037, + "loss": 1.4059, + "theoretical_loss": 3.887184625974083, + "tokens_seen": 572653568 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004174022066198596, + "loss": 1.4573, + "theoretical_loss": 3.8871381713302835, + "tokens_seen": 572719104 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004173921765295888, + "loss": 1.4035, + "theoretical_loss": 3.8870917234901907, + "tokens_seen": 572784640 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041738214643931797, + "loss": 1.3933, + "theoretical_loss": 3.8870452824520294, + "tokens_seen": 572850176 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041737211634904715, + "loss": 1.2978, + "theoretical_loss": 3.8869988482140254, + "tokens_seen": 572915712 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004173620862587764, + "loss": 1.4394, + "theoretical_loss": 3.886952420774406, + "tokens_seen": 572981248 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004173520561685055, + "loss": 1.476, + "theoretical_loss": 3.8869060001313973, + "tokens_seen": 573046784 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041734202607823475, + "loss": 1.4502, + "theoretical_loss": 3.8868595862832285, + "tokens_seen": 573112320 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004173319959879639, + "loss": 1.4777, + "theoretical_loss": 3.886813179228127, + "tokens_seen": 573177856 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004173219658976931, + "loss": 1.5198, + "theoretical_loss": 3.886766778964323, + "tokens_seen": 573243392 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004173119358074223, + "loss": 1.4965, + "theoretical_loss": 3.8867203854900456, + "tokens_seen": 573308928 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004173019057171515, + "loss": 1.4589, + "theoretical_loss": 3.886673998803526, + "tokens_seen": 573374464 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 196122, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3662922382354736, + "objective/train/theoretical_loss": 3.886627618902995, + "objective/train/tokens_used": 573440000, + "theoretical_loss": 3.886627618902995, + "tokens_seen": 573440000 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041729187562688065, + "loss": 1.3404, + "theoretical_loss": 3.886627618902995, + "tokens_seen": 573440000 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041728184553660984, + "loss": 1.4517, + "theoretical_loss": 3.886581245786685, + "tokens_seen": 573505536 + }, + { + "epoch": 0.17, + "learning_rate": 0.000417271815446339, + "loss": 1.4091, + "theoretical_loss": 3.8865348794528276, + "tokens_seen": 573571072 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041726178535606825, + "loss": 1.4755, + "theoretical_loss": 3.8864885198996575, + "tokens_seen": 573636608 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004172517552657974, + "loss": 1.3582, + "theoretical_loss": 3.8864421671254066, + "tokens_seen": 573702144 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004172417251755266, + "loss": 1.3572, + "theoretical_loss": 3.8863958211283114, + "tokens_seen": 573767680 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004172316950852558, + "loss": 1.6081, + "theoretical_loss": 3.8863494819066062, + "tokens_seen": 573833216 + }, + { + "epoch": 0.17, + "learning_rate": 0.000417221664994985, + "loss": 1.3177, + "theoretical_loss": 3.8863031494585267, + "tokens_seen": 573898752 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041721163490471416, + "loss": 1.438, + "theoretical_loss": 3.8862568237823103, + "tokens_seen": 573964288 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041720160481444334, + "loss": 1.4784, + "theoretical_loss": 3.8862105048761926, + "tokens_seen": 574029824 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004171915747241725, + "loss": 1.3916, + "theoretical_loss": 3.8861641927384127, + "tokens_seen": 574095360 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041718154463390176, + "loss": 1.4014, + "theoretical_loss": 3.8861178873672095, + "tokens_seen": 574160896 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004171715145436309, + "loss": 1.4372, + "theoretical_loss": 3.886071588760821, + "tokens_seen": 574226432 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004171614844533601, + "loss": 1.3943, + "theoretical_loss": 3.886025296917487, + "tokens_seen": 574291968 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041715145436308924, + "loss": 1.4459, + "theoretical_loss": 3.8859790118354494, + "tokens_seen": 574357504 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004171414242728185, + "loss": 1.3545, + "theoretical_loss": 3.8859327335129477, + "tokens_seen": 574423040 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041713139418254766, + "loss": 1.4132, + "theoretical_loss": 3.8858864619482247, + "tokens_seen": 574488576 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041712136409227684, + "loss": 1.4493, + "theoretical_loss": 3.8858401971395224, + "tokens_seen": 574554112 + }, + { + "epoch": 0.17, + "learning_rate": 0.000417111334002006, + "loss": 1.4441, + "theoretical_loss": 3.885793939085085, + "tokens_seen": 574619648 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004171013039117352, + "loss": 1.3389, + "theoretical_loss": 3.8857476877831543, + "tokens_seen": 574685184 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004170912738214644, + "loss": 1.4475, + "theoretical_loss": 3.8857014432319765, + "tokens_seen": 574750720 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004170812437311936, + "loss": 1.3585, + "theoretical_loss": 3.8856552054297957, + "tokens_seen": 574816256 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041707121364092275, + "loss": 1.4259, + "theoretical_loss": 3.8856089743748576, + "tokens_seen": 574881792 + }, + { + "epoch": 0.17, + "learning_rate": 0.000417061183550652, + "loss": 1.4059, + "theoretical_loss": 3.885562750065409, + "tokens_seen": 574947328 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041705115346038116, + "loss": 1.4112, + "theoretical_loss": 3.8855165324996976, + "tokens_seen": 575012864 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041704112337011035, + "loss": 1.391, + "theoretical_loss": 3.88547032167597, + "tokens_seen": 575078400 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004170310932798395, + "loss": 1.3254, + "theoretical_loss": 3.885424117592475, + "tokens_seen": 575143936 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004170210631895687, + "loss": 1.4535, + "theoretical_loss": 3.885377920247461, + "tokens_seen": 575209472 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004170110330992979, + "loss": 1.5195, + "theoretical_loss": 3.885331729639179, + "tokens_seen": 575275008 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004170010030090271, + "loss": 1.4532, + "theoretical_loss": 3.8852855457658784, + "tokens_seen": 575340544 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041699097291875625, + "loss": 1.4588, + "theoretical_loss": 3.88523936862581, + "tokens_seen": 575406080 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004169809428284855, + "loss": 1.4706, + "theoretical_loss": 3.885193198217226, + "tokens_seen": 575471616 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004169709127382146, + "loss": 1.4221, + "theoretical_loss": 3.8851470345383783, + "tokens_seen": 575537152 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041696088264794385, + "loss": 1.4796, + "theoretical_loss": 3.8851008775875204, + "tokens_seen": 575602688 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041695085255767303, + "loss": 1.411, + "theoretical_loss": 3.8850547273629052, + "tokens_seen": 575668224 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004169408224674022, + "loss": 1.3593, + "theoretical_loss": 3.8850085838627875, + "tokens_seen": 575733760 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004169307923771314, + "loss": 1.5613, + "theoretical_loss": 3.8849624470854223, + "tokens_seen": 575799296 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004169207622868606, + "loss": 1.3838, + "theoretical_loss": 3.8849163170290644, + "tokens_seen": 575864832 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041691073219658975, + "loss": 1.4623, + "theoretical_loss": 3.8848701936919703, + "tokens_seen": 575930368 + }, + { + "epoch": 0.17, + "learning_rate": 0.000416900702106319, + "loss": 1.4351, + "theoretical_loss": 3.8848240770723974, + "tokens_seen": 575995904 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004168906720160481, + "loss": 1.4394, + "theoretical_loss": 3.8847779671686022, + "tokens_seen": 576061440 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041688064192577735, + "loss": 1.3277, + "theoretical_loss": 3.8847318639788435, + "tokens_seen": 576126976 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041687061183550653, + "loss": 1.4495, + "theoretical_loss": 3.8846857675013804, + "tokens_seen": 576192512 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004168605817452357, + "loss": 1.3897, + "theoretical_loss": 3.884639677734471, + "tokens_seen": 576258048 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004168505516549649, + "loss": 1.4901, + "theoretical_loss": 3.884593594676377, + "tokens_seen": 576323584 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004168405215646941, + "loss": 1.3741, + "theoretical_loss": 3.884547518325359, + "tokens_seen": 576389120 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041683049147442326, + "loss": 1.5007, + "theoretical_loss": 3.8845014486796776, + "tokens_seen": 576454656 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004168204613841525, + "loss": 1.494, + "theoretical_loss": 3.8844553857375947, + "tokens_seen": 576520192 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004168104312938816, + "loss": 1.4868, + "theoretical_loss": 3.884409329497374, + "tokens_seen": 576585728 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041680040120361085, + "loss": 1.4751, + "theoretical_loss": 3.884363279957278, + "tokens_seen": 576651264 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 197523, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.496764898300171, + "objective/train/theoretical_loss": 3.884317237115571, + "objective/train/tokens_used": 576716800, + "theoretical_loss": 3.884317237115571, + "tokens_seen": 576716800 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041679037111334, + "loss": 1.4777, + "theoretical_loss": 3.884317237115571, + "tokens_seen": 576716800 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004167803410230692, + "loss": 1.5797, + "theoretical_loss": 3.884271200970517, + "tokens_seen": 576782336 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004167703109327984, + "loss": 1.4721, + "theoretical_loss": 3.8842251715203826, + "tokens_seen": 576847872 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004167602808425276, + "loss": 1.4746, + "theoretical_loss": 3.8841791487634327, + "tokens_seen": 576913408 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041675025075225676, + "loss": 1.468, + "theoretical_loss": 3.8841331326979347, + "tokens_seen": 576978944 + }, + { + "epoch": 0.17, + "learning_rate": 0.000416740220661986, + "loss": 1.5835, + "theoretical_loss": 3.8840871233221548, + "tokens_seen": 577044480 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004167301905717151, + "loss": 1.5283, + "theoretical_loss": 3.8840411206343615, + "tokens_seen": 577110016 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041672016048144436, + "loss": 1.4116, + "theoretical_loss": 3.883995124632823, + "tokens_seen": 577175552 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004167101303911735, + "loss": 1.4561, + "theoretical_loss": 3.8839491353158087, + "tokens_seen": 577241088 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004167001003009027, + "loss": 1.4276, + "theoretical_loss": 3.8839031526815884, + "tokens_seen": 577306624 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004166900702106319, + "loss": 1.4497, + "theoretical_loss": 3.8838571767284327, + "tokens_seen": 577372160 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004166800401203611, + "loss": 1.3806, + "theoretical_loss": 3.883811207454612, + "tokens_seen": 577437696 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041667001003009026, + "loss": 1.5372, + "theoretical_loss": 3.883765244858399, + "tokens_seen": 577503232 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041665997993981944, + "loss": 1.4363, + "theoretical_loss": 3.8837192889380647, + "tokens_seen": 577568768 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004166499498495487, + "loss": 1.3861, + "theoretical_loss": 3.8836733396918834, + "tokens_seen": 577634304 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041663991975927786, + "loss": 1.4269, + "theoretical_loss": 3.8836273971181283, + "tokens_seen": 577699840 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041662988966900704, + "loss": 1.472, + "theoretical_loss": 3.883581461215074, + "tokens_seen": 577765376 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004166198595787362, + "loss": 1.4806, + "theoretical_loss": 3.8835355319809954, + "tokens_seen": 577830912 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004166098294884654, + "loss": 1.4625, + "theoretical_loss": 3.883489609414167, + "tokens_seen": 577896448 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004165997993981946, + "loss": 1.3386, + "theoretical_loss": 3.8834436935128664, + "tokens_seen": 577961984 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004165897693079238, + "loss": 1.382, + "theoretical_loss": 3.8833977842753704, + "tokens_seen": 578027520 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041657973921765295, + "loss": 1.4655, + "theoretical_loss": 3.883351881699956, + "tokens_seen": 578093056 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004165697091273822, + "loss": 1.4099, + "theoretical_loss": 3.883305985784901, + "tokens_seen": 578158592 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041655967903711136, + "loss": 1.3957, + "theoretical_loss": 3.8832600965284847, + "tokens_seen": 578224128 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041654964894684055, + "loss": 1.3857, + "theoretical_loss": 3.8832142139289862, + "tokens_seen": 578289664 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004165396188565697, + "loss": 1.5435, + "theoretical_loss": 3.883168337984686, + "tokens_seen": 578355200 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004165295887662989, + "loss": 1.4766, + "theoretical_loss": 3.883122468693865, + "tokens_seen": 578420736 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004165195586760281, + "loss": 1.3521, + "theoretical_loss": 3.8830766060548036, + "tokens_seen": 578486272 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004165095285857573, + "loss": 1.413, + "theoretical_loss": 3.8830307500657844, + "tokens_seen": 578551808 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041649949849548645, + "loss": 1.5761, + "theoretical_loss": 3.8829849007250905, + "tokens_seen": 578617344 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004164894684052157, + "loss": 1.3736, + "theoretical_loss": 3.8829390580310044, + "tokens_seen": 578682880 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004164794383149448, + "loss": 1.4668, + "theoretical_loss": 3.88289322198181, + "tokens_seen": 578748416 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041646940822467405, + "loss": 1.4697, + "theoretical_loss": 3.8828473925757923, + "tokens_seen": 578813952 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041645937813440323, + "loss": 1.4636, + "theoretical_loss": 3.882801569811236, + "tokens_seen": 578879488 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004164493480441324, + "loss": 1.4609, + "theoretical_loss": 3.8827557536864274, + "tokens_seen": 578945024 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004164393179538616, + "loss": 1.4468, + "theoretical_loss": 3.882709944199653, + "tokens_seen": 579010560 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004164292878635908, + "loss": 1.4194, + "theoretical_loss": 3.8826641413491987, + "tokens_seen": 579076096 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041641925777331995, + "loss": 1.4647, + "theoretical_loss": 3.882618345133354, + "tokens_seen": 579141632 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004164092276830492, + "loss": 1.4101, + "theoretical_loss": 3.882572555550406, + "tokens_seen": 579207168 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004163991975927783, + "loss": 1.4831, + "theoretical_loss": 3.882526772598644, + "tokens_seen": 579272704 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041638916750250755, + "loss": 1.4564, + "theoretical_loss": 3.8824809962763576, + "tokens_seen": 579338240 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041637913741223673, + "loss": 1.5219, + "theoretical_loss": 3.882435226581837, + "tokens_seen": 579403776 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004163691073219659, + "loss": 1.38, + "theoretical_loss": 3.8823894635133738, + "tokens_seen": 579469312 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004163590772316951, + "loss": 1.409, + "theoretical_loss": 3.8823437070692584, + "tokens_seen": 579534848 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004163490471414243, + "loss": 1.3321, + "theoretical_loss": 3.882297957247784, + "tokens_seen": 579600384 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041633901705115346, + "loss": 1.4019, + "theoretical_loss": 3.8822522140472424, + "tokens_seen": 579665920 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004163289869608827, + "loss": 1.3864, + "theoretical_loss": 3.882206477465928, + "tokens_seen": 579731456 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004163189568706118, + "loss": 1.368, + "theoretical_loss": 3.882160747502134, + "tokens_seen": 579796992 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041630892678034105, + "loss": 1.3903, + "theoretical_loss": 3.8821150241541558, + "tokens_seen": 579862528 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004162988966900702, + "loss": 1.4308, + "theoretical_loss": 3.8820693074202888, + "tokens_seen": 579928064 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 198694, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4188852310180664, + "objective/train/theoretical_loss": 3.8820235972988275, + "objective/train/tokens_used": 579993600, + "theoretical_loss": 3.8820235972988275, + "tokens_seen": 579993600 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004162888665997994, + "loss": 1.4545, + "theoretical_loss": 3.8820235972988275, + "tokens_seen": 579993600 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004162788365095286, + "loss": 1.4516, + "theoretical_loss": 3.8819778937880702, + "tokens_seen": 580059136 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004162688064192578, + "loss": 1.5252, + "theoretical_loss": 3.8819321968863134, + "tokens_seen": 580124672 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041625877632898696, + "loss": 1.4577, + "theoretical_loss": 3.8818865065918553, + "tokens_seen": 580190208 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004162487462387162, + "loss": 1.4346, + "theoretical_loss": 3.8818408229029937, + "tokens_seen": 580255744 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004162387161484453, + "loss": 1.4859, + "theoretical_loss": 3.881795145818028, + "tokens_seen": 580321280 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041622868605817456, + "loss": 1.4847, + "theoretical_loss": 3.8817494753352584, + "tokens_seen": 580386816 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004162186559679037, + "loss": 1.4043, + "theoretical_loss": 3.881703811452985, + "tokens_seen": 580452352 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004162086258776329, + "loss": 1.4543, + "theoretical_loss": 3.8816581541695085, + "tokens_seen": 580517888 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004161985957873621, + "loss": 1.4677, + "theoretical_loss": 3.8816125034831312, + "tokens_seen": 580583424 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004161885656970913, + "loss": 1.4944, + "theoretical_loss": 3.881566859392154, + "tokens_seen": 580648960 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041617853560682046, + "loss": 1.5293, + "theoretical_loss": 3.8815212218948814, + "tokens_seen": 580714496 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041616850551654964, + "loss": 1.4175, + "theoretical_loss": 3.8814755909896164, + "tokens_seen": 580780032 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004161584754262788, + "loss": 1.3677, + "theoretical_loss": 3.881429966674663, + "tokens_seen": 580845568 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041614844533600806, + "loss": 1.4174, + "theoretical_loss": 3.881384348948325, + "tokens_seen": 580911104 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004161384152457372, + "loss": 1.5132, + "theoretical_loss": 3.8813387378089095, + "tokens_seen": 580976640 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004161283851554664, + "loss": 1.461, + "theoretical_loss": 3.8812931332547214, + "tokens_seen": 581042176 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041611835506519555, + "loss": 1.451, + "theoretical_loss": 3.881247535284068, + "tokens_seen": 581107712 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004161083249749248, + "loss": 1.3145, + "theoretical_loss": 3.881201943895256, + "tokens_seen": 581173248 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041609829488465397, + "loss": 1.3905, + "theoretical_loss": 3.8811563590865936, + "tokens_seen": 581238784 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041608826479438315, + "loss": 1.48, + "theoretical_loss": 3.8811107808563894, + "tokens_seen": 581304320 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041607823470411233, + "loss": 1.5177, + "theoretical_loss": 3.8810652092029523, + "tokens_seen": 581369856 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041606820461384156, + "loss": 1.3987, + "theoretical_loss": 3.881019644124593, + "tokens_seen": 581435392 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004160581745235707, + "loss": 1.3797, + "theoretical_loss": 3.8809740856196204, + "tokens_seen": 581500928 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004160481444332999, + "loss": 1.4506, + "theoretical_loss": 3.880928533686346, + "tokens_seen": 581566464 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041603811434302905, + "loss": 1.5051, + "theoretical_loss": 3.8808829883230826, + "tokens_seen": 581632000 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004160280842527583, + "loss": 1.4308, + "theoretical_loss": 3.880837449528141, + "tokens_seen": 581697536 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041601805416248747, + "loss": 1.4472, + "theoretical_loss": 3.8807919172998346, + "tokens_seen": 581763072 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041600802407221665, + "loss": 1.5333, + "theoretical_loss": 3.8807463916364773, + "tokens_seen": 581828608 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041599799398194583, + "loss": 1.4049, + "theoretical_loss": 3.8807008725363836, + "tokens_seen": 581894144 + }, + { + "epoch": 0.18, + "learning_rate": 0.000415987963891675, + "loss": 1.493, + "theoretical_loss": 3.880655359997867, + "tokens_seen": 581959680 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004159779338014042, + "loss": 1.3773, + "theoretical_loss": 3.8806098540192435, + "tokens_seen": 582025216 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041596790371113343, + "loss": 1.4751, + "theoretical_loss": 3.880564354598829, + "tokens_seen": 582090752 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041595787362086256, + "loss": 1.4397, + "theoretical_loss": 3.8805188617349406, + "tokens_seen": 582156288 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004159478435305918, + "loss": 1.4717, + "theoretical_loss": 3.880473375425895, + "tokens_seen": 582221824 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004159378134403209, + "loss": 1.4297, + "theoretical_loss": 3.880427895670011, + "tokens_seen": 582287360 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041592778335005015, + "loss": 1.4833, + "theoretical_loss": 3.8803824224656056, + "tokens_seen": 582352896 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041591775325977934, + "loss": 1.4148, + "theoretical_loss": 3.880336955810999, + "tokens_seen": 582418432 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004159077231695085, + "loss": 1.5555, + "theoretical_loss": 3.880291495704511, + "tokens_seen": 582483968 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041589769307923775, + "loss": 1.5214, + "theoretical_loss": 3.880246042144461, + "tokens_seen": 582549504 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041588766298896693, + "loss": 1.4464, + "theoretical_loss": 3.880200595129171, + "tokens_seen": 582615040 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004158776328986961, + "loss": 1.3474, + "theoretical_loss": 3.880155154656962, + "tokens_seen": 582680576 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004158676028084253, + "loss": 1.5011, + "theoretical_loss": 3.880109720726157, + "tokens_seen": 582746112 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004158575727181545, + "loss": 1.3791, + "theoretical_loss": 3.880064293335078, + "tokens_seen": 582811648 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041584754262788366, + "loss": 1.5201, + "theoretical_loss": 3.880018872482049, + "tokens_seen": 582877184 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004158375125376129, + "loss": 1.3618, + "theoretical_loss": 3.8799734581653933, + "tokens_seen": 582942720 + }, + { + "epoch": 0.18, + "learning_rate": 0.000415827482447342, + "loss": 1.4854, + "theoretical_loss": 3.8799280503834366, + "tokens_seen": 583008256 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041581745235707126, + "loss": 1.5027, + "theoretical_loss": 3.879882649134503, + "tokens_seen": 583073792 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004158074222668004, + "loss": 1.3918, + "theoretical_loss": 3.8798372544169197, + "tokens_seen": 583139328 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004157973921765296, + "loss": 1.3974, + "theoretical_loss": 3.879791866229013, + "tokens_seen": 583204864 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 199553, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3207201957702637, + "objective/train/theoretical_loss": 3.879746484569109, + "objective/train/tokens_used": 583270400, + "theoretical_loss": 3.879746484569109, + "tokens_seen": 583270400 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004157873620862588, + "loss": 1.3939, + "theoretical_loss": 3.879746484569109, + "tokens_seen": 583270400 + }, + { + "epoch": 0.18, + "learning_rate": 0.000415777331995988, + "loss": 1.4506, + "theoretical_loss": 3.879701109435537, + "tokens_seen": 583335936 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041576730190571716, + "loss": 1.3538, + "theoretical_loss": 3.8796557408266237, + "tokens_seen": 583401472 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004157572718154464, + "loss": 1.5186, + "theoretical_loss": 3.8796103787406997, + "tokens_seen": 583467008 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004157472417251755, + "loss": 1.4279, + "theoretical_loss": 3.8795650231760934, + "tokens_seen": 583532544 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041573721163490476, + "loss": 1.4352, + "theoretical_loss": 3.879519674131136, + "tokens_seen": 583598080 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004157271815446339, + "loss": 1.4395, + "theoretical_loss": 3.8794743316041584, + "tokens_seen": 583663616 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004157171514543631, + "loss": 1.4291, + "theoretical_loss": 3.879428995593491, + "tokens_seen": 583729152 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004157071213640923, + "loss": 1.3632, + "theoretical_loss": 3.8793836660974668, + "tokens_seen": 583794688 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004156970912738215, + "loss": 1.4222, + "theoretical_loss": 3.8793383431144175, + "tokens_seen": 583860224 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041568706118355066, + "loss": 1.4815, + "theoretical_loss": 3.879293026642678, + "tokens_seen": 583925760 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041567703109327984, + "loss": 1.4437, + "theoretical_loss": 3.879247716680581, + "tokens_seen": 583991296 + }, + { + "epoch": 0.18, + "learning_rate": 0.000415667001003009, + "loss": 1.5088, + "theoretical_loss": 3.8792024132264613, + "tokens_seen": 584056832 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041565697091273826, + "loss": 1.3914, + "theoretical_loss": 3.8791571162786544, + "tokens_seen": 584122368 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004156469408224674, + "loss": 1.492, + "theoretical_loss": 3.8791118258354955, + "tokens_seen": 584187904 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004156369107321966, + "loss": 1.4201, + "theoretical_loss": 3.8790665418953214, + "tokens_seen": 584253440 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041562688064192575, + "loss": 1.4496, + "theoretical_loss": 3.879021264456469, + "tokens_seen": 584318976 + }, + { + "epoch": 0.18, + "learning_rate": 0.000415616850551655, + "loss": 1.4905, + "theoretical_loss": 3.878975993517276, + "tokens_seen": 584384512 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041560682046138417, + "loss": 1.4233, + "theoretical_loss": 3.8789307290760804, + "tokens_seen": 584450048 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041559679037111335, + "loss": 1.3895, + "theoretical_loss": 3.8788854711312206, + "tokens_seen": 584515584 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041558676028084253, + "loss": 1.4743, + "theoretical_loss": 3.878840219681037, + "tokens_seen": 584581120 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041557673019057176, + "loss": 1.4938, + "theoretical_loss": 3.8787949747238697, + "tokens_seen": 584646656 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004155667001003009, + "loss": 1.3852, + "theoretical_loss": 3.878749736258058, + "tokens_seen": 584712192 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004155566700100301, + "loss": 1.3898, + "theoretical_loss": 3.8787045042819446, + "tokens_seen": 584777728 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041554663991975925, + "loss": 1.3267, + "theoretical_loss": 3.8786592787938705, + "tokens_seen": 584843264 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004155366098294885, + "loss": 1.4215, + "theoretical_loss": 3.878614059792178, + "tokens_seen": 584908800 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041552657973921767, + "loss": 1.4084, + "theoretical_loss": 3.878568847275212, + "tokens_seen": 584974336 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041551654964894685, + "loss": 1.3649, + "theoretical_loss": 3.8785236412413138, + "tokens_seen": 585039872 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041550651955867603, + "loss": 1.4672, + "theoretical_loss": 3.878478441688829, + "tokens_seen": 585105408 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004154964894684052, + "loss": 1.469, + "theoretical_loss": 3.878433248616102, + "tokens_seen": 585170944 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004154864593781344, + "loss": 1.4459, + "theoretical_loss": 3.878388062021479, + "tokens_seen": 585236480 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041547642928786363, + "loss": 1.4778, + "theoretical_loss": 3.8783428819033055, + "tokens_seen": 585302016 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041546639919759276, + "loss": 1.42, + "theoretical_loss": 3.8782977082599284, + "tokens_seen": 585367552 + }, + { + "epoch": 0.18, + "learning_rate": 0.000415456369107322, + "loss": 1.3519, + "theoretical_loss": 3.8782525410896955, + "tokens_seen": 585433088 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004154463390170511, + "loss": 1.4303, + "theoretical_loss": 3.8782073803909545, + "tokens_seen": 585498624 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041543630892678035, + "loss": 1.376, + "theoretical_loss": 3.878162226162053, + "tokens_seen": 585564160 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041542627883650954, + "loss": 1.437, + "theoretical_loss": 3.878117078401342, + "tokens_seen": 585629696 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004154162487462387, + "loss": 1.5172, + "theoretical_loss": 3.87807193710717, + "tokens_seen": 585695232 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004154062186559679, + "loss": 1.4047, + "theoretical_loss": 3.8780268022778874, + "tokens_seen": 585760768 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041539618856569713, + "loss": 1.5122, + "theoretical_loss": 3.877981673911846, + "tokens_seen": 585826304 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041538615847542626, + "loss": 1.4169, + "theoretical_loss": 3.8779365520073963, + "tokens_seen": 585891840 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004153761283851555, + "loss": 1.4443, + "theoretical_loss": 3.8778914365628916, + "tokens_seen": 585957376 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004153660982948846, + "loss": 1.4911, + "theoretical_loss": 3.877846327576684, + "tokens_seen": 586022912 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041535606820461386, + "loss": 1.4732, + "theoretical_loss": 3.8778012250471265, + "tokens_seen": 586088448 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041534603811434304, + "loss": 1.4558, + "theoretical_loss": 3.8777561289725746, + "tokens_seen": 586153984 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004153360080240722, + "loss": 1.5405, + "theoretical_loss": 3.8777110393513814, + "tokens_seen": 586219520 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004153259779338014, + "loss": 1.3696, + "theoretical_loss": 3.877665956181903, + "tokens_seen": 586285056 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004153159478435306, + "loss": 1.3516, + "theoretical_loss": 3.8776208794624942, + "tokens_seen": 586350592 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041530591775325976, + "loss": 1.3756, + "theoretical_loss": 3.8775758091915127, + "tokens_seen": 586416128 + }, + { + "epoch": 0.18, + "learning_rate": 0.000415295887662989, + "loss": 1.4819, + "theoretical_loss": 3.8775307453673147, + "tokens_seen": 586481664 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 200717, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5720303058624268, + "objective/train/theoretical_loss": 3.8774856879882584, + "objective/train/tokens_used": 586547200, + "theoretical_loss": 3.8774856879882584, + "tokens_seen": 586547200 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004152858575727181, + "loss": 1.4964, + "theoretical_loss": 3.8774856879882584, + "tokens_seen": 586547200 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041527582748244736, + "loss": 1.4482, + "theoretical_loss": 3.8774406370527013, + "tokens_seen": 586612736 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004152657973921765, + "loss": 1.5188, + "theoretical_loss": 3.8773955925590036, + "tokens_seen": 586678272 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004152557673019057, + "loss": 1.4089, + "theoretical_loss": 3.8773505545055227, + "tokens_seen": 586743808 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004152457372116349, + "loss": 1.4362, + "theoretical_loss": 3.87730552289062, + "tokens_seen": 586809344 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004152357071213641, + "loss": 1.4432, + "theoretical_loss": 3.8772604977126557, + "tokens_seen": 586874880 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041522567703109327, + "loss": 1.3411, + "theoretical_loss": 3.8772154789699913, + "tokens_seen": 586940416 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004152156469408225, + "loss": 1.5, + "theoretical_loss": 3.8771704666609885, + "tokens_seen": 587005952 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041520561685055163, + "loss": 1.4045, + "theoretical_loss": 3.8771254607840095, + "tokens_seen": 587071488 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041519558676028086, + "loss": 1.4476, + "theoretical_loss": 3.877080461337418, + "tokens_seen": 587137024 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041518555667001, + "loss": 1.3688, + "theoretical_loss": 3.877035468319577, + "tokens_seen": 587202560 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004151755265797392, + "loss": 1.3062, + "theoretical_loss": 3.8769904817288507, + "tokens_seen": 587268096 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004151654964894684, + "loss": 1.4699, + "theoretical_loss": 3.8769455015636045, + "tokens_seen": 587333632 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004151554663991976, + "loss": 1.4805, + "theoretical_loss": 3.8769005278222033, + "tokens_seen": 587399168 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004151454363089268, + "loss": 1.5125, + "theoretical_loss": 3.876855560503013, + "tokens_seen": 587464704 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041513540621865595, + "loss": 1.5034, + "theoretical_loss": 3.8768105996044007, + "tokens_seen": 587530240 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004151253761283852, + "loss": 1.4398, + "theoretical_loss": 3.876765645124734, + "tokens_seen": 587595776 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041511534603811437, + "loss": 1.4684, + "theoretical_loss": 3.876720697062379, + "tokens_seen": 587661312 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041510531594784355, + "loss": 1.4118, + "theoretical_loss": 3.876675755415706, + "tokens_seen": 587726848 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041509528585757273, + "loss": 1.5156, + "theoretical_loss": 3.8766308201830837, + "tokens_seen": 587792384 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041508525576730196, + "loss": 1.5356, + "theoretical_loss": 3.8765858913628803, + "tokens_seen": 587857920 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004150752256770311, + "loss": 1.362, + "theoretical_loss": 3.8765409689534676, + "tokens_seen": 587923456 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004150651955867603, + "loss": 1.5109, + "theoretical_loss": 3.8764960529532155, + "tokens_seen": 587988992 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041505516549648945, + "loss": 1.4394, + "theoretical_loss": 3.876451143360496, + "tokens_seen": 588054528 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004150451354062187, + "loss": 1.4158, + "theoretical_loss": 3.8764062401736803, + "tokens_seen": 588120064 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041503510531594787, + "loss": 1.4507, + "theoretical_loss": 3.8763613433911415, + "tokens_seen": 588185600 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041502507522567705, + "loss": 1.4446, + "theoretical_loss": 3.8763164530112535, + "tokens_seen": 588251136 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041501504513540623, + "loss": 1.3622, + "theoretical_loss": 3.8762715690323883, + "tokens_seen": 588316672 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004150050150451354, + "loss": 1.5598, + "theoretical_loss": 3.876226691452922, + "tokens_seen": 588382208 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004149949849548646, + "loss": 1.414, + "theoretical_loss": 3.8761818202712286, + "tokens_seen": 588447744 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041498495486459383, + "loss": 1.37, + "theoretical_loss": 3.8761369554856837, + "tokens_seen": 588513280 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041497492477432296, + "loss": 1.5803, + "theoretical_loss": 3.8760920970946637, + "tokens_seen": 588578816 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004149648946840522, + "loss": 1.459, + "theoretical_loss": 3.8760472450965455, + "tokens_seen": 588644352 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004149548645937813, + "loss": 1.5056, + "theoretical_loss": 3.876002399489706, + "tokens_seen": 588709888 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041494483450351055, + "loss": 1.4893, + "theoretical_loss": 3.8759575602725236, + "tokens_seen": 588775424 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041493480441323974, + "loss": 1.3756, + "theoretical_loss": 3.8759127274433762, + "tokens_seen": 588840960 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004149247743229689, + "loss": 1.4757, + "theoretical_loss": 3.8758679010006434, + "tokens_seen": 588906496 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004149147442326981, + "loss": 1.4625, + "theoretical_loss": 3.875823080942705, + "tokens_seen": 588972032 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041490471414242733, + "loss": 1.3526, + "theoretical_loss": 3.875778267267941, + "tokens_seen": 589037568 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041489468405215646, + "loss": 1.4389, + "theoretical_loss": 3.8757334599747324, + "tokens_seen": 589103104 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004148846539618857, + "loss": 1.3686, + "theoretical_loss": 3.8756886590614608, + "tokens_seen": 589168640 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004148746238716148, + "loss": 1.4376, + "theoretical_loss": 3.8756438645265074, + "tokens_seen": 589234176 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041486459378134406, + "loss": 1.5091, + "theoretical_loss": 3.8755990763682564, + "tokens_seen": 589299712 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041485456369107324, + "loss": 1.3661, + "theoretical_loss": 3.87555429458509, + "tokens_seen": 589365248 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004148445336008024, + "loss": 1.4165, + "theoretical_loss": 3.8755095191753917, + "tokens_seen": 589430784 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004148345035105316, + "loss": 1.5012, + "theoretical_loss": 3.8754647501375468, + "tokens_seen": 589496320 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004148244734202608, + "loss": 1.5172, + "theoretical_loss": 3.8754199874699404, + "tokens_seen": 589561856 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041481444332998996, + "loss": 1.4297, + "theoretical_loss": 3.875375231170957, + "tokens_seen": 589627392 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004148044132397192, + "loss": 1.503, + "theoretical_loss": 3.8753304812389837, + "tokens_seen": 589692928 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004147943831494483, + "loss": 1.4047, + "theoretical_loss": 3.8752857376724075, + "tokens_seen": 589758464 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 201572, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3953429460525513, + "objective/train/theoretical_loss": 3.875241000469615, + "objective/train/tokens_used": 589824000, + "theoretical_loss": 3.875241000469615, + "tokens_seen": 589824000 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041478435305917756, + "loss": 1.4187, + "theoretical_loss": 3.875241000469615, + "tokens_seen": 589824000 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004147743229689067, + "loss": 1.3881, + "theoretical_loss": 3.8751962696289945, + "tokens_seen": 589889536 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004147642928786359, + "loss": 1.4744, + "theoretical_loss": 3.8751515451489347, + "tokens_seen": 589955072 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004147542627883651, + "loss": 1.4203, + "theoretical_loss": 3.875106827027824, + "tokens_seen": 590020608 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004147442326980943, + "loss": 1.3968, + "theoretical_loss": 3.875062115264053, + "tokens_seen": 590086144 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041473420260782347, + "loss": 1.394, + "theoretical_loss": 3.875017409856012, + "tokens_seen": 590151680 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004147241725175527, + "loss": 1.4874, + "theoretical_loss": 3.8749727108020915, + "tokens_seen": 590217216 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041471414242728183, + "loss": 1.4781, + "theoretical_loss": 3.8749280181006824, + "tokens_seen": 590282752 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041470411233701106, + "loss": 1.3798, + "theoretical_loss": 3.874883331750178, + "tokens_seen": 590348288 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004146940822467402, + "loss": 1.5026, + "theoretical_loss": 3.874838651748971, + "tokens_seen": 590413824 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004146840521564694, + "loss": 1.3342, + "theoretical_loss": 3.874793978095453, + "tokens_seen": 590479360 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004146740220661986, + "loss": 1.4208, + "theoretical_loss": 3.874749310788019, + "tokens_seen": 590544896 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004146639919759278, + "loss": 1.4358, + "theoretical_loss": 3.8747046498250626, + "tokens_seen": 590610432 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041465396188565697, + "loss": 1.5106, + "theoretical_loss": 3.87465999520498, + "tokens_seen": 590675968 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041464393179538615, + "loss": 1.4422, + "theoretical_loss": 3.8746153469261664, + "tokens_seen": 590741504 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041463390170511533, + "loss": 1.3896, + "theoretical_loss": 3.8745707049870175, + "tokens_seen": 590807040 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041462387161484457, + "loss": 1.4861, + "theoretical_loss": 3.8745260693859302, + "tokens_seen": 590872576 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004146138415245737, + "loss": 1.4336, + "theoretical_loss": 3.8744814401213024, + "tokens_seen": 590938112 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041460381143430293, + "loss": 1.4852, + "theoretical_loss": 3.8744368171915307, + "tokens_seen": 591003648 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004145937813440321, + "loss": 1.5232, + "theoretical_loss": 3.8743922005950147, + "tokens_seen": 591069184 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004145837512537613, + "loss": 1.4694, + "theoretical_loss": 3.8743475903301534, + "tokens_seen": 591134720 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004145737211634905, + "loss": 1.4404, + "theoretical_loss": 3.874302986395346, + "tokens_seen": 591200256 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041456369107321965, + "loss": 1.5509, + "theoretical_loss": 3.8742583887889928, + "tokens_seen": 591265792 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041455366098294884, + "loss": 1.5118, + "theoretical_loss": 3.874213797509495, + "tokens_seen": 591331328 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041454363089267807, + "loss": 1.4899, + "theoretical_loss": 3.874169212555253, + "tokens_seen": 591396864 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004145336008024072, + "loss": 1.4781, + "theoretical_loss": 3.87412463392467, + "tokens_seen": 591462400 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041452357071213643, + "loss": 1.4947, + "theoretical_loss": 3.8740800616161484, + "tokens_seen": 591527936 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041451354062186556, + "loss": 1.4696, + "theoretical_loss": 3.8740354956280907, + "tokens_seen": 591593472 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004145035105315948, + "loss": 1.4338, + "theoretical_loss": 3.873990935958901, + "tokens_seen": 591659008 + }, + { + "epoch": 0.18, + "learning_rate": 0.000414493480441324, + "loss": 1.4895, + "theoretical_loss": 3.8739463826069835, + "tokens_seen": 591724544 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041448345035105316, + "loss": 1.3567, + "theoretical_loss": 3.8739018355707433, + "tokens_seen": 591790080 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041447342026078234, + "loss": 1.5245, + "theoretical_loss": 3.8738572948485848, + "tokens_seen": 591855616 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004144633901705115, + "loss": 1.4987, + "theoretical_loss": 3.873812760438916, + "tokens_seen": 591921152 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004144533600802407, + "loss": 1.3857, + "theoretical_loss": 3.873768232340142, + "tokens_seen": 591986688 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041444332998996994, + "loss": 1.3833, + "theoretical_loss": 3.87372371055067, + "tokens_seen": 592052224 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041443329989969906, + "loss": 1.4994, + "theoretical_loss": 3.8736791950689087, + "tokens_seen": 592117760 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004144232698094283, + "loss": 1.5585, + "theoretical_loss": 3.8736346858932658, + "tokens_seen": 592183296 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041441323971915753, + "loss": 1.385, + "theoretical_loss": 3.8735901830221495, + "tokens_seen": 592248832 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041440320962888666, + "loss": 1.4169, + "theoretical_loss": 3.873545686453971, + "tokens_seen": 592314368 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004143931795386159, + "loss": 1.5487, + "theoretical_loss": 3.8735011961871395, + "tokens_seen": 592379904 + }, + { + "epoch": 0.18, + "learning_rate": 0.000414383149448345, + "loss": 1.4831, + "theoretical_loss": 3.873456712220066, + "tokens_seen": 592445440 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041437311935807426, + "loss": 1.3843, + "theoretical_loss": 3.873412234551161, + "tokens_seen": 592510976 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041436308926780344, + "loss": 1.4296, + "theoretical_loss": 3.8733677631788366, + "tokens_seen": 592576512 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004143530591775326, + "loss": 1.434, + "theoretical_loss": 3.8733232981015058, + "tokens_seen": 592642048 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004143430290872618, + "loss": 1.436, + "theoretical_loss": 3.8732788393175808, + "tokens_seen": 592707584 + }, + { + "epoch": 0.18, + "learning_rate": 0.000414332998996991, + "loss": 1.5066, + "theoretical_loss": 3.8732343868254753, + "tokens_seen": 592773120 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041432296890672016, + "loss": 1.383, + "theoretical_loss": 3.873189940623604, + "tokens_seen": 592838656 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004143129388164494, + "loss": 1.4072, + "theoretical_loss": 3.8731455007103808, + "tokens_seen": 592904192 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004143029087261785, + "loss": 1.5608, + "theoretical_loss": 3.873101067084221, + "tokens_seen": 592969728 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041429287863590776, + "loss": 1.4397, + "theoretical_loss": 3.873056639743541, + "tokens_seen": 593035264 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 202779, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4067751169204712, + "objective/train/theoretical_loss": 3.8730122186867573, + "objective/train/tokens_used": 593100800, + "theoretical_loss": 3.8730122186867573, + "tokens_seen": 593100800 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004142828485456369, + "loss": 1.5007, + "theoretical_loss": 3.8730122186867573, + "tokens_seen": 593100800 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004142728184553661, + "loss": 1.5255, + "theoretical_loss": 3.872967803912286, + "tokens_seen": 593166336 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004142627883650953, + "loss": 1.4644, + "theoretical_loss": 3.872923395418546, + "tokens_seen": 593231872 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004142527582748245, + "loss": 1.4057, + "theoretical_loss": 3.872878993203954, + "tokens_seen": 593297408 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041424272818455367, + "loss": 1.4694, + "theoretical_loss": 3.8728345972669294, + "tokens_seen": 593362944 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004142326980942829, + "loss": 1.412, + "theoretical_loss": 3.872790207605891, + "tokens_seen": 593428480 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041422266800401203, + "loss": 1.3989, + "theoretical_loss": 3.87274582421926, + "tokens_seen": 593494016 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041421263791374126, + "loss": 1.372, + "theoretical_loss": 3.872701447105455, + "tokens_seen": 593559552 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004142026078234704, + "loss": 1.4886, + "theoretical_loss": 3.8726570762628985, + "tokens_seen": 593625088 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004141925777331996, + "loss": 1.4416, + "theoretical_loss": 3.872612711690011, + "tokens_seen": 593690624 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004141825476429288, + "loss": 1.4888, + "theoretical_loss": 3.8725683533852155, + "tokens_seen": 593756160 + }, + { + "epoch": 0.18, + "learning_rate": 0.000414172517552658, + "loss": 1.3802, + "theoretical_loss": 3.8725240013469344, + "tokens_seen": 593821696 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041416248746238717, + "loss": 1.3837, + "theoretical_loss": 3.8724796555735903, + "tokens_seen": 593887232 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041415245737211635, + "loss": 1.3763, + "theoretical_loss": 3.8724353160636085, + "tokens_seen": 593952768 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041414242728184553, + "loss": 1.4165, + "theoretical_loss": 3.872390982815412, + "tokens_seen": 594018304 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041413239719157477, + "loss": 1.3582, + "theoretical_loss": 3.8723466558274264, + "tokens_seen": 594083840 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004141223671013039, + "loss": 1.3704, + "theoretical_loss": 3.872302335098077, + "tokens_seen": 594149376 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041411233701103313, + "loss": 1.45, + "theoretical_loss": 3.872258020625791, + "tokens_seen": 594214912 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004141023069207623, + "loss": 1.3582, + "theoretical_loss": 3.8722137124089935, + "tokens_seen": 594280448 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004140922768304915, + "loss": 1.4712, + "theoretical_loss": 3.8721694104461126, + "tokens_seen": 594345984 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004140822467402207, + "loss": 1.4484, + "theoretical_loss": 3.872125114735576, + "tokens_seen": 594411520 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041407221664994985, + "loss": 1.4019, + "theoretical_loss": 3.8720808252758125, + "tokens_seen": 594477056 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041406218655967904, + "loss": 1.5126, + "theoretical_loss": 3.872036542065251, + "tokens_seen": 594542592 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041405215646940827, + "loss": 1.4092, + "theoretical_loss": 3.87199226510232, + "tokens_seen": 594608128 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004140421263791374, + "loss": 1.414, + "theoretical_loss": 3.871947994385451, + "tokens_seen": 594673664 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041403209628886663, + "loss": 1.3764, + "theoretical_loss": 3.871903729913074, + "tokens_seen": 594739200 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041402206619859576, + "loss": 1.4632, + "theoretical_loss": 3.8718594716836203, + "tokens_seen": 594804736 + }, + { + "epoch": 0.18, + "learning_rate": 0.000414012036108325, + "loss": 1.3984, + "theoretical_loss": 3.871815219695522, + "tokens_seen": 594870272 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004140020060180542, + "loss": 1.4366, + "theoretical_loss": 3.8717709739472106, + "tokens_seen": 594935808 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041399197592778336, + "loss": 1.5373, + "theoretical_loss": 3.87172673443712, + "tokens_seen": 595001344 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041398194583751254, + "loss": 1.3533, + "theoretical_loss": 3.8716825011636837, + "tokens_seen": 595066880 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004139719157472417, + "loss": 1.4582, + "theoretical_loss": 3.871638274125335, + "tokens_seen": 595132416 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004139618856569709, + "loss": 1.5438, + "theoretical_loss": 3.871594053320509, + "tokens_seen": 595197952 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041395185556670014, + "loss": 1.3938, + "theoretical_loss": 3.871549838747641, + "tokens_seen": 595263488 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041394182547642926, + "loss": 1.3867, + "theoretical_loss": 3.871505630405167, + "tokens_seen": 595329024 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004139317953861585, + "loss": 1.3973, + "theoretical_loss": 3.8714614282915227, + "tokens_seen": 595394560 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004139217652958877, + "loss": 1.4677, + "theoretical_loss": 3.8714172324051455, + "tokens_seen": 595460096 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041391173520561686, + "loss": 1.4685, + "theoretical_loss": 3.8713730427444726, + "tokens_seen": 595525632 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041390170511534604, + "loss": 1.5126, + "theoretical_loss": 3.8713288593079422, + "tokens_seen": 595591168 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004138916750250752, + "loss": 1.4198, + "theoretical_loss": 3.871284682093993, + "tokens_seen": 595656704 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004138816449348044, + "loss": 1.4709, + "theoretical_loss": 3.871240511101064, + "tokens_seen": 595722240 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041387161484453364, + "loss": 1.4027, + "theoretical_loss": 3.8711963463275945, + "tokens_seen": 595787776 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041386158475426277, + "loss": 1.4097, + "theoretical_loss": 3.871152187772026, + "tokens_seen": 595853312 + }, + { + "epoch": 0.18, + "learning_rate": 0.000413851554663992, + "loss": 1.4031, + "theoretical_loss": 3.8711080354327976, + "tokens_seen": 595918848 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041384152457372113, + "loss": 1.401, + "theoretical_loss": 3.8710638893083518, + "tokens_seen": 595984384 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041383149448345036, + "loss": 1.4, + "theoretical_loss": 3.8710197493971306, + "tokens_seen": 596049920 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041382146439317954, + "loss": 1.4147, + "theoretical_loss": 3.870975615697576, + "tokens_seen": 596115456 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004138114343029087, + "loss": 1.3721, + "theoretical_loss": 3.8709314882081323, + "tokens_seen": 596180992 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004138014042126379, + "loss": 1.4499, + "theoretical_loss": 3.870887366927242, + "tokens_seen": 596246528 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004137913741223671, + "loss": 1.4394, + "theoretical_loss": 3.8708432518533495, + "tokens_seen": 596312064 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 203693, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4731204509735107, + "objective/train/theoretical_loss": 3.8707991429848994, + "objective/train/tokens_used": 596377600, + "theoretical_loss": 3.8707991429848994, + "tokens_seen": 596377600 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041378134403209627, + "loss": 1.4018, + "theoretical_loss": 3.8707991429848994, + "tokens_seen": 596377600 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004137713139418255, + "loss": 1.4651, + "theoretical_loss": 3.8707550403203377, + "tokens_seen": 596443136 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041376128385155463, + "loss": 1.3478, + "theoretical_loss": 3.8707109438581098, + "tokens_seen": 596508672 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041375125376128387, + "loss": 1.4783, + "theoretical_loss": 3.8706668535966626, + "tokens_seen": 596574208 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041374122367101305, + "loss": 1.4918, + "theoretical_loss": 3.8706227695344424, + "tokens_seen": 596639744 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041373119358074223, + "loss": 1.4132, + "theoretical_loss": 3.8705786916698974, + "tokens_seen": 596705280 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004137211634904714, + "loss": 1.4106, + "theoretical_loss": 3.870534620001476, + "tokens_seen": 596770816 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004137111334002006, + "loss": 1.4827, + "theoretical_loss": 3.870490554527626, + "tokens_seen": 596836352 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041370110330992977, + "loss": 1.4501, + "theoretical_loss": 3.8704464952467967, + "tokens_seen": 596901888 + }, + { + "epoch": 0.18, + "learning_rate": 0.000413691073219659, + "loss": 1.4373, + "theoretical_loss": 3.870402442157439, + "tokens_seen": 596967424 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041368104312938813, + "loss": 1.4058, + "theoretical_loss": 3.8703583952580023, + "tokens_seen": 597032960 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041367101303911737, + "loss": 1.4773, + "theoretical_loss": 3.8703143545469376, + "tokens_seen": 597098496 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041366098294884655, + "loss": 1.4801, + "theoretical_loss": 3.870270320022697, + "tokens_seen": 597164032 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041365095285857573, + "loss": 1.3399, + "theoretical_loss": 3.8702262916837316, + "tokens_seen": 597229568 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041364092276830497, + "loss": 1.5491, + "theoretical_loss": 3.8701822695284953, + "tokens_seen": 597295104 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004136308926780341, + "loss": 1.4241, + "theoretical_loss": 3.8701382535554396, + "tokens_seen": 597360640 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041362086258776333, + "loss": 1.5854, + "theoretical_loss": 3.8700942437630195, + "tokens_seen": 597426176 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004136108324974925, + "loss": 1.4579, + "theoretical_loss": 3.870050240149689, + "tokens_seen": 597491712 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004136008024072217, + "loss": 1.4272, + "theoretical_loss": 3.870006242713903, + "tokens_seen": 597557248 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004135907723169509, + "loss": 1.4391, + "theoretical_loss": 3.8699622514541163, + "tokens_seen": 597622784 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041358074222668005, + "loss": 1.4368, + "theoretical_loss": 3.8699182663687854, + "tokens_seen": 597688320 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041357071213640924, + "loss": 1.5711, + "theoretical_loss": 3.8698742874563665, + "tokens_seen": 597753856 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041356068204613847, + "loss": 1.4766, + "theoretical_loss": 3.8698303147153172, + "tokens_seen": 597819392 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004135506519558676, + "loss": 1.5497, + "theoretical_loss": 3.8697863481440944, + "tokens_seen": 597884928 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041354062186559683, + "loss": 1.4377, + "theoretical_loss": 3.8697423877411565, + "tokens_seen": 597950464 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041353059177532596, + "loss": 1.4443, + "theoretical_loss": 3.8696984335049622, + "tokens_seen": 598016000 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004135205616850552, + "loss": 1.4163, + "theoretical_loss": 3.869654485433971, + "tokens_seen": 598081536 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004135105315947844, + "loss": 1.4226, + "theoretical_loss": 3.869610543526643, + "tokens_seen": 598147072 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041350050150451356, + "loss": 1.4568, + "theoretical_loss": 3.8695666077814375, + "tokens_seen": 598212608 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041349047141424274, + "loss": 1.5187, + "theoretical_loss": 3.869522678196816, + "tokens_seen": 598278144 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004134804413239719, + "loss": 1.5113, + "theoretical_loss": 3.86947875477124, + "tokens_seen": 598343680 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004134704112337011, + "loss": 1.5128, + "theoretical_loss": 3.8694348375031717, + "tokens_seen": 598409216 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041346038114343034, + "loss": 1.366, + "theoretical_loss": 3.8693909263910737, + "tokens_seen": 598474752 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041345035105315946, + "loss": 1.3806, + "theoretical_loss": 3.869347021433409, + "tokens_seen": 598540288 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004134403209628887, + "loss": 1.4387, + "theoretical_loss": 3.869303122628641, + "tokens_seen": 598605824 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004134302908726179, + "loss": 1.6058, + "theoretical_loss": 3.869259229975235, + "tokens_seen": 598671360 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041342026078234706, + "loss": 1.3909, + "theoretical_loss": 3.869215343471654, + "tokens_seen": 598736896 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041341023069207624, + "loss": 1.4797, + "theoretical_loss": 3.8691714631163645, + "tokens_seen": 598802432 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004134002006018054, + "loss": 1.4584, + "theoretical_loss": 3.8691275889078325, + "tokens_seen": 598867968 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004133901705115346, + "loss": 1.4028, + "theoretical_loss": 3.8690837208445243, + "tokens_seen": 598933504 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041338014042126384, + "loss": 1.4107, + "theoretical_loss": 3.869039858924906, + "tokens_seen": 598999040 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041337011033099297, + "loss": 1.4811, + "theoretical_loss": 3.868996003147447, + "tokens_seen": 599064576 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004133600802407222, + "loss": 1.4371, + "theoretical_loss": 3.868952153510614, + "tokens_seen": 599130112 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041335005015045133, + "loss": 1.3767, + "theoretical_loss": 3.868908310012876, + "tokens_seen": 599195648 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041334002006018056, + "loss": 1.4977, + "theoretical_loss": 3.868864472652702, + "tokens_seen": 599261184 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041332998996990975, + "loss": 1.5176, + "theoretical_loss": 3.868820641428562, + "tokens_seen": 599326720 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004133199598796389, + "loss": 1.4434, + "theoretical_loss": 3.868776816338926, + "tokens_seen": 599392256 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004133099297893681, + "loss": 1.3792, + "theoretical_loss": 3.8687329973822653, + "tokens_seen": 599457792 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004132998996990973, + "loss": 1.5308, + "theoretical_loss": 3.868689184557051, + "tokens_seen": 599523328 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041328986960882647, + "loss": 1.4149, + "theoretical_loss": 3.8686453778617547, + "tokens_seen": 599588864 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 204961, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2879407405853271, + "objective/train/theoretical_loss": 3.868601577294849, + "objective/train/tokens_used": 599654400, + "theoretical_loss": 3.868601577294849, + "tokens_seen": 599654400 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004132798395185557, + "loss": 1.3351, + "theoretical_loss": 3.868601577294849, + "tokens_seen": 599654400 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041326980942828483, + "loss": 1.3086, + "theoretical_loss": 3.8685577828548086, + "tokens_seen": 599719936 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041325977933801407, + "loss": 1.634, + "theoretical_loss": 3.8685139945401046, + "tokens_seen": 599785472 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041324974924774325, + "loss": 1.493, + "theoretical_loss": 3.8684702123492123, + "tokens_seen": 599851008 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041323971915747243, + "loss": 1.4824, + "theoretical_loss": 3.8684264362806062, + "tokens_seen": 599916544 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004132296890672016, + "loss": 1.3725, + "theoretical_loss": 3.868382666332762, + "tokens_seen": 599982080 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004132196589769308, + "loss": 1.4444, + "theoretical_loss": 3.8683389025041546, + "tokens_seen": 600047616 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041320962888665997, + "loss": 1.4521, + "theoretical_loss": 3.8682951447932608, + "tokens_seen": 600113152 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004131995987963892, + "loss": 1.4542, + "theoretical_loss": 3.8682513931985576, + "tokens_seen": 600178688 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041318956870611833, + "loss": 1.4172, + "theoretical_loss": 3.8682076477185223, + "tokens_seen": 600244224 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041317953861584757, + "loss": 1.3935, + "theoretical_loss": 3.868163908351632, + "tokens_seen": 600309760 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004131695085255767, + "loss": 1.4957, + "theoretical_loss": 3.868120175096367, + "tokens_seen": 600375296 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041315947843530593, + "loss": 1.5317, + "theoretical_loss": 3.868076447951205, + "tokens_seen": 600440832 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004131494483450351, + "loss": 1.4252, + "theoretical_loss": 3.8680327269146257, + "tokens_seen": 600506368 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004131394182547643, + "loss": 1.4825, + "theoretical_loss": 3.8679890119851095, + "tokens_seen": 600571904 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004131293881644935, + "loss": 1.4995, + "theoretical_loss": 3.867945303161137, + "tokens_seen": 600637440 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004131193580742227, + "loss": 1.4592, + "theoretical_loss": 3.8679016004411895, + "tokens_seen": 600702976 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041310932798395184, + "loss": 1.4911, + "theoretical_loss": 3.867857903823748, + "tokens_seen": 600768512 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004130992978936811, + "loss": 1.417, + "theoretical_loss": 3.8678142133072964, + "tokens_seen": 600834048 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004130892678034102, + "loss": 1.4506, + "theoretical_loss": 3.8677705288903166, + "tokens_seen": 600899584 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041307923771313944, + "loss": 1.3657, + "theoretical_loss": 3.867726850571291, + "tokens_seen": 600965120 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004130692076228686, + "loss": 1.5113, + "theoretical_loss": 3.867683178348705, + "tokens_seen": 601030656 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004130591775325978, + "loss": 1.454, + "theoretical_loss": 3.867639512221043, + "tokens_seen": 601096192 + }, + { + "epoch": 0.18, + "learning_rate": 0.000413049147442327, + "loss": 1.3831, + "theoretical_loss": 3.8675958521867893, + "tokens_seen": 601161728 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041303911735205616, + "loss": 1.4029, + "theoretical_loss": 3.8675521982444296, + "tokens_seen": 601227264 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041302908726178534, + "loss": 1.4023, + "theoretical_loss": 3.8675085503924507, + "tokens_seen": 601292800 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004130190571715146, + "loss": 1.4829, + "theoretical_loss": 3.867464908629338, + "tokens_seen": 601358336 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004130090270812437, + "loss": 1.4426, + "theoretical_loss": 3.86742127295358, + "tokens_seen": 601423872 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041299899699097294, + "loss": 1.392, + "theoretical_loss": 3.8673776433636635, + "tokens_seen": 601489408 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041298896690070207, + "loss": 1.4629, + "theoretical_loss": 3.8673340198580766, + "tokens_seen": 601554944 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004129789368104313, + "loss": 1.4059, + "theoretical_loss": 3.867290402435309, + "tokens_seen": 601620480 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004129689067201605, + "loss": 1.3693, + "theoretical_loss": 3.8672467910938497, + "tokens_seen": 601686016 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041295887662988966, + "loss": 1.428, + "theoretical_loss": 3.8672031858321887, + "tokens_seen": 601751552 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041294884653961884, + "loss": 1.4195, + "theoretical_loss": 3.8671595866488158, + "tokens_seen": 601817088 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004129388164493481, + "loss": 1.4032, + "theoretical_loss": 3.867115993542222, + "tokens_seen": 601882624 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004129287863590772, + "loss": 1.4298, + "theoretical_loss": 3.8670724065109, + "tokens_seen": 601948160 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041291875626880644, + "loss": 1.47, + "theoretical_loss": 3.8670288255533403, + "tokens_seen": 602013696 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004129087261785356, + "loss": 1.4113, + "theoretical_loss": 3.8669852506680362, + "tokens_seen": 602079232 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004128986960882648, + "loss": 1.4981, + "theoretical_loss": 3.866941681853481, + "tokens_seen": 602144768 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041288866599799404, + "loss": 1.3937, + "theoretical_loss": 3.866898119108168, + "tokens_seen": 602210304 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041287863590772317, + "loss": 1.428, + "theoretical_loss": 3.866854562430592, + "tokens_seen": 602275840 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004128686058174524, + "loss": 1.3837, + "theoretical_loss": 3.8668110118192462, + "tokens_seen": 602341376 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041285857572718153, + "loss": 1.434, + "theoretical_loss": 3.8667674672726275, + "tokens_seen": 602406912 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041284854563691076, + "loss": 1.5226, + "theoretical_loss": 3.866723928789231, + "tokens_seen": 602472448 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041283851554663995, + "loss": 1.4534, + "theoretical_loss": 3.8666803963675536, + "tokens_seen": 602537984 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004128284854563691, + "loss": 1.348, + "theoretical_loss": 3.866636870006091, + "tokens_seen": 602603520 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004128184553660983, + "loss": 1.4577, + "theoretical_loss": 3.866593349703342, + "tokens_seen": 602669056 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004128084252758275, + "loss": 1.3677, + "theoretical_loss": 3.8665498354578034, + "tokens_seen": 602734592 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041279839518555667, + "loss": 1.4237, + "theoretical_loss": 3.866506327267974, + "tokens_seen": 602800128 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004127883650952859, + "loss": 1.5129, + "theoretical_loss": 3.866462825132353, + "tokens_seen": 602865664 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 205832, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3989498615264893, + "objective/train/theoretical_loss": 3.86641932904944, + "objective/train/tokens_used": 602931200, + "theoretical_loss": 3.86641932904944, + "tokens_seen": 602931200 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041277833500501503, + "loss": 1.4074, + "theoretical_loss": 3.86641932904944, + "tokens_seen": 602931200 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041276830491474427, + "loss": 1.4119, + "theoretical_loss": 3.866375839017735, + "tokens_seen": 602996736 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041275827482447345, + "loss": 1.4841, + "theoretical_loss": 3.8663323550357385, + "tokens_seen": 603062272 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041274824473420263, + "loss": 1.3388, + "theoretical_loss": 3.866288877101952, + "tokens_seen": 603127808 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004127382146439318, + "loss": 1.4773, + "theoretical_loss": 3.866245405214877, + "tokens_seen": 603193344 + }, + { + "epoch": 0.18, + "learning_rate": 0.000412728184553661, + "loss": 1.4373, + "theoretical_loss": 3.8662019393730156, + "tokens_seen": 603258880 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041271815446339017, + "loss": 1.2811, + "theoretical_loss": 3.86615847957487, + "tokens_seen": 603324416 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004127081243731194, + "loss": 1.3849, + "theoretical_loss": 3.866115025818945, + "tokens_seen": 603389952 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041269809428284854, + "loss": 1.3501, + "theoretical_loss": 3.8660715781037434, + "tokens_seen": 603455488 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041268806419257777, + "loss": 1.5095, + "theoretical_loss": 3.8660281364277695, + "tokens_seen": 603521024 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004126780341023069, + "loss": 1.4114, + "theoretical_loss": 3.865984700789528, + "tokens_seen": 603586560 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041266800401203613, + "loss": 1.5435, + "theoretical_loss": 3.865941271187525, + "tokens_seen": 603652096 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004126579739217653, + "loss": 1.3912, + "theoretical_loss": 3.865897847620266, + "tokens_seen": 603717632 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004126479438314945, + "loss": 1.4785, + "theoretical_loss": 3.865854430086258, + "tokens_seen": 603783168 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004126379137412237, + "loss": 1.4349, + "theoretical_loss": 3.865811018584007, + "tokens_seen": 603848704 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004126278836509529, + "loss": 1.4506, + "theoretical_loss": 3.8657676131120224, + "tokens_seen": 603914240 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041261785356068204, + "loss": 1.4765, + "theoretical_loss": 3.86572421366881, + "tokens_seen": 603979776 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004126078234704113, + "loss": 1.4442, + "theoretical_loss": 3.86568082025288, + "tokens_seen": 604045312 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004125977933801404, + "loss": 1.3392, + "theoretical_loss": 3.8656374328627408, + "tokens_seen": 604110848 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041258776328986964, + "loss": 1.4791, + "theoretical_loss": 3.8655940514969025, + "tokens_seen": 604176384 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004125777331995988, + "loss": 1.4926, + "theoretical_loss": 3.865550676153875, + "tokens_seen": 604241920 + }, + { + "epoch": 0.18, + "learning_rate": 0.000412567703109328, + "loss": 1.4366, + "theoretical_loss": 3.8655073068321695, + "tokens_seen": 604307456 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004125576730190572, + "loss": 1.4757, + "theoretical_loss": 3.8654639435302967, + "tokens_seen": 604372992 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041254764292878636, + "loss": 1.395, + "theoretical_loss": 3.865420586246768, + "tokens_seen": 604438528 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041253761283851554, + "loss": 1.3865, + "theoretical_loss": 3.865377234980097, + "tokens_seen": 604504064 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004125275827482448, + "loss": 1.4311, + "theoretical_loss": 3.865333889728795, + "tokens_seen": 604569600 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004125175526579739, + "loss": 1.4203, + "theoretical_loss": 3.8652905504913777, + "tokens_seen": 604635136 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041250752256770314, + "loss": 1.4303, + "theoretical_loss": 3.865247217266356, + "tokens_seen": 604700672 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041249749247743227, + "loss": 1.3825, + "theoretical_loss": 3.865203890052247, + "tokens_seen": 604766208 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004124874623871615, + "loss": 1.5407, + "theoretical_loss": 3.865160568847564, + "tokens_seen": 604831744 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004124774322968907, + "loss": 1.3607, + "theoretical_loss": 3.865117253650823, + "tokens_seen": 604897280 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041246740220661986, + "loss": 1.3866, + "theoretical_loss": 3.8650739444605406, + "tokens_seen": 604962816 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041245737211634904, + "loss": 1.2774, + "theoretical_loss": 3.8650306412752324, + "tokens_seen": 605028352 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004124473420260783, + "loss": 1.4623, + "theoretical_loss": 3.8649873440934157, + "tokens_seen": 605093888 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004124373119358074, + "loss": 1.499, + "theoretical_loss": 3.8649440529136085, + "tokens_seen": 605159424 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041242728184553664, + "loss": 1.4162, + "theoretical_loss": 3.864900767734329, + "tokens_seen": 605224960 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041241725175526577, + "loss": 1.5624, + "theoretical_loss": 3.8648574885540947, + "tokens_seen": 605290496 + }, + { + "epoch": 0.18, + "learning_rate": 0.000412407221664995, + "loss": 1.3701, + "theoretical_loss": 3.8648142153714264, + "tokens_seen": 605356032 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004123971915747242, + "loss": 1.4378, + "theoretical_loss": 3.8647709481848427, + "tokens_seen": 605421568 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041238716148445337, + "loss": 1.3856, + "theoretical_loss": 3.8647276869928637, + "tokens_seen": 605487104 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041237713139418255, + "loss": 1.4051, + "theoretical_loss": 3.864684431794011, + "tokens_seen": 605552640 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041236710130391173, + "loss": 1.4593, + "theoretical_loss": 3.864641182586806, + "tokens_seen": 605618176 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004123570712136409, + "loss": 1.5073, + "theoretical_loss": 3.8645979393697694, + "tokens_seen": 605683712 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041234704112337015, + "loss": 1.4168, + "theoretical_loss": 3.864554702141424, + "tokens_seen": 605749248 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041233701103309927, + "loss": 1.4037, + "theoretical_loss": 3.8645114709002932, + "tokens_seen": 605814784 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004123269809428285, + "loss": 1.4002, + "theoretical_loss": 3.8644682456448995, + "tokens_seen": 605880320 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041231695085255763, + "loss": 1.3742, + "theoretical_loss": 3.864425026373768, + "tokens_seen": 605945856 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041230692076228687, + "loss": 1.4381, + "theoretical_loss": 3.864381813085422, + "tokens_seen": 606011392 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041229689067201605, + "loss": 1.4385, + "theoretical_loss": 3.8643386057783866, + "tokens_seen": 606076928 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041228686058174523, + "loss": 1.4593, + "theoretical_loss": 3.8642954044511875, + "tokens_seen": 606142464 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 207066, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3976249694824219, + "objective/train/theoretical_loss": 3.8642522091023506, + "objective/train/tokens_used": 606208000, + "theoretical_loss": 3.8642522091023506, + "tokens_seen": 606208000 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004122768304914744, + "loss": 1.5123, + "theoretical_loss": 3.8642522091023506, + "tokens_seen": 606208000 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041226680040120365, + "loss": 1.4315, + "theoretical_loss": 3.864209019730403, + "tokens_seen": 606273536 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004122567703109328, + "loss": 1.5234, + "theoretical_loss": 3.8641658363338705, + "tokens_seen": 606339072 + }, + { + "epoch": 0.18, + "learning_rate": 0.000412246740220662, + "loss": 1.4001, + "theoretical_loss": 3.8641226589112816, + "tokens_seen": 606404608 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041223671013039114, + "loss": 1.4637, + "theoretical_loss": 3.8640794874611646, + "tokens_seen": 606470144 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004122266800401204, + "loss": 1.4306, + "theoretical_loss": 3.8640363219820473, + "tokens_seen": 606535680 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041221664994984955, + "loss": 1.2474, + "theoretical_loss": 3.8639931624724593, + "tokens_seen": 606601216 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041220661985957874, + "loss": 1.4767, + "theoretical_loss": 3.8639500089309307, + "tokens_seen": 606666752 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121965897693079, + "loss": 1.3682, + "theoretical_loss": 3.86390686135599, + "tokens_seen": 606732288 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121865596790371, + "loss": 1.4438, + "theoretical_loss": 3.86386371974617, + "tokens_seen": 606797824 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121765295887663, + "loss": 1.3388, + "theoretical_loss": 3.863820584100001, + "tokens_seen": 606863360 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121664994984955, + "loss": 1.4743, + "theoretical_loss": 3.863777454416014, + "tokens_seen": 606928896 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121564694082247, + "loss": 1.5433, + "theoretical_loss": 3.863734330692742, + "tokens_seen": 606994432 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121464393179539, + "loss": 1.3988, + "theoretical_loss": 3.8636912129287175, + "tokens_seen": 607059968 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121364092276831, + "loss": 1.4847, + "theoretical_loss": 3.8636481011224744, + "tokens_seen": 607125504 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041212637913741224, + "loss": 1.4625, + "theoretical_loss": 3.8636049952725466, + "tokens_seen": 607191040 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121163490471415, + "loss": 1.4192, + "theoretical_loss": 3.863561895377467, + "tokens_seen": 607256576 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121063189568706, + "loss": 1.3762, + "theoretical_loss": 3.8635188014357715, + "tokens_seen": 607322112 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041209628886659984, + "loss": 1.4941, + "theoretical_loss": 3.8634757134459954, + "tokens_seen": 607387648 + }, + { + "epoch": 0.18, + "learning_rate": 0.000412086258776329, + "loss": 1.2928, + "theoretical_loss": 3.8634326314066745, + "tokens_seen": 607453184 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004120762286860582, + "loss": 1.3607, + "theoretical_loss": 3.8633895553163455, + "tokens_seen": 607518720 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004120661985957874, + "loss": 1.3228, + "theoretical_loss": 3.8633464851735444, + "tokens_seen": 607584256 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041205616850551656, + "loss": 1.3814, + "theoretical_loss": 3.8633034209768096, + "tokens_seen": 607649792 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041204613841524574, + "loss": 1.3442, + "theoretical_loss": 3.8632603627246787, + "tokens_seen": 607715328 + }, + { + "epoch": 0.18, + "learning_rate": 0.000412036108324975, + "loss": 1.4944, + "theoretical_loss": 3.86321731041569, + "tokens_seen": 607780864 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004120260782347041, + "loss": 1.4472, + "theoretical_loss": 3.863174264048383, + "tokens_seen": 607846400 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041201604814443334, + "loss": 1.3005, + "theoretical_loss": 3.8631312236212967, + "tokens_seen": 607911936 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041200601805416247, + "loss": 1.3328, + "theoretical_loss": 3.863088189132971, + "tokens_seen": 607977472 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004119959879638917, + "loss": 1.4355, + "theoretical_loss": 3.8630451605819474, + "tokens_seen": 608043008 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004119859578736209, + "loss": 1.5126, + "theoretical_loss": 3.8630021379667654, + "tokens_seen": 608108544 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041197592778335006, + "loss": 1.4461, + "theoretical_loss": 3.862959121285968, + "tokens_seen": 608174080 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041196589769307924, + "loss": 1.3855, + "theoretical_loss": 3.8629161105380962, + "tokens_seen": 608239616 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004119558676028085, + "loss": 1.439, + "theoretical_loss": 3.862873105721694, + "tokens_seen": 608305152 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004119458375125376, + "loss": 1.4095, + "theoretical_loss": 3.862830106835303, + "tokens_seen": 608370688 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041193580742226684, + "loss": 1.4525, + "theoretical_loss": 3.8627871138774674, + "tokens_seen": 608436224 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041192577733199597, + "loss": 1.3971, + "theoretical_loss": 3.862744126846732, + "tokens_seen": 608501760 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004119157472417252, + "loss": 1.4875, + "theoretical_loss": 3.8627011457416405, + "tokens_seen": 608567296 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004119057171514544, + "loss": 1.3721, + "theoretical_loss": 3.862658170560738, + "tokens_seen": 608632832 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041189568706118357, + "loss": 1.4779, + "theoretical_loss": 3.862615201302571, + "tokens_seen": 608698368 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041188565697091275, + "loss": 1.5816, + "theoretical_loss": 3.8625722379656855, + "tokens_seen": 608763904 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041187562688064193, + "loss": 1.3719, + "theoretical_loss": 3.862529280548628, + "tokens_seen": 608829440 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004118655967903711, + "loss": 1.3893, + "theoretical_loss": 3.8624863290499456, + "tokens_seen": 608894976 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041185556670010035, + "loss": 1.3861, + "theoretical_loss": 3.862443383468186, + "tokens_seen": 608960512 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041184553660982947, + "loss": 1.3982, + "theoretical_loss": 3.8624004438018984, + "tokens_seen": 609026048 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004118355065195587, + "loss": 1.3979, + "theoretical_loss": 3.8623575100496303, + "tokens_seen": 609091584 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041182547642928783, + "loss": 1.3908, + "theoretical_loss": 3.8623145822099314, + "tokens_seen": 609157120 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041181544633901707, + "loss": 1.4076, + "theoretical_loss": 3.8622716602813516, + "tokens_seen": 609222656 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041180541624874625, + "loss": 1.469, + "theoretical_loss": 3.862228744262441, + "tokens_seen": 609288192 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041179538615847543, + "loss": 1.3311, + "theoretical_loss": 3.8621858341517514, + "tokens_seen": 609353728 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004117853560682046, + "loss": 1.4763, + "theoretical_loss": 3.8621429299478325, + "tokens_seen": 609419264 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 207906, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3655412197113037, + "objective/train/theoretical_loss": 3.862100031649237, + "objective/train/tokens_used": 609484800, + "theoretical_loss": 3.862100031649237, + "tokens_seen": 609484800 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041177532597793385, + "loss": 1.3679, + "theoretical_loss": 3.862100031649237, + "tokens_seen": 609484800 + }, + { + "epoch": 0.18, + "learning_rate": 0.000411765295887663, + "loss": 1.4883, + "theoretical_loss": 3.8620571392545173, + "tokens_seen": 609550336 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004117552657973922, + "loss": 1.3978, + "theoretical_loss": 3.8620142527622265, + "tokens_seen": 609615872 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041174523570712134, + "loss": 1.2391, + "theoretical_loss": 3.8619713721709172, + "tokens_seen": 609681408 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004117352056168506, + "loss": 1.5168, + "theoretical_loss": 3.861928497479144, + "tokens_seen": 609746944 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041172517552657975, + "loss": 1.4225, + "theoretical_loss": 3.86188562868546, + "tokens_seen": 609812480 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041171514543630894, + "loss": 1.4774, + "theoretical_loss": 3.8618427657884222, + "tokens_seen": 609878016 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004117051153460381, + "loss": 1.4042, + "theoretical_loss": 3.861799908786584, + "tokens_seen": 609943552 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004116950852557673, + "loss": 1.3356, + "theoretical_loss": 3.8617570576785027, + "tokens_seen": 610009088 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004116850551654965, + "loss": 1.4131, + "theoretical_loss": 3.8617142124627346, + "tokens_seen": 610074624 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004116750250752257, + "loss": 1.4141, + "theoretical_loss": 3.8616713731378356, + "tokens_seen": 610140160 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041166499498495484, + "loss": 1.5105, + "theoretical_loss": 3.8616285397023637, + "tokens_seen": 610205696 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004116549648946841, + "loss": 1.3551, + "theoretical_loss": 3.861585712154877, + "tokens_seen": 610271232 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004116449348044132, + "loss": 1.4797, + "theoretical_loss": 3.8615428904939337, + "tokens_seen": 610336768 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041163490471414244, + "loss": 1.4566, + "theoretical_loss": 3.861500074718093, + "tokens_seen": 610402304 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004116248746238716, + "loss": 1.3026, + "theoretical_loss": 3.8614572648259142, + "tokens_seen": 610467840 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004116148445336008, + "loss": 1.4494, + "theoretical_loss": 3.8614144608159577, + "tokens_seen": 610533376 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041160481444333, + "loss": 1.3942, + "theoretical_loss": 3.8613716626867833, + "tokens_seen": 610598912 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004115947843530592, + "loss": 1.4515, + "theoretical_loss": 3.861328870436952, + "tokens_seen": 610664448 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041158475426278834, + "loss": 1.3963, + "theoretical_loss": 3.8612860840650267, + "tokens_seen": 610729984 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004115747241725176, + "loss": 1.4661, + "theoretical_loss": 3.8612433035695672, + "tokens_seen": 610795520 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004115646940822467, + "loss": 1.3155, + "theoretical_loss": 3.861200528949138, + "tokens_seen": 610861056 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041155466399197594, + "loss": 1.4283, + "theoretical_loss": 3.8611577602023006, + "tokens_seen": 610926592 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004115446339017051, + "loss": 1.4025, + "theoretical_loss": 3.8611149973276193, + "tokens_seen": 610992128 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004115346038114343, + "loss": 1.4351, + "theoretical_loss": 3.861072240323658, + "tokens_seen": 611057664 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004115245737211635, + "loss": 1.5012, + "theoretical_loss": 3.861029489188981, + "tokens_seen": 611123200 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041151454363089267, + "loss": 1.416, + "theoretical_loss": 3.8609867439221537, + "tokens_seen": 611188736 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041150451354062185, + "loss": 1.2814, + "theoretical_loss": 3.860944004521742, + "tokens_seen": 611254272 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004114944834503511, + "loss": 1.3663, + "theoretical_loss": 3.860901270986311, + "tokens_seen": 611319808 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004114844533600802, + "loss": 1.398, + "theoretical_loss": 3.8608585433144276, + "tokens_seen": 611385344 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041147442326980944, + "loss": 1.4482, + "theoretical_loss": 3.860815821504659, + "tokens_seen": 611450880 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041146439317953857, + "loss": 1.4999, + "theoretical_loss": 3.860773105555573, + "tokens_seen": 611516416 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004114543630892678, + "loss": 1.5392, + "theoretical_loss": 3.860730395465737, + "tokens_seen": 611581952 + }, + { + "epoch": 0.19, + "learning_rate": 0.000411444332998997, + "loss": 1.3917, + "theoretical_loss": 3.86068769123372, + "tokens_seen": 611647488 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041143430290872617, + "loss": 1.4501, + "theoretical_loss": 3.860644992858091, + "tokens_seen": 611713024 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041142427281845535, + "loss": 1.4131, + "theoretical_loss": 3.8606023003374195, + "tokens_seen": 611778560 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004114142427281846, + "loss": 1.285, + "theoretical_loss": 3.860559613670276, + "tokens_seen": 611844096 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041140421263791377, + "loss": 1.3966, + "theoretical_loss": 3.86051693285523, + "tokens_seen": 611909632 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041139418254764295, + "loss": 1.4, + "theoretical_loss": 3.8604742578908535, + "tokens_seen": 611975168 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041138415245737213, + "loss": 1.4336, + "theoretical_loss": 3.860431588775718, + "tokens_seen": 612040704 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004113741223671013, + "loss": 1.4095, + "theoretical_loss": 3.860388925508395, + "tokens_seen": 612106240 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041136409227683055, + "loss": 1.286, + "theoretical_loss": 3.8603462680874583, + "tokens_seen": 612171776 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041135406218655967, + "loss": 1.3609, + "theoretical_loss": 3.8603036165114797, + "tokens_seen": 612237312 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004113440320962889, + "loss": 1.4574, + "theoretical_loss": 3.8602609707790334, + "tokens_seen": 612302848 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041133400200601803, + "loss": 1.409, + "theoretical_loss": 3.860218330888693, + "tokens_seen": 612368384 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041132397191574727, + "loss": 1.3961, + "theoretical_loss": 3.8601756968390335, + "tokens_seen": 612433920 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041131394182547645, + "loss": 1.5618, + "theoretical_loss": 3.86013306862863, + "tokens_seen": 612499456 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041130391173520563, + "loss": 1.5252, + "theoretical_loss": 3.8600904462560583, + "tokens_seen": 612564992 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004112938816449348, + "loss": 1.5508, + "theoretical_loss": 3.8600478297198944, + "tokens_seen": 612630528 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041128385155466405, + "loss": 1.4555, + "theoretical_loss": 3.860005219018714, + "tokens_seen": 612696064 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 209248, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3387970924377441, + "objective/train/theoretical_loss": 3.8599626141510948, + "objective/train/tokens_used": 612761600, + "theoretical_loss": 3.8599626141510948, + "tokens_seen": 612761600 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004112738214643932, + "loss": 1.4192, + "theoretical_loss": 3.8599626141510948, + "tokens_seen": 612761600 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004112637913741224, + "loss": 1.5163, + "theoretical_loss": 3.859920015115615, + "tokens_seen": 612827136 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041125376128385154, + "loss": 1.3815, + "theoretical_loss": 3.8598774219108516, + "tokens_seen": 612892672 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004112437311935808, + "loss": 1.4135, + "theoretical_loss": 3.859834834535384, + "tokens_seen": 612958208 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041123370110330995, + "loss": 1.355, + "theoretical_loss": 3.859792252987791, + "tokens_seen": 613023744 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041122367101303914, + "loss": 1.4019, + "theoretical_loss": 3.859749677266652, + "tokens_seen": 613089280 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004112136409227683, + "loss": 1.3331, + "theoretical_loss": 3.8597071073705473, + "tokens_seen": 613154816 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004112036108324975, + "loss": 1.4634, + "theoretical_loss": 3.8596645432980567, + "tokens_seen": 613220352 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004111935807422267, + "loss": 1.441, + "theoretical_loss": 3.859621985047763, + "tokens_seen": 613285888 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004111835506519559, + "loss": 1.5162, + "theoretical_loss": 3.859579432618246, + "tokens_seen": 613351424 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041117352056168504, + "loss": 1.457, + "theoretical_loss": 3.8595368860080894, + "tokens_seen": 613416960 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004111634904714143, + "loss": 1.36, + "theoretical_loss": 3.8594943452158743, + "tokens_seen": 613482496 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004111534603811434, + "loss": 1.4249, + "theoretical_loss": 3.8594518102401842, + "tokens_seen": 613548032 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041114343029087264, + "loss": 1.4445, + "theoretical_loss": 3.8594092810796026, + "tokens_seen": 613613568 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004111334002006018, + "loss": 1.3509, + "theoretical_loss": 3.859366757732714, + "tokens_seen": 613679104 + }, + { + "epoch": 0.19, + "learning_rate": 0.000411123370110331, + "loss": 1.4581, + "theoretical_loss": 3.8593242401981027, + "tokens_seen": 613744640 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004111133400200602, + "loss": 1.336, + "theoretical_loss": 3.859281728474354, + "tokens_seen": 613810176 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004111033099297894, + "loss": 1.4482, + "theoretical_loss": 3.8592392225600527, + "tokens_seen": 613875712 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041109327983951854, + "loss": 1.4647, + "theoretical_loss": 3.859196722453785, + "tokens_seen": 613941248 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004110832497492478, + "loss": 1.3605, + "theoretical_loss": 3.859154228154138, + "tokens_seen": 614006784 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004110732196589769, + "loss": 1.4228, + "theoretical_loss": 3.8591117396596983, + "tokens_seen": 614072320 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041106318956870614, + "loss": 1.4425, + "theoretical_loss": 3.859069256969054, + "tokens_seen": 614137856 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004110531594784353, + "loss": 1.5116, + "theoretical_loss": 3.859026780080792, + "tokens_seen": 614203392 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004110431293881645, + "loss": 1.3658, + "theoretical_loss": 3.8589843089935014, + "tokens_seen": 614268928 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004110330992978937, + "loss": 1.4233, + "theoretical_loss": 3.8589418437057716, + "tokens_seen": 614334464 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041102306920762287, + "loss": 1.2414, + "theoretical_loss": 3.8588993842161923, + "tokens_seen": 614400000 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041101303911735205, + "loss": 1.4657, + "theoretical_loss": 3.858856930523352, + "tokens_seen": 614465536 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004110030090270813, + "loss": 1.448, + "theoretical_loss": 3.8588144826258426, + "tokens_seen": 614531072 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004109929789368104, + "loss": 1.3312, + "theoretical_loss": 3.8587720405222545, + "tokens_seen": 614596608 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041098294884653965, + "loss": 1.4424, + "theoretical_loss": 3.8587296042111796, + "tokens_seen": 614662144 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041097291875626877, + "loss": 1.3611, + "theoretical_loss": 3.8586871736912087, + "tokens_seen": 614727680 + }, + { + "epoch": 0.19, + "learning_rate": 0.000410962888665998, + "loss": 1.3226, + "theoretical_loss": 3.858644748960936, + "tokens_seen": 614793216 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004109528585757272, + "loss": 1.3903, + "theoretical_loss": 3.858602330018953, + "tokens_seen": 614858752 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041094282848545637, + "loss": 1.4181, + "theoretical_loss": 3.858559916863853, + "tokens_seen": 614924288 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041093279839518555, + "loss": 1.4315, + "theoretical_loss": 3.8585175094942317, + "tokens_seen": 614989824 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004109227683049148, + "loss": 1.3864, + "theoretical_loss": 3.8584751079086814, + "tokens_seen": 615055360 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004109127382146439, + "loss": 1.4151, + "theoretical_loss": 3.8584327121057984, + "tokens_seen": 615120896 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041090270812437315, + "loss": 1.4494, + "theoretical_loss": 3.858390322084178, + "tokens_seen": 615186432 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004108926780341023, + "loss": 1.4886, + "theoretical_loss": 3.8583479378424155, + "tokens_seen": 615251968 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004108826479438315, + "loss": 1.4212, + "theoretical_loss": 3.858305559379107, + "tokens_seen": 615317504 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004108726178535607, + "loss": 1.5024, + "theoretical_loss": 3.8582631866928505, + "tokens_seen": 615383040 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041086258776328987, + "loss": 1.4294, + "theoretical_loss": 3.858220819782243, + "tokens_seen": 615448576 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041085255767301905, + "loss": 1.3984, + "theoretical_loss": 3.858178458645882, + "tokens_seen": 615514112 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041084252758274823, + "loss": 1.4757, + "theoretical_loss": 3.858136103282366, + "tokens_seen": 615579648 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004108324974924774, + "loss": 1.4043, + "theoretical_loss": 3.8580937536902935, + "tokens_seen": 615645184 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041082246740220665, + "loss": 1.3632, + "theoretical_loss": 3.8580514098682643, + "tokens_seen": 615710720 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004108124373119358, + "loss": 1.2953, + "theoretical_loss": 3.8580090718148776, + "tokens_seen": 615776256 + }, + { + "epoch": 0.19, + "learning_rate": 0.000410802407221665, + "loss": 1.4226, + "theoretical_loss": 3.8579667395287345, + "tokens_seen": 615841792 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004107923771313942, + "loss": 1.4719, + "theoretical_loss": 3.8579244130084356, + "tokens_seen": 615907328 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004107823470411234, + "loss": 1.4471, + "theoretical_loss": 3.8578820922525816, + "tokens_seen": 615972864 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 210517, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2874932289123535, + "objective/train/theoretical_loss": 3.857839777259775, + "objective/train/tokens_used": 616038400, + "theoretical_loss": 3.857839777259775, + "tokens_seen": 616038400 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041077231695085256, + "loss": 1.3269, + "theoretical_loss": 3.857839777259775, + "tokens_seen": 616038400 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041076228686058174, + "loss": 1.4493, + "theoretical_loss": 3.857797468028618, + "tokens_seen": 616103936 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004107522567703109, + "loss": 1.3153, + "theoretical_loss": 3.8577551645577124, + "tokens_seen": 616169472 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041074222668004015, + "loss": 1.4707, + "theoretical_loss": 3.857712866845662, + "tokens_seen": 616235008 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004107321965897693, + "loss": 1.3559, + "theoretical_loss": 3.857670574891071, + "tokens_seen": 616300544 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004107221664994985, + "loss": 1.5543, + "theoretical_loss": 3.857628288692543, + "tokens_seen": 616366080 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041071213640922764, + "loss": 1.4461, + "theoretical_loss": 3.857586008248683, + "tokens_seen": 616431616 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004107021063189569, + "loss": 1.3718, + "theoretical_loss": 3.8575437335580958, + "tokens_seen": 616497152 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041069207622868606, + "loss": 1.3714, + "theoretical_loss": 3.8575014646193875, + "tokens_seen": 616562688 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041068204613841524, + "loss": 1.4505, + "theoretical_loss": 3.8574592014311637, + "tokens_seen": 616628224 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004106720160481444, + "loss": 1.5171, + "theoretical_loss": 3.857416943992032, + "tokens_seen": 616693760 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004106619859578736, + "loss": 1.3693, + "theoretical_loss": 3.8573746923005983, + "tokens_seen": 616759296 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041065195586760284, + "loss": 1.414, + "theoretical_loss": 3.857332446355471, + "tokens_seen": 616824832 + }, + { + "epoch": 0.19, + "learning_rate": 0.000410641925777332, + "loss": 1.4086, + "theoretical_loss": 3.857290206155258, + "tokens_seen": 616890368 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004106318956870612, + "loss": 1.4538, + "theoretical_loss": 3.8572479716985675, + "tokens_seen": 616955904 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004106218655967904, + "loss": 1.3915, + "theoretical_loss": 3.857205742984009, + "tokens_seen": 617021440 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004106118355065196, + "loss": 1.461, + "theoretical_loss": 3.857163520010192, + "tokens_seen": 617086976 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041060180541624874, + "loss": 1.5136, + "theoretical_loss": 3.8571213027757274, + "tokens_seen": 617152512 + }, + { + "epoch": 0.19, + "learning_rate": 0.000410591775325978, + "loss": 1.4521, + "theoretical_loss": 3.857079091279224, + "tokens_seen": 617218048 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004105817452357071, + "loss": 1.3507, + "theoretical_loss": 3.857036885519293, + "tokens_seen": 617283584 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041057171514543634, + "loss": 1.3923, + "theoretical_loss": 3.8569946854945467, + "tokens_seen": 617349120 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004105616850551655, + "loss": 1.4574, + "theoretical_loss": 3.8569524912035975, + "tokens_seen": 617414656 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004105516549648947, + "loss": 1.3677, + "theoretical_loss": 3.8569103026450566, + "tokens_seen": 617480192 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004105416248746239, + "loss": 1.4439, + "theoretical_loss": 3.8568681198175376, + "tokens_seen": 617545728 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041053159478435307, + "loss": 1.3001, + "theoretical_loss": 3.856825942719654, + "tokens_seen": 617611264 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041052156469408225, + "loss": 1.4421, + "theoretical_loss": 3.856783771350019, + "tokens_seen": 617676800 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004105115346038115, + "loss": 1.3919, + "theoretical_loss": 3.8567416057072474, + "tokens_seen": 617742336 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004105015045135406, + "loss": 1.3777, + "theoretical_loss": 3.8566994457899546, + "tokens_seen": 617807872 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041049147442326985, + "loss": 1.4082, + "theoretical_loss": 3.8566572915967554, + "tokens_seen": 617873408 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041048144433299897, + "loss": 1.3529, + "theoretical_loss": 3.8566151431262656, + "tokens_seen": 617938944 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004104714142427282, + "loss": 1.3967, + "theoretical_loss": 3.856573000377101, + "tokens_seen": 618004480 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004104613841524574, + "loss": 1.4737, + "theoretical_loss": 3.8565308633478796, + "tokens_seen": 618070016 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041045135406218657, + "loss": 1.4597, + "theoretical_loss": 3.856488732037218, + "tokens_seen": 618135552 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041044132397191575, + "loss": 1.3647, + "theoretical_loss": 3.856446606443734, + "tokens_seen": 618201088 + }, + { + "epoch": 0.19, + "learning_rate": 0.000410431293881645, + "loss": 1.3849, + "theoretical_loss": 3.8564044865660456, + "tokens_seen": 618266624 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004104212637913741, + "loss": 1.4709, + "theoretical_loss": 3.8563623724027716, + "tokens_seen": 618332160 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041041123370110335, + "loss": 1.4214, + "theoretical_loss": 3.856320263952531, + "tokens_seen": 618397696 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004104012036108325, + "loss": 1.4396, + "theoretical_loss": 3.856278161213944, + "tokens_seen": 618463232 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004103911735205617, + "loss": 1.3719, + "theoretical_loss": 3.856236064185631, + "tokens_seen": 618528768 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004103811434302909, + "loss": 1.4148, + "theoretical_loss": 3.856193972866212, + "tokens_seen": 618594304 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041037111334002007, + "loss": 1.4489, + "theoretical_loss": 3.8561518872543075, + "tokens_seen": 618659840 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041036108324974925, + "loss": 1.347, + "theoretical_loss": 3.85610980734854, + "tokens_seen": 618725376 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041035105315947844, + "loss": 1.424, + "theoretical_loss": 3.8560677331475315, + "tokens_seen": 618790912 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004103410230692076, + "loss": 1.3986, + "theoretical_loss": 3.8560256646499047, + "tokens_seen": 618856448 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041033099297893685, + "loss": 1.4186, + "theoretical_loss": 3.855983601854282, + "tokens_seen": 618921984 + }, + { + "epoch": 0.19, + "learning_rate": 0.000410320962888666, + "loss": 1.4958, + "theoretical_loss": 3.8559415447592875, + "tokens_seen": 618987520 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004103109327983952, + "loss": 1.4404, + "theoretical_loss": 3.8558994933635447, + "tokens_seen": 619053056 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004103009027081244, + "loss": 1.4463, + "theoretical_loss": 3.855857447665678, + "tokens_seen": 619118592 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004102908726178536, + "loss": 1.4448, + "theoretical_loss": 3.855815407664313, + "tokens_seen": 619184128 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041028084252758276, + "loss": 1.3987, + "theoretical_loss": 3.8557733733580744, + "tokens_seen": 619249664 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 211365, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6160850524902344, + "objective/train/theoretical_loss": 3.855731344745589, + "objective/train/tokens_used": 619315200, + "theoretical_loss": 3.855731344745589, + "tokens_seen": 619315200 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041027081243731194, + "loss": 1.5406, + "theoretical_loss": 3.855731344745589, + "tokens_seen": 619315200 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004102607823470411, + "loss": 1.3547, + "theoretical_loss": 3.8556893218254817, + "tokens_seen": 619380736 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041025075225677035, + "loss": 1.5176, + "theoretical_loss": 3.855647304596381, + "tokens_seen": 619446272 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004102407221664995, + "loss": 1.5472, + "theoretical_loss": 3.8556052930569127, + "tokens_seen": 619511808 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004102306920762287, + "loss": 1.3563, + "theoretical_loss": 3.8555632872057055, + "tokens_seen": 619577344 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041022066198595784, + "loss": 1.4078, + "theoretical_loss": 3.8555212870413875, + "tokens_seen": 619642880 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004102106318956871, + "loss": 1.4379, + "theoretical_loss": 3.8554792925625874, + "tokens_seen": 619708416 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041020060180541626, + "loss": 1.4382, + "theoretical_loss": 3.855437303767934, + "tokens_seen": 619773952 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041019057171514544, + "loss": 1.4831, + "theoretical_loss": 3.855395320656058, + "tokens_seen": 619839488 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004101805416248746, + "loss": 1.457, + "theoretical_loss": 3.8553533432255884, + "tokens_seen": 619905024 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004101705115346038, + "loss": 1.5306, + "theoretical_loss": 3.855311371475157, + "tokens_seen": 619970560 + }, + { + "epoch": 0.19, + "learning_rate": 0.000410160481444333, + "loss": 1.4522, + "theoretical_loss": 3.8552694054033942, + "tokens_seen": 620036096 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004101504513540622, + "loss": 1.5127, + "theoretical_loss": 3.855227445008931, + "tokens_seen": 620101632 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041014042126379135, + "loss": 1.3427, + "theoretical_loss": 3.8551854902904017, + "tokens_seen": 620167168 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004101303911735206, + "loss": 1.2554, + "theoretical_loss": 3.855143541246436, + "tokens_seen": 620232704 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041012036108324976, + "loss": 1.367, + "theoretical_loss": 3.855101597875669, + "tokens_seen": 620298240 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041011033099297894, + "loss": 1.5375, + "theoretical_loss": 3.8550596601767326, + "tokens_seen": 620363776 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004101003009027081, + "loss": 1.3383, + "theoretical_loss": 3.8550177281482627, + "tokens_seen": 620429312 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004100902708124373, + "loss": 1.3627, + "theoretical_loss": 3.8549758017888918, + "tokens_seen": 620494848 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004100802407221665, + "loss": 1.4259, + "theoretical_loss": 3.8549338810972564, + "tokens_seen": 620560384 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004100702106318957, + "loss": 1.4714, + "theoretical_loss": 3.8548919660719907, + "tokens_seen": 620625920 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041006018054162485, + "loss": 1.3031, + "theoretical_loss": 3.854850056711731, + "tokens_seen": 620691456 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004100501504513541, + "loss": 1.525, + "theoretical_loss": 3.8548081530151137, + "tokens_seen": 620756992 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004100401203610832, + "loss": 1.4851, + "theoretical_loss": 3.8547662549807757, + "tokens_seen": 620822528 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041003009027081245, + "loss": 1.5895, + "theoretical_loss": 3.854724362607354, + "tokens_seen": 620888064 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041002006018054163, + "loss": 1.3473, + "theoretical_loss": 3.854682475893486, + "tokens_seen": 620953600 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004100100300902708, + "loss": 1.3493, + "theoretical_loss": 3.854640594837811, + "tokens_seen": 621019136 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041, + "loss": 1.4134, + "theoretical_loss": 3.8545987194389664, + "tokens_seen": 621084672 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040998996990972917, + "loss": 1.4383, + "theoretical_loss": 3.8545568496955926, + "tokens_seen": 621150208 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040997993981945835, + "loss": 1.2904, + "theoretical_loss": 3.854514985606328, + "tokens_seen": 621215744 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004099699097291876, + "loss": 1.4787, + "theoretical_loss": 3.854473127169814, + "tokens_seen": 621281280 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004099598796389167, + "loss": 1.48, + "theoretical_loss": 3.85443127438469, + "tokens_seen": 621346816 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040994984954864595, + "loss": 1.4596, + "theoretical_loss": 3.8543894272495978, + "tokens_seen": 621412352 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040993981945837513, + "loss": 1.3681, + "theoretical_loss": 3.8543475857631786, + "tokens_seen": 621477888 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004099297893681043, + "loss": 1.4688, + "theoretical_loss": 3.854305749924075, + "tokens_seen": 621543424 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004099197592778335, + "loss": 1.4672, + "theoretical_loss": 3.854263919730928, + "tokens_seen": 621608960 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004099097291875627, + "loss": 1.3701, + "theoretical_loss": 3.8542220951823816, + "tokens_seen": 621674496 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004098996990972919, + "loss": 1.4688, + "theoretical_loss": 3.854180276277079, + "tokens_seen": 621740032 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004098896690070211, + "loss": 1.4821, + "theoretical_loss": 3.8541384630136646, + "tokens_seen": 621805568 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004098796389167503, + "loss": 1.3877, + "theoretical_loss": 3.854096655390781, + "tokens_seen": 621871104 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040986960882647945, + "loss": 1.4491, + "theoretical_loss": 3.8540548534070753, + "tokens_seen": 621936640 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040985957873620864, + "loss": 1.5364, + "theoretical_loss": 3.854013057061191, + "tokens_seen": 622002176 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004098495486459378, + "loss": 1.4557, + "theoretical_loss": 3.853971266351775, + "tokens_seen": 622067712 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040983951855566705, + "loss": 1.4952, + "theoretical_loss": 3.853929481277472, + "tokens_seen": 622133248 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004098294884653962, + "loss": 1.3974, + "theoretical_loss": 3.8538877018369306, + "tokens_seen": 622198784 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004098194583751254, + "loss": 1.494, + "theoretical_loss": 3.8538459280287967, + "tokens_seen": 622264320 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004098094282848546, + "loss": 1.5195, + "theoretical_loss": 3.853804159851718, + "tokens_seen": 622329856 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004097993981945838, + "loss": 1.3011, + "theoretical_loss": 3.853762397304343, + "tokens_seen": 622395392 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040978936810431296, + "loss": 1.4012, + "theoretical_loss": 3.853720640385319, + "tokens_seen": 622460928 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040977933801404214, + "loss": 1.387, + "theoretical_loss": 3.853678889093297, + "tokens_seen": 622526464 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 212592, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4932806491851807, + "objective/train/theoretical_loss": 3.8536371434269254, + "objective/train/tokens_used": 622592000, + "theoretical_loss": 3.8536371434269254, + "tokens_seen": 622592000 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004097693079237713, + "loss": 1.5096, + "theoretical_loss": 3.8536371434269254, + "tokens_seen": 622592000 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040975927783350055, + "loss": 1.3459, + "theoretical_loss": 3.853595403384854, + "tokens_seen": 622657536 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004097492477432297, + "loss": 1.492, + "theoretical_loss": 3.853553668965733, + "tokens_seen": 622723072 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004097392176529589, + "loss": 1.5565, + "theoretical_loss": 3.8535119401682136, + "tokens_seen": 622788608 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040972918756268804, + "loss": 1.4086, + "theoretical_loss": 3.8534702169909476, + "tokens_seen": 622854144 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004097191574724173, + "loss": 1.4144, + "theoretical_loss": 3.853428499432586, + "tokens_seen": 622919680 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040970912738214646, + "loss": 1.3788, + "theoretical_loss": 3.8533867874917815, + "tokens_seen": 622985216 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040969909729187564, + "loss": 1.2752, + "theoretical_loss": 3.8533450811671877, + "tokens_seen": 623050752 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004096890672016048, + "loss": 1.4469, + "theoretical_loss": 3.8533033804574557, + "tokens_seen": 623116288 + }, + { + "epoch": 0.19, + "learning_rate": 0.000409679037111334, + "loss": 1.3764, + "theoretical_loss": 3.8532616853612405, + "tokens_seen": 623181824 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004096690070210632, + "loss": 1.4652, + "theoretical_loss": 3.853219995877196, + "tokens_seen": 623247360 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004096589769307924, + "loss": 1.3691, + "theoretical_loss": 3.853178312003977, + "tokens_seen": 623312896 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040964894684052155, + "loss": 1.4164, + "theoretical_loss": 3.853136633740239, + "tokens_seen": 623378432 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004096389167502508, + "loss": 1.3249, + "theoretical_loss": 3.853094961084636, + "tokens_seen": 623443968 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040962888665997996, + "loss": 1.4062, + "theoretical_loss": 3.853053294035825, + "tokens_seen": 623509504 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040961885656970914, + "loss": 1.3786, + "theoretical_loss": 3.8530116325924624, + "tokens_seen": 623575040 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004096088264794383, + "loss": 1.4046, + "theoretical_loss": 3.8529699767532053, + "tokens_seen": 623640576 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004095987963891675, + "loss": 1.4143, + "theoretical_loss": 3.85292832651671, + "tokens_seen": 623706112 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004095887662988967, + "loss": 1.4054, + "theoretical_loss": 3.8528866818816354, + "tokens_seen": 623771648 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004095787362086259, + "loss": 1.4808, + "theoretical_loss": 3.8528450428466403, + "tokens_seen": 623837184 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040956870611835505, + "loss": 1.4015, + "theoretical_loss": 3.852803409410382, + "tokens_seen": 623902720 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004095586760280843, + "loss": 1.4162, + "theoretical_loss": 3.85276178157152, + "tokens_seen": 623968256 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004095486459378134, + "loss": 1.4255, + "theoretical_loss": 3.852720159328715, + "tokens_seen": 624033792 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040953861584754265, + "loss": 1.4914, + "theoretical_loss": 3.852678542680626, + "tokens_seen": 624099328 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040952858575727183, + "loss": 1.4366, + "theoretical_loss": 3.8526369316259137, + "tokens_seen": 624164864 + }, + { + "epoch": 0.19, + "learning_rate": 0.000409518555667001, + "loss": 1.5645, + "theoretical_loss": 3.8525953261632404, + "tokens_seen": 624230400 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004095085255767302, + "loss": 1.4011, + "theoretical_loss": 3.852553726291266, + "tokens_seen": 624295936 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040949849548645937, + "loss": 1.3871, + "theoretical_loss": 3.8525121320086537, + "tokens_seen": 624361472 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040948846539618855, + "loss": 1.3294, + "theoretical_loss": 3.8524705433140656, + "tokens_seen": 624427008 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004094784353059178, + "loss": 1.4146, + "theoretical_loss": 3.8524289602061645, + "tokens_seen": 624492544 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004094684052156469, + "loss": 1.3949, + "theoretical_loss": 3.8523873826836135, + "tokens_seen": 624558080 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040945837512537615, + "loss": 1.4344, + "theoretical_loss": 3.8523458107450765, + "tokens_seen": 624623616 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040944834503510533, + "loss": 1.4751, + "theoretical_loss": 3.8523042443892184, + "tokens_seen": 624689152 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004094383149448345, + "loss": 1.43, + "theoretical_loss": 3.852262683614703, + "tokens_seen": 624754688 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004094282848545637, + "loss": 1.3712, + "theoretical_loss": 3.8522211284201964, + "tokens_seen": 624820224 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004094182547642929, + "loss": 1.4641, + "theoretical_loss": 3.852179578804364, + "tokens_seen": 624885760 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040940822467402206, + "loss": 1.5001, + "theoretical_loss": 3.852138034765872, + "tokens_seen": 624951296 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004093981945837513, + "loss": 1.4993, + "theoretical_loss": 3.8520964963033864, + "tokens_seen": 625016832 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004093881644934804, + "loss": 1.3621, + "theoretical_loss": 3.8520549634155747, + "tokens_seen": 625082368 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040937813440320965, + "loss": 1.5337, + "theoretical_loss": 3.852013436101104, + "tokens_seen": 625147904 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004093681043129388, + "loss": 1.3864, + "theoretical_loss": 3.851971914358643, + "tokens_seen": 625213440 + }, + { + "epoch": 0.19, + "learning_rate": 0.000409358074222668, + "loss": 1.4354, + "theoretical_loss": 3.85193039818686, + "tokens_seen": 625278976 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004093480441323972, + "loss": 1.3779, + "theoretical_loss": 3.8518888875844226, + "tokens_seen": 625344512 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004093380140421264, + "loss": 1.4879, + "theoretical_loss": 3.8518473825500017, + "tokens_seen": 625410048 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040932798395185556, + "loss": 1.5164, + "theoretical_loss": 3.8518058830822666, + "tokens_seen": 625475584 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004093179538615848, + "loss": 1.4601, + "theoretical_loss": 3.8517643891798867, + "tokens_seen": 625541120 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004093079237713139, + "loss": 1.3838, + "theoretical_loss": 3.8517229008415343, + "tokens_seen": 625606656 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040929789368104316, + "loss": 1.4055, + "theoretical_loss": 3.8516814180658794, + "tokens_seen": 625672192 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004092878635907723, + "loss": 1.5325, + "theoretical_loss": 3.8516399408515936, + "tokens_seen": 625737728 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004092778335005015, + "loss": 1.4499, + "theoretical_loss": 3.8515984691973495, + "tokens_seen": 625803264 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 213430, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4723697900772095, + "objective/train/theoretical_loss": 3.85155700310182, + "objective/train/tokens_used": 625868800, + "theoretical_loss": 3.85155700310182, + "tokens_seen": 625868800 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004092678034102307, + "loss": 1.4288, + "theoretical_loss": 3.85155700310182, + "tokens_seen": 625868800 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004092577733199599, + "loss": 1.3957, + "theoretical_loss": 3.8515155425636767, + "tokens_seen": 625934336 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040924774322968906, + "loss": 1.3929, + "theoretical_loss": 3.851474087581594, + "tokens_seen": 625999872 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040923771313941824, + "loss": 1.4186, + "theoretical_loss": 3.8514326381542454, + "tokens_seen": 626065408 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004092276830491474, + "loss": 1.3943, + "theoretical_loss": 3.851391194280305, + "tokens_seen": 626130944 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040921765295887666, + "loss": 1.3493, + "theoretical_loss": 3.8513497559584495, + "tokens_seen": 626196480 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004092076228686058, + "loss": 1.4762, + "theoretical_loss": 3.851308323187352, + "tokens_seen": 626262016 + }, + { + "epoch": 0.19, + "learning_rate": 0.000409197592778335, + "loss": 1.3999, + "theoretical_loss": 3.851266895965689, + "tokens_seen": 626327552 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040918756268806415, + "loss": 1.4598, + "theoretical_loss": 3.8512254742921357, + "tokens_seen": 626393088 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004091775325977934, + "loss": 1.2946, + "theoretical_loss": 3.8511840581653707, + "tokens_seen": 626458624 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040916750250752257, + "loss": 1.426, + "theoretical_loss": 3.85114264758407, + "tokens_seen": 626524160 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040915747241725175, + "loss": 1.4621, + "theoretical_loss": 3.8511012425469104, + "tokens_seen": 626589696 + }, + { + "epoch": 0.19, + "learning_rate": 0.000409147442326981, + "loss": 1.3592, + "theoretical_loss": 3.851059843052571, + "tokens_seen": 626655232 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040913741223671016, + "loss": 1.4718, + "theoretical_loss": 3.85101844909973, + "tokens_seen": 626720768 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040912738214643934, + "loss": 1.5252, + "theoretical_loss": 3.8509770606870664, + "tokens_seen": 626786304 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004091173520561685, + "loss": 1.3654, + "theoretical_loss": 3.8509356778132586, + "tokens_seen": 626851840 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004091073219658977, + "loss": 1.4818, + "theoretical_loss": 3.8508943004769876, + "tokens_seen": 626917376 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004090972918756269, + "loss": 1.4134, + "theoretical_loss": 3.8508529286769324, + "tokens_seen": 626982912 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004090872617853561, + "loss": 1.4354, + "theoretical_loss": 3.850811562411775, + "tokens_seen": 627048448 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040907723169508525, + "loss": 1.4678, + "theoretical_loss": 3.8507702016801963, + "tokens_seen": 627113984 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004090672016048145, + "loss": 1.4272, + "theoretical_loss": 3.850728846480877, + "tokens_seen": 627179520 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004090571715145436, + "loss": 1.3847, + "theoretical_loss": 3.8506874968125, + "tokens_seen": 627245056 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040904714142427285, + "loss": 1.3331, + "theoretical_loss": 3.8506461526737477, + "tokens_seen": 627310592 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040903711133400203, + "loss": 1.5105, + "theoretical_loss": 3.8506048140633027, + "tokens_seen": 627376128 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004090270812437312, + "loss": 1.3685, + "theoretical_loss": 3.850563480979849, + "tokens_seen": 627441664 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004090170511534604, + "loss": 1.4974, + "theoretical_loss": 3.8505221534220695, + "tokens_seen": 627507200 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040900702106318957, + "loss": 1.4632, + "theoretical_loss": 3.8504808313886496, + "tokens_seen": 627572736 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040899699097291875, + "loss": 1.4725, + "theoretical_loss": 3.850439514878273, + "tokens_seen": 627638272 + }, + { + "epoch": 0.19, + "learning_rate": 0.000408986960882648, + "loss": 1.474, + "theoretical_loss": 3.8503982038896263, + "tokens_seen": 627703808 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004089769307923771, + "loss": 1.3769, + "theoretical_loss": 3.850356898421394, + "tokens_seen": 627769344 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040896690070210635, + "loss": 1.3358, + "theoretical_loss": 3.850315598472262, + "tokens_seen": 627834880 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040895687061183553, + "loss": 1.4425, + "theoretical_loss": 3.8502743040409184, + "tokens_seen": 627900416 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004089468405215647, + "loss": 1.4122, + "theoretical_loss": 3.850233015126048, + "tokens_seen": 627965952 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004089368104312939, + "loss": 1.3359, + "theoretical_loss": 3.8501917317263405, + "tokens_seen": 628031488 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004089267803410231, + "loss": 1.5649, + "theoretical_loss": 3.8501504538404827, + "tokens_seen": 628097024 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040891675025075226, + "loss": 1.4695, + "theoretical_loss": 3.850109181467163, + "tokens_seen": 628162560 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004089067201604815, + "loss": 1.4743, + "theoretical_loss": 3.85006791460507, + "tokens_seen": 628228096 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004088966900702106, + "loss": 1.4279, + "theoretical_loss": 3.850026653252894, + "tokens_seen": 628293632 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040888665997993985, + "loss": 1.5004, + "theoretical_loss": 3.8499853974093234, + "tokens_seen": 628359168 + }, + { + "epoch": 0.19, + "learning_rate": 0.000408876629889669, + "loss": 1.3995, + "theoretical_loss": 3.8499441470730487, + "tokens_seen": 628424704 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004088665997993982, + "loss": 1.5307, + "theoretical_loss": 3.849902902242761, + "tokens_seen": 628490240 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004088565697091274, + "loss": 1.4403, + "theoretical_loss": 3.849861662917151, + "tokens_seen": 628555776 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004088465396188566, + "loss": 1.3583, + "theoretical_loss": 3.8498204290949105, + "tokens_seen": 628621312 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040883650952858576, + "loss": 1.4843, + "theoretical_loss": 3.849779200774731, + "tokens_seen": 628686848 + }, + { + "epoch": 0.19, + "learning_rate": 0.000408826479438315, + "loss": 1.4145, + "theoretical_loss": 3.8497379779553054, + "tokens_seen": 628752384 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004088164493480441, + "loss": 1.4054, + "theoretical_loss": 3.849696760635326, + "tokens_seen": 628817920 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040880641925777336, + "loss": 1.502, + "theoretical_loss": 3.8496555488134865, + "tokens_seen": 628883456 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004087963891675025, + "loss": 1.4238, + "theoretical_loss": 3.84961434248848, + "tokens_seen": 628948992 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004087863590772317, + "loss": 1.5453, + "theoretical_loss": 3.8495731416590018, + "tokens_seen": 629014528 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004087763289869609, + "loss": 1.3688, + "theoretical_loss": 3.8495319463237454, + "tokens_seen": 629080064 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 214597, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4572769403457642, + "objective/train/theoretical_loss": 3.8494907564814067, + "objective/train/tokens_used": 629145600, + "theoretical_loss": 3.8494907564814067, + "tokens_seen": 629145600 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004087662988966901, + "loss": 1.3962, + "theoretical_loss": 3.8494907564814067, + "tokens_seen": 629145600 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040875626880641926, + "loss": 1.3657, + "theoretical_loss": 3.8494495721306805, + "tokens_seen": 629211136 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040874623871614844, + "loss": 1.3799, + "theoretical_loss": 3.849408393270264, + "tokens_seen": 629276672 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004087362086258776, + "loss": 1.4142, + "theoretical_loss": 3.849367219898852, + "tokens_seen": 629342208 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040872617853560686, + "loss": 1.5438, + "theoretical_loss": 3.849326052015142, + "tokens_seen": 629407744 + }, + { + "epoch": 0.19, + "learning_rate": 0.000408716148445336, + "loss": 1.3991, + "theoretical_loss": 3.8492848896178318, + "tokens_seen": 629473280 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004087061183550652, + "loss": 1.4813, + "theoretical_loss": 3.8492437327056184, + "tokens_seen": 629538816 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040869608826479435, + "loss": 1.497, + "theoretical_loss": 3.8492025812772006, + "tokens_seen": 629604352 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004086860581745236, + "loss": 1.4579, + "theoretical_loss": 3.849161435331277, + "tokens_seen": 629669888 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040867602808425277, + "loss": 1.364, + "theoretical_loss": 3.849120294866546, + "tokens_seen": 629735424 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040866599799398195, + "loss": 1.5423, + "theoretical_loss": 3.8490791598817076, + "tokens_seen": 629800960 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040865596790371113, + "loss": 1.3264, + "theoretical_loss": 3.8490380303754623, + "tokens_seen": 629866496 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040864593781344036, + "loss": 1.4766, + "theoretical_loss": 3.8489969063465095, + "tokens_seen": 629932032 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004086359077231695, + "loss": 1.4519, + "theoretical_loss": 3.8489557877935505, + "tokens_seen": 629997568 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004086258776328987, + "loss": 1.4145, + "theoretical_loss": 3.848914674715287, + "tokens_seen": 630063104 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040861584754262785, + "loss": 1.3949, + "theoretical_loss": 3.84887356711042, + "tokens_seen": 630128640 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004086058174523571, + "loss": 1.5401, + "theoretical_loss": 3.8488324649776517, + "tokens_seen": 630194176 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040859578736208627, + "loss": 1.3449, + "theoretical_loss": 3.848791368315686, + "tokens_seen": 630259712 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040858575727181545, + "loss": 1.4374, + "theoretical_loss": 3.848750277123224, + "tokens_seen": 630325248 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040857572718154463, + "loss": 1.4786, + "theoretical_loss": 3.848709191398971, + "tokens_seen": 630390784 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004085656970912738, + "loss": 1.3665, + "theoretical_loss": 3.8486681111416297, + "tokens_seen": 630456320 + }, + { + "epoch": 0.19, + "learning_rate": 0.000408555667001003, + "loss": 1.3422, + "theoretical_loss": 3.848627036349905, + "tokens_seen": 630521856 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040854563691073223, + "loss": 1.4137, + "theoretical_loss": 3.848585967022502, + "tokens_seen": 630587392 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040853560682046136, + "loss": 1.4107, + "theoretical_loss": 3.8485449031581256, + "tokens_seen": 630652928 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004085255767301906, + "loss": 1.5621, + "theoretical_loss": 3.8485038447554816, + "tokens_seen": 630718464 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004085155466399197, + "loss": 1.4306, + "theoretical_loss": 3.848462791813276, + "tokens_seen": 630784000 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040850551654964895, + "loss": 1.4636, + "theoretical_loss": 3.8484217443302153, + "tokens_seen": 630849536 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040849548645937814, + "loss": 1.3344, + "theoretical_loss": 3.8483807023050076, + "tokens_seen": 630915072 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004084854563691073, + "loss": 1.4795, + "theoretical_loss": 3.848339665736359, + "tokens_seen": 630980608 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004084754262788365, + "loss": 1.5341, + "theoretical_loss": 3.8482986346229784, + "tokens_seen": 631046144 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040846539618856573, + "loss": 1.3952, + "theoretical_loss": 3.8482576089635736, + "tokens_seen": 631111680 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040845536609829486, + "loss": 1.4154, + "theoretical_loss": 3.848216588756854, + "tokens_seen": 631177216 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004084453360080241, + "loss": 1.4295, + "theoretical_loss": 3.8481755740015275, + "tokens_seen": 631242752 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004084353059177532, + "loss": 1.3908, + "theoretical_loss": 3.848134564696305, + "tokens_seen": 631308288 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040842527582748246, + "loss": 1.4209, + "theoretical_loss": 3.8480935608398967, + "tokens_seen": 631373824 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040841524573721164, + "loss": 1.4546, + "theoretical_loss": 3.8480525624310125, + "tokens_seen": 631439360 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004084052156469408, + "loss": 1.4005, + "theoretical_loss": 3.8480115694683636, + "tokens_seen": 631504896 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040839518555667005, + "loss": 1.4151, + "theoretical_loss": 3.847970581950662, + "tokens_seen": 631570432 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004083851554663992, + "loss": 1.3936, + "theoretical_loss": 3.8479295998766188, + "tokens_seen": 631635968 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004083751253761284, + "loss": 1.489, + "theoretical_loss": 3.8478886232449465, + "tokens_seen": 631701504 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004083650952858576, + "loss": 1.4279, + "theoretical_loss": 3.847847652054358, + "tokens_seen": 631767040 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004083550651955868, + "loss": 1.3861, + "theoretical_loss": 3.8478066863035663, + "tokens_seen": 631832576 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040834503510531596, + "loss": 1.4161, + "theoretical_loss": 3.8477657259912856, + "tokens_seen": 631898112 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004083350050150452, + "loss": 1.3878, + "theoretical_loss": 3.847724771116229, + "tokens_seen": 631963648 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004083249749247743, + "loss": 1.4363, + "theoretical_loss": 3.847683821677112, + "tokens_seen": 632029184 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040831494483450356, + "loss": 1.4694, + "theoretical_loss": 3.847642877672649, + "tokens_seen": 632094720 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004083049147442327, + "loss": 1.4146, + "theoretical_loss": 3.847601939101555, + "tokens_seen": 632160256 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004082948846539619, + "loss": 1.405, + "theoretical_loss": 3.8475610059625467, + "tokens_seen": 632225792 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004082848545636911, + "loss": 1.3997, + "theoretical_loss": 3.84752007825434, + "tokens_seen": 632291328 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004082748244734203, + "loss": 1.377, + "theoretical_loss": 3.8474791559756514, + "tokens_seen": 632356864 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 215497, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4086350202560425, + "objective/train/theoretical_loss": 3.8474382391251982, + "objective/train/tokens_used": 632422400, + "theoretical_loss": 3.8474382391251982, + "tokens_seen": 632422400 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040826479438314946, + "loss": 1.4463, + "theoretical_loss": 3.8474382391251982, + "tokens_seen": 632422400 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040825476429287864, + "loss": 1.3743, + "theoretical_loss": 3.8473973277016977, + "tokens_seen": 632487936 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004082447342026078, + "loss": 1.4619, + "theoretical_loss": 3.847356421703868, + "tokens_seen": 632553472 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040823470411233706, + "loss": 1.4277, + "theoretical_loss": 3.8473155211304277, + "tokens_seen": 632619008 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004082246740220662, + "loss": 1.3442, + "theoretical_loss": 3.847274625980096, + "tokens_seen": 632684544 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004082146439317954, + "loss": 1.4306, + "theoretical_loss": 3.8472337362515914, + "tokens_seen": 632750080 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040820461384152455, + "loss": 1.4232, + "theoretical_loss": 3.8471928519436345, + "tokens_seen": 632815616 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004081945837512538, + "loss": 1.4837, + "theoretical_loss": 3.8471519730549444, + "tokens_seen": 632881152 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040818455366098297, + "loss": 1.4475, + "theoretical_loss": 3.847111099584242, + "tokens_seen": 632946688 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040817452357071215, + "loss": 1.4409, + "theoretical_loss": 3.8470702315302496, + "tokens_seen": 633012224 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040816449348044133, + "loss": 1.4593, + "theoretical_loss": 3.847029368891687, + "tokens_seen": 633077760 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040815446339017056, + "loss": 1.4683, + "theoretical_loss": 3.846988511667277, + "tokens_seen": 633143296 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004081444332998997, + "loss": 1.5426, + "theoretical_loss": 3.8469476598557417, + "tokens_seen": 633208832 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004081344032096289, + "loss": 1.5143, + "theoretical_loss": 3.8469068134558038, + "tokens_seen": 633274368 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040812437311935805, + "loss": 1.4131, + "theoretical_loss": 3.846865972466187, + "tokens_seen": 633339904 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004081143430290873, + "loss": 1.3702, + "theoretical_loss": 3.846825136885615, + "tokens_seen": 633405440 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040810431293881647, + "loss": 1.401, + "theoretical_loss": 3.8467843067128102, + "tokens_seen": 633470976 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040809428284854565, + "loss": 1.4448, + "theoretical_loss": 3.8467434819464996, + "tokens_seen": 633536512 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040808425275827483, + "loss": 1.4285, + "theoretical_loss": 3.8467026625854066, + "tokens_seen": 633602048 + }, + { + "epoch": 0.19, + "learning_rate": 0.000408074222668004, + "loss": 1.5044, + "theoretical_loss": 3.8466618486282567, + "tokens_seen": 633667584 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004080641925777332, + "loss": 1.4577, + "theoretical_loss": 3.8466210400737757, + "tokens_seen": 633733120 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040805416248746243, + "loss": 1.3548, + "theoretical_loss": 3.8465802369206905, + "tokens_seen": 633798656 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040804413239719156, + "loss": 1.4091, + "theoretical_loss": 3.846539439167727, + "tokens_seen": 633864192 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004080341023069208, + "loss": 1.5297, + "theoretical_loss": 3.846498646813613, + "tokens_seen": 633929728 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004080240722166499, + "loss": 1.4651, + "theoretical_loss": 3.8464578598570753, + "tokens_seen": 633995264 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040801404212637915, + "loss": 1.414, + "theoretical_loss": 3.8464170782968417, + "tokens_seen": 634060800 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040800401203610834, + "loss": 1.3874, + "theoretical_loss": 3.8463763021316417, + "tokens_seen": 634126336 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004079939819458375, + "loss": 1.4179, + "theoretical_loss": 3.8463355313602037, + "tokens_seen": 634191872 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004079839518555667, + "loss": 1.3834, + "theoretical_loss": 3.8462947659812565, + "tokens_seen": 634257408 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040797392176529593, + "loss": 1.2767, + "theoretical_loss": 3.84625400599353, + "tokens_seen": 634322944 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040796389167502506, + "loss": 1.4496, + "theoretical_loss": 3.846213251395755, + "tokens_seen": 634388480 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004079538615847543, + "loss": 1.4506, + "theoretical_loss": 3.846172502186661, + "tokens_seen": 634454016 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004079438314944834, + "loss": 1.3864, + "theoretical_loss": 3.8461317583649794, + "tokens_seen": 634519552 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040793380140421266, + "loss": 1.5003, + "theoretical_loss": 3.8460910199294416, + "tokens_seen": 634585088 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040792377131394184, + "loss": 1.4427, + "theoretical_loss": 3.8460502868787794, + "tokens_seen": 634650624 + }, + { + "epoch": 0.19, + "learning_rate": 0.000407913741223671, + "loss": 1.4519, + "theoretical_loss": 3.8460095592117254, + "tokens_seen": 634716160 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004079037111334002, + "loss": 1.4658, + "theoretical_loss": 3.845968836927012, + "tokens_seen": 634781696 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004078936810431294, + "loss": 1.4297, + "theoretical_loss": 3.8459281200233724, + "tokens_seen": 634847232 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040788365095285856, + "loss": 1.5219, + "theoretical_loss": 3.8458874084995402, + "tokens_seen": 634912768 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004078736208625878, + "loss": 1.4843, + "theoretical_loss": 3.8458467023542493, + "tokens_seen": 634978304 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004078635907723169, + "loss": 1.4797, + "theoretical_loss": 3.845806001586234, + "tokens_seen": 635043840 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040785356068204616, + "loss": 1.4238, + "theoretical_loss": 3.8457653061942296, + "tokens_seen": 635109376 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004078435305917753, + "loss": 1.5101, + "theoretical_loss": 3.8457246161769705, + "tokens_seen": 635174912 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004078335005015045, + "loss": 1.5397, + "theoretical_loss": 3.8456839315331934, + "tokens_seen": 635240448 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004078234704112337, + "loss": 1.41, + "theoretical_loss": 3.8456432522616337, + "tokens_seen": 635305984 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004078134403209629, + "loss": 1.3641, + "theoretical_loss": 3.845602578361029, + "tokens_seen": 635371520 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040780341023069207, + "loss": 1.2852, + "theoretical_loss": 3.8455619098301144, + "tokens_seen": 635437056 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004077933801404213, + "loss": 1.4472, + "theoretical_loss": 3.8455212466676287, + "tokens_seen": 635502592 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040778335005015043, + "loss": 1.4344, + "theoretical_loss": 3.8454805888723094, + "tokens_seen": 635568128 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040777331995987966, + "loss": 1.4525, + "theoretical_loss": 3.845439936442895, + "tokens_seen": 635633664 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 216757, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4686564207077026, + "objective/train/theoretical_loss": 3.8453992893781246, + "objective/train/tokens_used": 635699200, + "theoretical_loss": 3.8453992893781246, + "tokens_seen": 635699200 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004077632898696088, + "loss": 1.4927, + "theoretical_loss": 3.8453992893781246, + "tokens_seen": 635699200 + }, + { + "epoch": 0.19, + "learning_rate": 0.000407753259779338, + "loss": 1.3824, + "theoretical_loss": 3.845358647676736, + "tokens_seen": 635764736 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004077432296890672, + "loss": 1.3867, + "theoretical_loss": 3.8453180113374694, + "tokens_seen": 635830272 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004077331995987964, + "loss": 1.3772, + "theoretical_loss": 3.845277380359065, + "tokens_seen": 635895808 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040772316950852557, + "loss": 1.4655, + "theoretical_loss": 3.845236754740263, + "tokens_seen": 635961344 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040771313941825475, + "loss": 1.41, + "theoretical_loss": 3.8451961344798042, + "tokens_seen": 636026880 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040770310932798393, + "loss": 1.4376, + "theoretical_loss": 3.84515551957643, + "tokens_seen": 636092416 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040769307923771317, + "loss": 1.4222, + "theoretical_loss": 3.845114910028882, + "tokens_seen": 636157952 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004076830491474423, + "loss": 1.3969, + "theoretical_loss": 3.8450743058359023, + "tokens_seen": 636223488 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040767301905717153, + "loss": 1.3186, + "theoretical_loss": 3.845033706996233, + "tokens_seen": 636289024 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004076629889669007, + "loss": 1.3756, + "theoretical_loss": 3.8449931135086177, + "tokens_seen": 636354560 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004076529588766299, + "loss": 1.3837, + "theoretical_loss": 3.8449525253717995, + "tokens_seen": 636420096 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004076429287863591, + "loss": 1.546, + "theoretical_loss": 3.844911942584522, + "tokens_seen": 636485632 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040763289869608825, + "loss": 1.5102, + "theoretical_loss": 3.84487136514553, + "tokens_seen": 636551168 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004076228686058175, + "loss": 1.399, + "theoretical_loss": 3.8448307930535672, + "tokens_seen": 636616704 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040761283851554667, + "loss": 1.3711, + "theoretical_loss": 3.8447902263073797, + "tokens_seen": 636682240 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040760280842527585, + "loss": 1.3877, + "theoretical_loss": 3.8447496649057125, + "tokens_seen": 636747776 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040759277833500503, + "loss": 1.4674, + "theoretical_loss": 3.844709108847311, + "tokens_seen": 636813312 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004075827482447342, + "loss": 1.4608, + "theoretical_loss": 3.8446685581309223, + "tokens_seen": 636878848 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004075727181544634, + "loss": 1.4758, + "theoretical_loss": 3.844628012755293, + "tokens_seen": 636944384 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040756268806419263, + "loss": 1.5474, + "theoretical_loss": 3.8445874727191702, + "tokens_seen": 637009920 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040755265797392176, + "loss": 1.4505, + "theoretical_loss": 3.8445469380213013, + "tokens_seen": 637075456 + }, + { + "epoch": 0.19, + "learning_rate": 0.000407542627883651, + "loss": 1.2944, + "theoretical_loss": 3.844506408660435, + "tokens_seen": 637140992 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004075325977933801, + "loss": 1.413, + "theoretical_loss": 3.844465884635319, + "tokens_seen": 637206528 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040752256770310935, + "loss": 1.4198, + "theoretical_loss": 3.8444253659447023, + "tokens_seen": 637272064 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040751253761283854, + "loss": 1.493, + "theoretical_loss": 3.8443848525873343, + "tokens_seen": 637337600 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004075025075225677, + "loss": 1.326, + "theoretical_loss": 3.844344344561965, + "tokens_seen": 637403136 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004074924774322969, + "loss": 1.3981, + "theoretical_loss": 3.844303841867344, + "tokens_seen": 637468672 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040748244734202613, + "loss": 1.509, + "theoretical_loss": 3.8442633445022225, + "tokens_seen": 637534208 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040747241725175526, + "loss": 1.4283, + "theoretical_loss": 3.844222852465351, + "tokens_seen": 637599744 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004074623871614845, + "loss": 1.4684, + "theoretical_loss": 3.844182365755481, + "tokens_seen": 637665280 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004074523570712136, + "loss": 1.4027, + "theoretical_loss": 3.844141884371364, + "tokens_seen": 637730816 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040744232698094286, + "loss": 1.4434, + "theoretical_loss": 3.8441014083117535, + "tokens_seen": 637796352 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040743229689067204, + "loss": 1.4088, + "theoretical_loss": 3.8440609375754007, + "tokens_seen": 637861888 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004074222668004012, + "loss": 1.4422, + "theoretical_loss": 3.844020472161059, + "tokens_seen": 637927424 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004074122367101304, + "loss": 1.4364, + "theoretical_loss": 3.8439800120674827, + "tokens_seen": 637992960 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004074022066198596, + "loss": 1.4304, + "theoretical_loss": 3.843939557293425, + "tokens_seen": 638058496 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040739217652958876, + "loss": 1.3639, + "theoretical_loss": 3.8438991078376405, + "tokens_seen": 638124032 + }, + { + "epoch": 0.19, + "learning_rate": 0.000407382146439318, + "loss": 1.5098, + "theoretical_loss": 3.843858663698884, + "tokens_seen": 638189568 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004073721163490471, + "loss": 1.3719, + "theoretical_loss": 3.84381822487591, + "tokens_seen": 638255104 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040736208625877636, + "loss": 1.4253, + "theoretical_loss": 3.843777791367475, + "tokens_seen": 638320640 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004073520561685055, + "loss": 1.4325, + "theoretical_loss": 3.843737363172335, + "tokens_seen": 638386176 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004073420260782347, + "loss": 1.4307, + "theoretical_loss": 3.843696940289246, + "tokens_seen": 638451712 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004073319959879639, + "loss": 1.3927, + "theoretical_loss": 3.843656522716965, + "tokens_seen": 638517248 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004073219658976931, + "loss": 1.4992, + "theoretical_loss": 3.843616110454249, + "tokens_seen": 638582784 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040731193580742227, + "loss": 1.473, + "theoretical_loss": 3.843575703499856, + "tokens_seen": 638648320 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004073019057171515, + "loss": 1.517, + "theoretical_loss": 3.843535301852544, + "tokens_seen": 638713856 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040729187562688063, + "loss": 1.3483, + "theoretical_loss": 3.843494905511072, + "tokens_seen": 638779392 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040728184553660986, + "loss": 1.3871, + "theoretical_loss": 3.8434545144741983, + "tokens_seen": 638844928 + }, + { + "epoch": 0.19, + "learning_rate": 0.000407271815446339, + "loss": 1.5852, + "theoretical_loss": 3.8434141287406827, + "tokens_seen": 638910464 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 217691, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.336153507232666, + "objective/train/theoretical_loss": 3.8433737483092845, + "objective/train/tokens_used": 638976000, + "theoretical_loss": 3.8433737483092845, + "tokens_seen": 638976000 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004072617853560682, + "loss": 1.4422, + "theoretical_loss": 3.8433737483092845, + "tokens_seen": 638976000 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004072517552657974, + "loss": 1.4557, + "theoretical_loss": 3.8433333731787647, + "tokens_seen": 639041536 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004072417251755266, + "loss": 1.4386, + "theoretical_loss": 3.843293003347883, + "tokens_seen": 639107072 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040723169508525577, + "loss": 1.4954, + "theoretical_loss": 3.843252638815401, + "tokens_seen": 639172608 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040722166499498495, + "loss": 1.2975, + "theoretical_loss": 3.8432122795800803, + "tokens_seen": 639238144 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040721163490471413, + "loss": 1.3967, + "theoretical_loss": 3.843171925640682, + "tokens_seen": 639303680 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040720160481444337, + "loss": 1.3571, + "theoretical_loss": 3.8431315769959697, + "tokens_seen": 639369216 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004071915747241725, + "loss": 1.5003, + "theoretical_loss": 3.8430912336447047, + "tokens_seen": 639434752 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040718154463390173, + "loss": 1.3441, + "theoretical_loss": 3.8430508955856504, + "tokens_seen": 639500288 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004071715145436309, + "loss": 1.4565, + "theoretical_loss": 3.843010562817571, + "tokens_seen": 639565824 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004071614844533601, + "loss": 1.4967, + "theoretical_loss": 3.84297023533923, + "tokens_seen": 639631360 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040715145436308927, + "loss": 1.4855, + "theoretical_loss": 3.842929913149392, + "tokens_seen": 639696896 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040714142427281845, + "loss": 1.4258, + "theoretical_loss": 3.8428895962468212, + "tokens_seen": 639762432 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040713139418254763, + "loss": 1.371, + "theoretical_loss": 3.842849284630284, + "tokens_seen": 639827968 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040712136409227687, + "loss": 1.417, + "theoretical_loss": 3.8428089782985446, + "tokens_seen": 639893504 + }, + { + "epoch": 0.19, + "learning_rate": 0.000407111334002006, + "loss": 1.5481, + "theoretical_loss": 3.84276867725037, + "tokens_seen": 639959040 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040710130391173523, + "loss": 1.5653, + "theoretical_loss": 3.842728381484526, + "tokens_seen": 640024576 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040709127382146436, + "loss": 1.5096, + "theoretical_loss": 3.8426880909997796, + "tokens_seen": 640090112 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004070812437311936, + "loss": 1.4221, + "theoretical_loss": 3.8426478057948983, + "tokens_seen": 640155648 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004070712136409228, + "loss": 1.3733, + "theoretical_loss": 3.8426075258686496, + "tokens_seen": 640221184 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040706118355065196, + "loss": 1.4537, + "theoretical_loss": 3.842567251219802, + "tokens_seen": 640286720 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040705115346038114, + "loss": 1.406, + "theoretical_loss": 3.8425269818471235, + "tokens_seen": 640352256 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004070411233701103, + "loss": 1.4432, + "theoretical_loss": 3.8424867177493836, + "tokens_seen": 640417792 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004070310932798395, + "loss": 1.3701, + "theoretical_loss": 3.8424464589253504, + "tokens_seen": 640483328 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040702106318956874, + "loss": 1.4254, + "theoretical_loss": 3.842406205373795, + "tokens_seen": 640548864 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040701103309929786, + "loss": 1.4327, + "theoretical_loss": 3.8423659570934867, + "tokens_seen": 640614400 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004070010030090271, + "loss": 1.3939, + "theoretical_loss": 3.842325714083197, + "tokens_seen": 640679936 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004069909729187563, + "loss": 1.2979, + "theoretical_loss": 3.842285476341696, + "tokens_seen": 640745472 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040698094282848546, + "loss": 1.3466, + "theoretical_loss": 3.842245243867755, + "tokens_seen": 640811008 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040697091273821464, + "loss": 1.4087, + "theoretical_loss": 3.842205016660147, + "tokens_seen": 640876544 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004069608826479438, + "loss": 1.4083, + "theoretical_loss": 3.8421647947176423, + "tokens_seen": 640942080 + }, + { + "epoch": 0.19, + "learning_rate": 0.000406950852557673, + "loss": 1.358, + "theoretical_loss": 3.842124578039016, + "tokens_seen": 641007616 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040694082246740224, + "loss": 1.3822, + "theoretical_loss": 3.842084366623039, + "tokens_seen": 641073152 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040693079237713137, + "loss": 1.4161, + "theoretical_loss": 3.842044160468485, + "tokens_seen": 641138688 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004069207622868606, + "loss": 1.3518, + "theoretical_loss": 3.8420039595741295, + "tokens_seen": 641204224 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040691073219658973, + "loss": 1.4138, + "theoretical_loss": 3.841963763938746, + "tokens_seen": 641269760 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040690070210631896, + "loss": 1.4114, + "theoretical_loss": 3.8419235735611075, + "tokens_seen": 641335296 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004068906720160482, + "loss": 1.4194, + "theoretical_loss": 3.8418833884399914, + "tokens_seen": 641400832 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004068806419257773, + "loss": 1.4654, + "theoretical_loss": 3.841843208574172, + "tokens_seen": 641466368 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040687061183550656, + "loss": 1.4464, + "theoretical_loss": 3.8418030339624254, + "tokens_seen": 641531904 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004068605817452357, + "loss": 1.4171, + "theoretical_loss": 3.8417628646035284, + "tokens_seen": 641597440 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004068505516549649, + "loss": 1.5319, + "theoretical_loss": 3.8417227004962573, + "tokens_seen": 641662976 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004068405215646941, + "loss": 1.458, + "theoretical_loss": 3.8416825416393894, + "tokens_seen": 641728512 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004068304914744233, + "loss": 1.3326, + "theoretical_loss": 3.841642388031702, + "tokens_seen": 641794048 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040682046138415247, + "loss": 1.38, + "theoretical_loss": 3.8416022396719733, + "tokens_seen": 641859584 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004068104312938817, + "loss": 1.313, + "theoretical_loss": 3.841562096558982, + "tokens_seen": 641925120 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040680040120361083, + "loss": 1.4038, + "theoretical_loss": 3.841521958691506, + "tokens_seen": 641990656 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040679037111334006, + "loss": 1.4686, + "theoretical_loss": 3.841481826068325, + "tokens_seen": 642056192 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004067803410230692, + "loss": 1.3984, + "theoretical_loss": 3.841441698688219, + "tokens_seen": 642121728 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004067703109327984, + "loss": 1.3675, + "theoretical_loss": 3.8414015765499676, + "tokens_seen": 642187264 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 218808, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.392930030822754, + "objective/train/theoretical_loss": 3.841361459652351, + "objective/train/tokens_used": 642252800, + "theoretical_loss": 3.841361459652351, + "tokens_seen": 642252800 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004067602808425276, + "loss": 1.4346, + "theoretical_loss": 3.841361459652351, + "tokens_seen": 642252800 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004067502507522568, + "loss": 1.4238, + "theoretical_loss": 3.8413213479941506, + "tokens_seen": 642318336 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040674022066198597, + "loss": 1.4343, + "theoretical_loss": 3.841281241574147, + "tokens_seen": 642383872 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040673019057171515, + "loss": 1.5135, + "theoretical_loss": 3.841241140391123, + "tokens_seen": 642449408 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040672016048144433, + "loss": 1.3744, + "theoretical_loss": 3.8412010444438587, + "tokens_seen": 642514944 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040671013039117357, + "loss": 1.5316, + "theoretical_loss": 3.8411609537311384, + "tokens_seen": 642580480 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004067001003009027, + "loss": 1.4192, + "theoretical_loss": 3.841120868251744, + "tokens_seen": 642646016 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040669007021063193, + "loss": 1.4151, + "theoretical_loss": 3.8410807880044593, + "tokens_seen": 642711552 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004066800401203611, + "loss": 1.3338, + "theoretical_loss": 3.8410407129880673, + "tokens_seen": 642777088 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004066700100300903, + "loss": 1.4559, + "theoretical_loss": 3.841000643201353, + "tokens_seen": 642842624 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040665997993981947, + "loss": 1.5541, + "theoretical_loss": 3.8409605786431, + "tokens_seen": 642908160 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040664994984954865, + "loss": 1.3579, + "theoretical_loss": 3.840920519312094, + "tokens_seen": 642973696 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040663991975927783, + "loss": 1.5204, + "theoretical_loss": 3.8408804652071193, + "tokens_seen": 643039232 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040662988966900707, + "loss": 1.4303, + "theoretical_loss": 3.8408404163269627, + "tokens_seen": 643104768 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004066198595787362, + "loss": 1.3406, + "theoretical_loss": 3.84080037267041, + "tokens_seen": 643170304 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040660982948846543, + "loss": 1.5067, + "theoretical_loss": 3.840760334236247, + "tokens_seen": 643235840 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040659979939819456, + "loss": 1.3916, + "theoretical_loss": 3.8407203010232616, + "tokens_seen": 643301376 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004065897693079238, + "loss": 1.4849, + "theoretical_loss": 3.840680273030241, + "tokens_seen": 643366912 + }, + { + "epoch": 0.19, + "learning_rate": 0.000406579739217653, + "loss": 1.4553, + "theoretical_loss": 3.840640250255972, + "tokens_seen": 643432448 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040656970912738216, + "loss": 1.3765, + "theoretical_loss": 3.840600232699244, + "tokens_seen": 643497984 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040655967903711134, + "loss": 1.3931, + "theoretical_loss": 3.8405602203588445, + "tokens_seen": 643563520 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004065496489468405, + "loss": 1.4241, + "theoretical_loss": 3.8405202132335634, + "tokens_seen": 643629056 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004065396188565697, + "loss": 1.3288, + "theoretical_loss": 3.8404802113221894, + "tokens_seen": 643694592 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040652958876629894, + "loss": 1.4555, + "theoretical_loss": 3.8404402146235124, + "tokens_seen": 643760128 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040651955867602806, + "loss": 1.3722, + "theoretical_loss": 3.8404002231363226, + "tokens_seen": 643825664 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004065095285857573, + "loss": 1.4145, + "theoretical_loss": 3.8403602368594107, + "tokens_seen": 643891200 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004064994984954865, + "loss": 1.422, + "theoretical_loss": 3.8403202557915677, + "tokens_seen": 643956736 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040648946840521566, + "loss": 1.3824, + "theoretical_loss": 3.8402802799315845, + "tokens_seen": 644022272 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040647943831494484, + "loss": 1.4774, + "theoretical_loss": 3.8402403092782533, + "tokens_seen": 644087808 + }, + { + "epoch": 0.2, + "learning_rate": 0.000406469408224674, + "loss": 1.4584, + "theoretical_loss": 3.8402003438303662, + "tokens_seen": 644153344 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004064593781344032, + "loss": 1.6687, + "theoretical_loss": 3.840160383586716, + "tokens_seen": 644218880 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040644934804413244, + "loss": 1.3421, + "theoretical_loss": 3.8401204285460957, + "tokens_seen": 644284416 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040643931795386157, + "loss": 1.4168, + "theoretical_loss": 3.840080478707298, + "tokens_seen": 644349952 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004064292878635908, + "loss": 1.5777, + "theoretical_loss": 3.840040534069118, + "tokens_seen": 644415488 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040641925777331993, + "loss": 1.3855, + "theoretical_loss": 3.8400005946303484, + "tokens_seen": 644481024 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040640922768304916, + "loss": 1.4382, + "theoretical_loss": 3.8399606603897847, + "tokens_seen": 644546560 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040639919759277834, + "loss": 1.4766, + "theoretical_loss": 3.8399207313462216, + "tokens_seen": 644612096 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004063891675025075, + "loss": 1.414, + "theoretical_loss": 3.8398808074984547, + "tokens_seen": 644677632 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004063791374122367, + "loss": 1.326, + "theoretical_loss": 3.8398408888452797, + "tokens_seen": 644743168 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004063691073219659, + "loss": 1.4206, + "theoretical_loss": 3.8398009753854927, + "tokens_seen": 644808704 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040635907723169507, + "loss": 1.4731, + "theoretical_loss": 3.839761067117891, + "tokens_seen": 644874240 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004063490471414243, + "loss": 1.4438, + "theoretical_loss": 3.8397211640412703, + "tokens_seen": 644939776 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040633901705115343, + "loss": 1.3863, + "theoretical_loss": 3.839681266154429, + "tokens_seen": 645005312 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040632898696088267, + "loss": 1.4235, + "theoretical_loss": 3.839641373456165, + "tokens_seen": 645070848 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040631895687061185, + "loss": 1.5068, + "theoretical_loss": 3.8396014859452756, + "tokens_seen": 645136384 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040630892678034103, + "loss": 1.4347, + "theoretical_loss": 3.8395616036205604, + "tokens_seen": 645201920 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004062988966900702, + "loss": 1.3856, + "theoretical_loss": 3.839521726480818, + "tokens_seen": 645267456 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004062888665997994, + "loss": 1.3279, + "theoretical_loss": 3.8394818545248475, + "tokens_seen": 645332992 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040627883650952857, + "loss": 1.4612, + "theoretical_loss": 3.8394419877514494, + "tokens_seen": 645398528 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004062688064192578, + "loss": 1.3907, + "theoretical_loss": 3.8394021261594236, + "tokens_seen": 645464064 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 220064, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2961300611495972, + "objective/train/theoretical_loss": 3.8393622697475704, + "objective/train/tokens_used": 645529600, + "theoretical_loss": 3.8393622697475704, + "tokens_seen": 645529600 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040625877632898693, + "loss": 1.4614, + "theoretical_loss": 3.8393622697475704, + "tokens_seen": 645529600 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040624874623871617, + "loss": 1.4412, + "theoretical_loss": 3.839322418514691, + "tokens_seen": 645595136 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004062387161484453, + "loss": 1.3176, + "theoretical_loss": 3.839282572459587, + "tokens_seen": 645660672 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040622868605817453, + "loss": 1.3, + "theoretical_loss": 3.83924273158106, + "tokens_seen": 645726208 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004062186559679037, + "loss": 1.3497, + "theoretical_loss": 3.8392028958779125, + "tokens_seen": 645791744 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004062086258776329, + "loss": 1.4149, + "theoretical_loss": 3.8391630653489472, + "tokens_seen": 645857280 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004061985957873621, + "loss": 1.4477, + "theoretical_loss": 3.8391232399929667, + "tokens_seen": 645922816 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040618856569709126, + "loss": 1.3832, + "theoretical_loss": 3.839083419808774, + "tokens_seen": 645988352 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040617853560682044, + "loss": 1.4434, + "theoretical_loss": 3.839043604795174, + "tokens_seen": 646053888 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040616850551654967, + "loss": 1.3095, + "theoretical_loss": 3.83900379495097, + "tokens_seen": 646119424 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004061584754262788, + "loss": 1.4176, + "theoretical_loss": 3.8389639902749675, + "tokens_seen": 646184960 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040614844533600804, + "loss": 1.3957, + "theoretical_loss": 3.83892419076597, + "tokens_seen": 646250496 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040613841524573727, + "loss": 1.387, + "theoretical_loss": 3.8388843964227846, + "tokens_seen": 646316032 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004061283851554664, + "loss": 1.4462, + "theoretical_loss": 3.8388446072442157, + "tokens_seen": 646381568 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040611835506519563, + "loss": 1.4525, + "theoretical_loss": 3.8388048232290704, + "tokens_seen": 646447104 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040610832497492476, + "loss": 1.366, + "theoretical_loss": 3.8387650443761556, + "tokens_seen": 646512640 + }, + { + "epoch": 0.2, + "learning_rate": 0.000406098294884654, + "loss": 1.4238, + "theoretical_loss": 3.838725270684277, + "tokens_seen": 646578176 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004060882647943832, + "loss": 1.2541, + "theoretical_loss": 3.8386855021522424, + "tokens_seen": 646643712 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040607823470411236, + "loss": 1.3519, + "theoretical_loss": 3.83864573877886, + "tokens_seen": 646709248 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040606820461384154, + "loss": 1.5823, + "theoretical_loss": 3.8386059805629382, + "tokens_seen": 646774784 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004060581745235707, + "loss": 1.5113, + "theoretical_loss": 3.838566227503285, + "tokens_seen": 646840320 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004060481444332999, + "loss": 1.384, + "theoretical_loss": 3.838526479598709, + "tokens_seen": 646905856 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040603811434302914, + "loss": 1.3717, + "theoretical_loss": 3.838486736848021, + "tokens_seen": 646971392 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040602808425275826, + "loss": 1.4058, + "theoretical_loss": 3.8384469992500296, + "tokens_seen": 647036928 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004060180541624875, + "loss": 1.4911, + "theoretical_loss": 3.8384072668035447, + "tokens_seen": 647102464 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004060080240722167, + "loss": 1.4352, + "theoretical_loss": 3.8383675395073777, + "tokens_seen": 647168000 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040599799398194586, + "loss": 1.4323, + "theoretical_loss": 3.838327817360339, + "tokens_seen": 647233536 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040598796389167504, + "loss": 1.4282, + "theoretical_loss": 3.8382881003612406, + "tokens_seen": 647299072 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004059779338014042, + "loss": 1.3128, + "theoretical_loss": 3.8382483885088927, + "tokens_seen": 647364608 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004059679037111334, + "loss": 1.397, + "theoretical_loss": 3.8382086818021097, + "tokens_seen": 647430144 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040595787362086264, + "loss": 1.4077, + "theoretical_loss": 3.838168980239702, + "tokens_seen": 647495680 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040594784353059177, + "loss": 1.477, + "theoretical_loss": 3.838129283820484, + "tokens_seen": 647561216 + }, + { + "epoch": 0.2, + "learning_rate": 0.000405937813440321, + "loss": 1.3418, + "theoretical_loss": 3.838089592543268, + "tokens_seen": 647626752 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040592778335005013, + "loss": 1.4784, + "theoretical_loss": 3.8380499064068685, + "tokens_seen": 647692288 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040591775325977936, + "loss": 1.3705, + "theoretical_loss": 3.8380102254100983, + "tokens_seen": 647757824 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040590772316950854, + "loss": 1.4103, + "theoretical_loss": 3.8379705495517733, + "tokens_seen": 647823360 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004058976930792377, + "loss": 1.4364, + "theoretical_loss": 3.8379308788307083, + "tokens_seen": 647888896 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004058876629889669, + "loss": 1.454, + "theoretical_loss": 3.837891213245718, + "tokens_seen": 647954432 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004058776328986961, + "loss": 1.3968, + "theoretical_loss": 3.837851552795618, + "tokens_seen": 648019968 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040586760280842527, + "loss": 1.4309, + "theoretical_loss": 3.8378118974792246, + "tokens_seen": 648085504 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004058575727181545, + "loss": 1.4649, + "theoretical_loss": 3.837772247295354, + "tokens_seen": 648151040 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040584754262788363, + "loss": 1.4905, + "theoretical_loss": 3.8377326022428235, + "tokens_seen": 648216576 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040583751253761287, + "loss": 1.4279, + "theoretical_loss": 3.8376929623204505, + "tokens_seen": 648282112 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040582748244734205, + "loss": 1.356, + "theoretical_loss": 3.8376533275270517, + "tokens_seen": 648347648 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040581745235707123, + "loss": 1.3295, + "theoretical_loss": 3.837613697861446, + "tokens_seen": 648413184 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004058074222668004, + "loss": 1.4041, + "theoretical_loss": 3.8375740733224513, + "tokens_seen": 648478720 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004057973921765296, + "loss": 1.3485, + "theoretical_loss": 3.8375344539088867, + "tokens_seen": 648544256 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040578736208625877, + "loss": 1.4249, + "theoretical_loss": 3.837494839619571, + "tokens_seen": 648609792 + }, + { + "epoch": 0.2, + "learning_rate": 0.000405777331995988, + "loss": 1.3583, + "theoretical_loss": 3.8374552304533243, + "tokens_seen": 648675328 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040576730190571713, + "loss": 1.404, + "theoretical_loss": 3.8374156264089665, + "tokens_seen": 648740864 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 220996, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4512208700180054, + "objective/train/theoretical_loss": 3.8373760274853175, + "objective/train/tokens_used": 648806400, + "theoretical_loss": 3.8373760274853175, + "tokens_seen": 648806400 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040575727181544637, + "loss": 1.4145, + "theoretical_loss": 3.8373760274853175, + "tokens_seen": 648806400 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004057472417251755, + "loss": 1.4681, + "theoretical_loss": 3.8373364336811986, + "tokens_seen": 648871936 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040573721163490473, + "loss": 1.4318, + "theoretical_loss": 3.8372968449954308, + "tokens_seen": 648937472 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004057271815446339, + "loss": 1.4426, + "theoretical_loss": 3.8372572614268354, + "tokens_seen": 649003008 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004057171514543631, + "loss": 1.4771, + "theoretical_loss": 3.8372176829742344, + "tokens_seen": 649068544 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004057071213640923, + "loss": 1.3883, + "theoretical_loss": 3.837178109636451, + "tokens_seen": 649134080 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040569709127382146, + "loss": 1.378, + "theoretical_loss": 3.837138541412306, + "tokens_seen": 649199616 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040568706118355064, + "loss": 1.3592, + "theoretical_loss": 3.837098978300624, + "tokens_seen": 649265152 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004056770310932799, + "loss": 1.3661, + "theoretical_loss": 3.8370594203002284, + "tokens_seen": 649330688 + }, + { + "epoch": 0.2, + "learning_rate": 0.000405667001003009, + "loss": 1.4421, + "theoretical_loss": 3.8370198674099427, + "tokens_seen": 649396224 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040565697091273824, + "loss": 1.3925, + "theoretical_loss": 3.8369803196285917, + "tokens_seen": 649461760 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004056469408224674, + "loss": 1.4364, + "theoretical_loss": 3.8369407769549992, + "tokens_seen": 649527296 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004056369107321966, + "loss": 1.3817, + "theoretical_loss": 3.8369012393879904, + "tokens_seen": 649592832 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004056268806419258, + "loss": 1.4452, + "theoretical_loss": 3.836861706926392, + "tokens_seen": 649658368 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040561685055165496, + "loss": 1.4073, + "theoretical_loss": 3.836822179569028, + "tokens_seen": 649723904 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040560682046138414, + "loss": 1.5185, + "theoretical_loss": 3.8367826573147257, + "tokens_seen": 649789440 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004055967903711134, + "loss": 1.4525, + "theoretical_loss": 3.836743140162312, + "tokens_seen": 649854976 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004055867602808425, + "loss": 1.4945, + "theoretical_loss": 3.8367036281106124, + "tokens_seen": 649920512 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040557673019057174, + "loss": 1.4592, + "theoretical_loss": 3.836664121158456, + "tokens_seen": 649986048 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040556670010030087, + "loss": 1.4421, + "theoretical_loss": 3.83662461930467, + "tokens_seen": 650051584 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004055566700100301, + "loss": 1.3949, + "theoretical_loss": 3.836585122548082, + "tokens_seen": 650117120 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004055466399197593, + "loss": 1.3662, + "theoretical_loss": 3.836545630887521, + "tokens_seen": 650182656 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040553660982948846, + "loss": 1.3332, + "theoretical_loss": 3.836506144321816, + "tokens_seen": 650248192 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040552657973921764, + "loss": 1.3474, + "theoretical_loss": 3.836466662849795, + "tokens_seen": 650313728 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004055165496489469, + "loss": 1.3222, + "theoretical_loss": 3.8364271864702904, + "tokens_seen": 650379264 + }, + { + "epoch": 0.2, + "learning_rate": 0.000405506519558676, + "loss": 1.407, + "theoretical_loss": 3.8363877151821297, + "tokens_seen": 650444800 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040549648946840524, + "loss": 1.3544, + "theoretical_loss": 3.8363482489841454, + "tokens_seen": 650510336 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040548645937813437, + "loss": 1.4223, + "theoretical_loss": 3.8363087878751667, + "tokens_seen": 650575872 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004054764292878636, + "loss": 1.4228, + "theoretical_loss": 3.8362693318540257, + "tokens_seen": 650641408 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004054663991975928, + "loss": 1.4062, + "theoretical_loss": 3.8362298809195545, + "tokens_seen": 650706944 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040545636910732197, + "loss": 1.4631, + "theoretical_loss": 3.8361904350705833, + "tokens_seen": 650772480 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040544633901705115, + "loss": 1.4519, + "theoretical_loss": 3.836150994305947, + "tokens_seen": 650838016 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040543630892678033, + "loss": 1.3164, + "theoretical_loss": 3.8361115586244763, + "tokens_seen": 650903552 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004054262788365095, + "loss": 1.4631, + "theoretical_loss": 3.8360721280250054, + "tokens_seen": 650969088 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040541624874623874, + "loss": 1.3611, + "theoretical_loss": 3.836032702506367, + "tokens_seen": 651034624 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040540621865596787, + "loss": 1.4917, + "theoretical_loss": 3.8359932820673968, + "tokens_seen": 651100160 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004053961885656971, + "loss": 1.4065, + "theoretical_loss": 3.8359538667069275, + "tokens_seen": 651165696 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004053861584754263, + "loss": 1.5337, + "theoretical_loss": 3.8359144564237946, + "tokens_seen": 651231232 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040537612838515547, + "loss": 1.4671, + "theoretical_loss": 3.8358750512168323, + "tokens_seen": 651296768 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004053660982948847, + "loss": 1.3983, + "theoretical_loss": 3.8358356510848775, + "tokens_seen": 651362304 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040535606820461383, + "loss": 1.4674, + "theoretical_loss": 3.8357962560267644, + "tokens_seen": 651427840 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040534603811434307, + "loss": 1.4927, + "theoretical_loss": 3.835756866041331, + "tokens_seen": 651493376 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040533600802407225, + "loss": 1.5006, + "theoretical_loss": 3.8357174811274133, + "tokens_seen": 651558912 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040532597793380143, + "loss": 1.3942, + "theoretical_loss": 3.8356781012838472, + "tokens_seen": 651624448 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004053159478435306, + "loss": 1.4251, + "theoretical_loss": 3.8356387265094716, + "tokens_seen": 651689984 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004053059177532598, + "loss": 1.3492, + "theoretical_loss": 3.835599356803124, + "tokens_seen": 651755520 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040529588766298897, + "loss": 1.4275, + "theoretical_loss": 3.835559992163642, + "tokens_seen": 651821056 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004052858575727182, + "loss": 1.3968, + "theoretical_loss": 3.835520632589864, + "tokens_seen": 651886592 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040527582748244733, + "loss": 1.3795, + "theoretical_loss": 3.83548127808063, + "tokens_seen": 651952128 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040526579739217657, + "loss": 1.437, + "theoretical_loss": 3.8354419286347783, + "tokens_seen": 652017664 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 222206, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2518059015274048, + "objective/train/theoretical_loss": 3.8354025842511494, + "objective/train/tokens_used": 652083200, + "theoretical_loss": 3.8354025842511494, + "tokens_seen": 652083200 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004052557673019057, + "loss": 1.3611, + "theoretical_loss": 3.8354025842511494, + "tokens_seen": 652083200 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040524573721163493, + "loss": 1.5138, + "theoretical_loss": 3.8353632449285833, + "tokens_seen": 652148736 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004052357071213641, + "loss": 1.3822, + "theoretical_loss": 3.8353239106659194, + "tokens_seen": 652214272 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004052256770310933, + "loss": 1.3124, + "theoretical_loss": 3.835284581462, + "tokens_seen": 652279808 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004052156469408225, + "loss": 1.3552, + "theoretical_loss": 3.8352452573156652, + "tokens_seen": 652345344 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040520561685055166, + "loss": 1.4888, + "theoretical_loss": 3.835205938225757, + "tokens_seen": 652410880 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040519558676028084, + "loss": 1.4866, + "theoretical_loss": 3.8351666241911175, + "tokens_seen": 652476416 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004051855566700101, + "loss": 1.5466, + "theoretical_loss": 3.8351273152105896, + "tokens_seen": 652541952 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004051755265797392, + "loss": 1.392, + "theoretical_loss": 3.8350880112830152, + "tokens_seen": 652607488 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040516549648946844, + "loss": 1.4745, + "theoretical_loss": 3.8350487124072377, + "tokens_seen": 652673024 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004051554663991976, + "loss": 1.4358, + "theoretical_loss": 3.835009418582101, + "tokens_seen": 652738560 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004051454363089268, + "loss": 1.3864, + "theoretical_loss": 3.8349701298064485, + "tokens_seen": 652804096 + }, + { + "epoch": 0.2, + "learning_rate": 0.000405135406218656, + "loss": 1.4699, + "theoretical_loss": 3.834930846079125, + "tokens_seen": 652869632 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040512537612838516, + "loss": 1.3652, + "theoretical_loss": 3.8348915673989743, + "tokens_seen": 652935168 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040511534603811434, + "loss": 1.4381, + "theoretical_loss": 3.8348522937648424, + "tokens_seen": 653000704 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004051053159478436, + "loss": 1.4359, + "theoretical_loss": 3.834813025175574, + "tokens_seen": 653066240 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004050952858575727, + "loss": 1.4865, + "theoretical_loss": 3.8347737616300153, + "tokens_seen": 653131776 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040508525576730194, + "loss": 1.381, + "theoretical_loss": 3.8347345031270126, + "tokens_seen": 653197312 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040507522567703107, + "loss": 1.4205, + "theoretical_loss": 3.834695249665412, + "tokens_seen": 653262848 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004050651955867603, + "loss": 1.3862, + "theoretical_loss": 3.8346560012440607, + "tokens_seen": 653328384 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004050551654964895, + "loss": 1.4601, + "theoretical_loss": 3.8346167578618062, + "tokens_seen": 653393920 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040504513540621866, + "loss": 1.4596, + "theoretical_loss": 3.834577519517496, + "tokens_seen": 653459456 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040503510531594784, + "loss": 1.4097, + "theoretical_loss": 3.8345382862099786, + "tokens_seen": 653524992 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004050250752256771, + "loss": 1.4786, + "theoretical_loss": 3.8344990579381015, + "tokens_seen": 653590528 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004050150451354062, + "loss": 1.4857, + "theoretical_loss": 3.8344598347007146, + "tokens_seen": 653656064 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040500501504513544, + "loss": 1.3851, + "theoretical_loss": 3.834420616496666, + "tokens_seen": 653721600 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040499498495486457, + "loss": 1.393, + "theoretical_loss": 3.8343814033248065, + "tokens_seen": 653787136 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004049849548645938, + "loss": 1.4233, + "theoretical_loss": 3.8343421951839853, + "tokens_seen": 653852672 + }, + { + "epoch": 0.2, + "learning_rate": 0.000404974924774323, + "loss": 1.4263, + "theoretical_loss": 3.834302992073053, + "tokens_seen": 653918208 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040496489468405217, + "loss": 1.3151, + "theoretical_loss": 3.8342637939908606, + "tokens_seen": 653983744 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040495486459378135, + "loss": 1.3663, + "theoretical_loss": 3.8342246009362584, + "tokens_seen": 654049280 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040494483450351053, + "loss": 1.5411, + "theoretical_loss": 3.8341854129080986, + "tokens_seen": 654114816 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004049348044132397, + "loss": 1.3252, + "theoretical_loss": 3.8341462299052327, + "tokens_seen": 654180352 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040492477432296894, + "loss": 1.3995, + "theoretical_loss": 3.8341070519265132, + "tokens_seen": 654245888 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040491474423269807, + "loss": 1.4336, + "theoretical_loss": 3.8340678789707927, + "tokens_seen": 654311424 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004049047141424273, + "loss": 1.4057, + "theoretical_loss": 3.834028711036924, + "tokens_seen": 654376960 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040489468405215643, + "loss": 1.4166, + "theoretical_loss": 3.8339895481237605, + "tokens_seen": 654442496 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040488465396188567, + "loss": 1.3367, + "theoretical_loss": 3.8339503902301564, + "tokens_seen": 654508032 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040487462387161485, + "loss": 1.4379, + "theoretical_loss": 3.833911237354965, + "tokens_seen": 654573568 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040486459378134403, + "loss": 1.4507, + "theoretical_loss": 3.8338720894970413, + "tokens_seen": 654639104 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004048545636910732, + "loss": 1.5019, + "theoretical_loss": 3.8338329466552405, + "tokens_seen": 654704640 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040484453360080245, + "loss": 1.539, + "theoretical_loss": 3.833793808828417, + "tokens_seen": 654770176 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004048345035105316, + "loss": 1.4428, + "theoretical_loss": 3.8337546760154275, + "tokens_seen": 654835712 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004048244734202608, + "loss": 1.4632, + "theoretical_loss": 3.8337155482151264, + "tokens_seen": 654901248 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040481444332998994, + "loss": 1.3823, + "theoretical_loss": 3.833676425426372, + "tokens_seen": 654966784 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040480441323971917, + "loss": 1.3704, + "theoretical_loss": 3.8336373076480204, + "tokens_seen": 655032320 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040479438314944835, + "loss": 1.4362, + "theoretical_loss": 3.8335981948789275, + "tokens_seen": 655097856 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040478435305917753, + "loss": 1.3661, + "theoretical_loss": 3.833559087117952, + "tokens_seen": 655163392 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004047743229689067, + "loss": 1.4489, + "theoretical_loss": 3.833519984363952, + "tokens_seen": 655228928 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004047642928786359, + "loss": 1.4523, + "theoretical_loss": 3.833480886615785, + "tokens_seen": 655294464 + }, + { + "debugging/length": 589.0, + "debugging/num_segments": 9, + "debugging/score": 0.06902207512072801, + "debugging/score_std": 0.023551935444711312, + "epoch": 0.2, + "objective/train/docs_used": 223092, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3888072967529297, + "objective/train/theoretical_loss": 3.83344179387231, + "objective/train/tokens_used": 655360000, + "theoretical_loss": 3.83344179387231, + "tokens_seen": 655360000 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004047542627883651, + "loss": 1.4582, + "theoretical_loss": 3.83344179387231, + "tokens_seen": 655360000 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004047442326980943, + "loss": 1.3585, + "theoretical_loss": 3.8334027061323863, + "tokens_seen": 655425536 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040473420260782344, + "loss": 1.4897, + "theoretical_loss": 3.8333636233948725, + "tokens_seen": 655491072 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004047241725175527, + "loss": 1.3875, + "theoretical_loss": 3.8333245456586287, + "tokens_seen": 655556608 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004047141424272818, + "loss": 1.4661, + "theoretical_loss": 3.8332854729225154, + "tokens_seen": 655622144 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040470411233701104, + "loss": 1.4245, + "theoretical_loss": 3.8332464051853927, + "tokens_seen": 655687680 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004046940822467402, + "loss": 1.5863, + "theoretical_loss": 3.8332073424461215, + "tokens_seen": 655753216 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004046840521564694, + "loss": 1.5076, + "theoretical_loss": 3.8331682847035635, + "tokens_seen": 655818752 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004046740220661986, + "loss": 1.4398, + "theoretical_loss": 3.8331292319565793, + "tokens_seen": 655884288 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004046639919759278, + "loss": 1.43, + "theoretical_loss": 3.833090184204032, + "tokens_seen": 655949824 + }, + { + "epoch": 0.2, + "learning_rate": 0.000404653961885657, + "loss": 1.4785, + "theoretical_loss": 3.833051141444783, + "tokens_seen": 656015360 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004046439317953862, + "loss": 1.4539, + "theoretical_loss": 3.833012103677696, + "tokens_seen": 656080896 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040463390170511536, + "loss": 1.3212, + "theoretical_loss": 3.8329730709016334, + "tokens_seen": 656146432 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040462387161484454, + "loss": 1.5135, + "theoretical_loss": 3.832934043115459, + "tokens_seen": 656211968 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004046138415245738, + "loss": 1.434, + "theoretical_loss": 3.8328950203180367, + "tokens_seen": 656277504 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004046038114343029, + "loss": 1.4062, + "theoretical_loss": 3.8328560025082306, + "tokens_seen": 656343040 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040459378134403214, + "loss": 1.4098, + "theoretical_loss": 3.832816989684905, + "tokens_seen": 656408576 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040458375125376127, + "loss": 1.394, + "theoretical_loss": 3.8327779818469256, + "tokens_seen": 656474112 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004045737211634905, + "loss": 1.2875, + "theoretical_loss": 3.8327389789931567, + "tokens_seen": 656539648 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004045636910732197, + "loss": 1.4201, + "theoretical_loss": 3.8326999811224653, + "tokens_seen": 656605184 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040455366098294886, + "loss": 1.4018, + "theoretical_loss": 3.8326609882337164, + "tokens_seen": 656670720 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040454363089267804, + "loss": 1.4676, + "theoretical_loss": 3.8326220003257774, + "tokens_seen": 656736256 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004045336008024073, + "loss": 1.3642, + "theoretical_loss": 3.832583017397514, + "tokens_seen": 656801792 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004045235707121364, + "loss": 1.3769, + "theoretical_loss": 3.8325440394477943, + "tokens_seen": 656867328 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040451354062186564, + "loss": 1.5141, + "theoretical_loss": 3.8325050664754854, + "tokens_seen": 656932864 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040450351053159477, + "loss": 1.4847, + "theoretical_loss": 3.8324660984794554, + "tokens_seen": 656998400 + }, + { + "epoch": 0.2, + "learning_rate": 0.000404493480441324, + "loss": 1.4246, + "theoretical_loss": 3.832427135458573, + "tokens_seen": 657063936 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004044834503510532, + "loss": 1.2796, + "theoretical_loss": 3.832388177411706, + "tokens_seen": 657129472 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040447342026078237, + "loss": 1.483, + "theoretical_loss": 3.832349224337724, + "tokens_seen": 657195008 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040446339017051155, + "loss": 1.4562, + "theoretical_loss": 3.832310276235497, + "tokens_seen": 657260544 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040445336008024073, + "loss": 1.5676, + "theoretical_loss": 3.8322713331038933, + "tokens_seen": 657326080 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004044433299899699, + "loss": 1.2765, + "theoretical_loss": 3.8322323949417845, + "tokens_seen": 657391616 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040443329989969915, + "loss": 1.3575, + "theoretical_loss": 3.8321934617480404, + "tokens_seen": 657457152 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040442326980942827, + "loss": 1.5309, + "theoretical_loss": 3.832154533521532, + "tokens_seen": 657522688 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004044132397191575, + "loss": 1.4745, + "theoretical_loss": 3.8321156102611305, + "tokens_seen": 657588224 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040440320962888663, + "loss": 1.3756, + "theoretical_loss": 3.832076691965708, + "tokens_seen": 657653760 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040439317953861587, + "loss": 1.4161, + "theoretical_loss": 3.832037778634136, + "tokens_seen": 657719296 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040438314944834505, + "loss": 1.4554, + "theoretical_loss": 3.831998870265287, + "tokens_seen": 657784832 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040437311935807423, + "loss": 1.4204, + "theoretical_loss": 3.831959966858034, + "tokens_seen": 657850368 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004043630892678034, + "loss": 1.3304, + "theoretical_loss": 3.8319210684112495, + "tokens_seen": 657915904 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040435305917753265, + "loss": 1.399, + "theoretical_loss": 3.8318821749238077, + "tokens_seen": 657981440 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004043430290872618, + "loss": 1.2855, + "theoretical_loss": 3.831843286394582, + "tokens_seen": 658046976 + }, + { + "epoch": 0.2, + "learning_rate": 0.000404332998996991, + "loss": 1.495, + "theoretical_loss": 3.8318044028224465, + "tokens_seen": 658112512 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040432296890672014, + "loss": 1.374, + "theoretical_loss": 3.831765524206276, + "tokens_seen": 658178048 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040431293881644937, + "loss": 1.5157, + "theoretical_loss": 3.8317266505449457, + "tokens_seen": 658243584 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040430290872617855, + "loss": 1.4343, + "theoretical_loss": 3.8316877818373305, + "tokens_seen": 658309120 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040429287863590773, + "loss": 1.4368, + "theoretical_loss": 3.8316489180823066, + "tokens_seen": 658374656 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004042828485456369, + "loss": 1.4666, + "theoretical_loss": 3.8316100592787494, + "tokens_seen": 658440192 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004042728184553661, + "loss": 1.4866, + "theoretical_loss": 3.8315712054255355, + "tokens_seen": 658505728 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004042627883650953, + "loss": 1.4027, + "theoretical_loss": 3.8315323565215422, + "tokens_seen": 658571264 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 224217, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5434553623199463, + "objective/train/theoretical_loss": 3.831493512565646, + "objective/train/tokens_used": 658636800, + "theoretical_loss": 3.831493512565646, + "tokens_seen": 658636800 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004042527582748245, + "loss": 1.5929, + "theoretical_loss": 3.831493512565646, + "tokens_seen": 658636800 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040424272818455364, + "loss": 1.4831, + "theoretical_loss": 3.831454673556725, + "tokens_seen": 658702336 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004042326980942829, + "loss": 1.4872, + "theoretical_loss": 3.8314158394936557, + "tokens_seen": 658767872 + }, + { + "epoch": 0.2, + "learning_rate": 0.000404222668004012, + "loss": 1.4725, + "theoretical_loss": 3.831377010375318, + "tokens_seen": 658833408 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040421263791374124, + "loss": 1.5069, + "theoretical_loss": 3.8313381862005906, + "tokens_seen": 658898944 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004042026078234704, + "loss": 1.4197, + "theoretical_loss": 3.8312993669683513, + "tokens_seen": 658964480 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004041925777331996, + "loss": 1.3918, + "theoretical_loss": 3.83126055267748, + "tokens_seen": 659030016 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004041825476429288, + "loss": 1.3928, + "theoretical_loss": 3.8312217433268563, + "tokens_seen": 659095552 + }, + { + "epoch": 0.2, + "learning_rate": 0.000404172517552658, + "loss": 1.435, + "theoretical_loss": 3.8311829389153607, + "tokens_seen": 659161088 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040416248746238714, + "loss": 1.3577, + "theoretical_loss": 3.8311441394418733, + "tokens_seen": 659226624 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004041524573721164, + "loss": 1.4848, + "theoretical_loss": 3.831105344905274, + "tokens_seen": 659292160 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004041424272818455, + "loss": 1.5049, + "theoretical_loss": 3.831066555304446, + "tokens_seen": 659357696 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040413239719157474, + "loss": 1.4135, + "theoretical_loss": 3.8310277706382694, + "tokens_seen": 659423232 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004041223671013039, + "loss": 1.416, + "theoretical_loss": 3.830988990905626, + "tokens_seen": 659488768 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004041123370110331, + "loss": 1.4308, + "theoretical_loss": 3.830950216105399, + "tokens_seen": 659554304 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004041023069207623, + "loss": 1.417, + "theoretical_loss": 3.83091144623647, + "tokens_seen": 659619840 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040409227683049147, + "loss": 1.424, + "theoretical_loss": 3.830872681297723, + "tokens_seen": 659685376 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040408224674022065, + "loss": 1.447, + "theoretical_loss": 3.8308339212880407, + "tokens_seen": 659750912 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004040722166499499, + "loss": 1.2909, + "theoretical_loss": 3.830795166206307, + "tokens_seen": 659816448 + }, + { + "epoch": 0.2, + "learning_rate": 0.000404062186559679, + "loss": 1.4944, + "theoretical_loss": 3.830756416051406, + "tokens_seen": 659881984 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040405215646940824, + "loss": 1.4003, + "theoretical_loss": 3.8307176708222217, + "tokens_seen": 659947520 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040404212637913737, + "loss": 1.398, + "theoretical_loss": 3.8306789305176396, + "tokens_seen": 660013056 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004040320962888666, + "loss": 1.3244, + "theoretical_loss": 3.830640195136545, + "tokens_seen": 660078592 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004040220661985958, + "loss": 1.4607, + "theoretical_loss": 3.830601464677823, + "tokens_seen": 660144128 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040401203610832497, + "loss": 1.3948, + "theoretical_loss": 3.8305627391403596, + "tokens_seen": 660209664 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040400200601805415, + "loss": 1.3885, + "theoretical_loss": 3.830524018523041, + "tokens_seen": 660275200 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004039919759277834, + "loss": 1.3939, + "theoretical_loss": 3.8304853028247536, + "tokens_seen": 660340736 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004039819458375125, + "loss": 1.543, + "theoretical_loss": 3.830446592044385, + "tokens_seen": 660406272 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040397191574724175, + "loss": 1.4279, + "theoretical_loss": 3.830407886180822, + "tokens_seen": 660471808 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004039618856569709, + "loss": 1.4824, + "theoretical_loss": 3.8303691852329522, + "tokens_seen": 660537344 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004039518555667001, + "loss": 1.5589, + "theoretical_loss": 3.830330489199665, + "tokens_seen": 660602880 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004039418254764293, + "loss": 1.4808, + "theoretical_loss": 3.8302917980798465, + "tokens_seen": 660668416 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040393179538615847, + "loss": 1.3914, + "theoretical_loss": 3.8302531118723877, + "tokens_seen": 660733952 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040392176529588765, + "loss": 1.3895, + "theoretical_loss": 3.830214430576177, + "tokens_seen": 660799488 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040391173520561683, + "loss": 1.4812, + "theoretical_loss": 3.8301757541901034, + "tokens_seen": 660865024 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040390170511534607, + "loss": 1.427, + "theoretical_loss": 3.8301370827130574, + "tokens_seen": 660930560 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040389167502507525, + "loss": 1.379, + "theoretical_loss": 3.8300984161439295, + "tokens_seen": 660996096 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040388164493480443, + "loss": 1.4878, + "theoretical_loss": 3.830059754481609, + "tokens_seen": 661061632 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004038716148445336, + "loss": 1.5569, + "theoretical_loss": 3.8300210977249884, + "tokens_seen": 661127168 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040386158475426285, + "loss": 1.4182, + "theoretical_loss": 3.8299824458729583, + "tokens_seen": 661192704 + }, + { + "epoch": 0.2, + "learning_rate": 0.000403851554663992, + "loss": 1.3985, + "theoretical_loss": 3.8299437989244103, + "tokens_seen": 661258240 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004038415245737212, + "loss": 1.3968, + "theoretical_loss": 3.8299051568782363, + "tokens_seen": 661323776 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040383149448345034, + "loss": 1.3493, + "theoretical_loss": 3.8298665197333293, + "tokens_seen": 661389312 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040382146439317957, + "loss": 1.3632, + "theoretical_loss": 3.829827887488582, + "tokens_seen": 661454848 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040381143430290875, + "loss": 1.353, + "theoretical_loss": 3.8297892601428867, + "tokens_seen": 661520384 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040380140421263794, + "loss": 1.4721, + "theoretical_loss": 3.8297506376951382, + "tokens_seen": 661585920 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004037913741223671, + "loss": 1.5706, + "theoretical_loss": 3.829712020144229, + "tokens_seen": 661651456 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004037813440320963, + "loss": 1.5065, + "theoretical_loss": 3.8296734074890546, + "tokens_seen": 661716992 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004037713139418255, + "loss": 1.484, + "theoretical_loss": 3.8296347997285083, + "tokens_seen": 661782528 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004037612838515547, + "loss": 1.4238, + "theoretical_loss": 3.829596196861486, + "tokens_seen": 661848064 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 225070, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5283631086349487, + "objective/train/theoretical_loss": 3.8295575988868826, + "objective/train/tokens_used": 661913600, + "theoretical_loss": 3.8295575988868826, + "tokens_seen": 661913600 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040375125376128384, + "loss": 1.4068, + "theoretical_loss": 3.8295575988868826, + "tokens_seen": 661913600 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004037412236710131, + "loss": 1.4385, + "theoretical_loss": 3.8295190058035935, + "tokens_seen": 661979136 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004037311935807422, + "loss": 1.4417, + "theoretical_loss": 3.8294804176105153, + "tokens_seen": 662044672 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040372116349047144, + "loss": 1.4644, + "theoretical_loss": 3.829441834306544, + "tokens_seen": 662110208 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004037111334002006, + "loss": 1.4347, + "theoretical_loss": 3.8294032558905764, + "tokens_seen": 662175744 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004037011033099298, + "loss": 1.4452, + "theoretical_loss": 3.829364682361509, + "tokens_seen": 662241280 + }, + { + "epoch": 0.2, + "learning_rate": 0.000403691073219659, + "loss": 1.4104, + "theoretical_loss": 3.82932611371824, + "tokens_seen": 662306816 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004036810431293882, + "loss": 1.3536, + "theoretical_loss": 3.829287549959667, + "tokens_seen": 662372352 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040367101303911734, + "loss": 1.3239, + "theoretical_loss": 3.8292489910846887, + "tokens_seen": 662437888 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004036609829488466, + "loss": 1.4219, + "theoretical_loss": 3.829210437092202, + "tokens_seen": 662503424 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004036509528585757, + "loss": 1.3076, + "theoretical_loss": 3.8291718879811074, + "tokens_seen": 662568960 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040364092276830494, + "loss": 1.4225, + "theoretical_loss": 3.8291333437503035, + "tokens_seen": 662634496 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004036308926780341, + "loss": 1.4496, + "theoretical_loss": 3.82909480439869, + "tokens_seen": 662700032 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004036208625877633, + "loss": 1.4572, + "theoretical_loss": 3.8290562699251662, + "tokens_seen": 662765568 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004036108324974925, + "loss": 1.4213, + "theoretical_loss": 3.8290177403286334, + "tokens_seen": 662831104 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040360080240722167, + "loss": 1.3872, + "theoretical_loss": 3.828979215607991, + "tokens_seen": 662896640 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040359077231695085, + "loss": 1.53, + "theoretical_loss": 3.8289406957621415, + "tokens_seen": 662962176 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004035807422266801, + "loss": 1.418, + "theoretical_loss": 3.828902180789985, + "tokens_seen": 663027712 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004035707121364092, + "loss": 1.4558, + "theoretical_loss": 3.828863670690424, + "tokens_seen": 663093248 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040356068204613844, + "loss": 1.4071, + "theoretical_loss": 3.8288251654623604, + "tokens_seen": 663158784 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040355065195586757, + "loss": 1.4933, + "theoretical_loss": 3.8287866651046962, + "tokens_seen": 663224320 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004035406218655968, + "loss": 1.4707, + "theoretical_loss": 3.8287481696163344, + "tokens_seen": 663289856 + }, + { + "epoch": 0.2, + "learning_rate": 0.000403530591775326, + "loss": 1.5021, + "theoretical_loss": 3.828709678996179, + "tokens_seen": 663355392 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040352056168505517, + "loss": 1.5141, + "theoretical_loss": 3.8286711932431325, + "tokens_seen": 663420928 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040351053159478435, + "loss": 1.4606, + "theoretical_loss": 3.8286327123560984, + "tokens_seen": 663486464 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004035005015045136, + "loss": 1.3759, + "theoretical_loss": 3.8285942363339824, + "tokens_seen": 663552000 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004034904714142427, + "loss": 1.4144, + "theoretical_loss": 3.828555765175688, + "tokens_seen": 663617536 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040348044132397195, + "loss": 1.4032, + "theoretical_loss": 3.8285172988801195, + "tokens_seen": 663683072 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004034704112337011, + "loss": 1.5047, + "theoretical_loss": 3.8284788374461836, + "tokens_seen": 663748608 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004034603811434303, + "loss": 1.4492, + "theoretical_loss": 3.8284403808727854, + "tokens_seen": 663814144 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004034503510531595, + "loss": 1.3888, + "theoretical_loss": 3.8284019291588303, + "tokens_seen": 663879680 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040344032096288867, + "loss": 1.4491, + "theoretical_loss": 3.8283634823032253, + "tokens_seen": 663945216 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040343029087261785, + "loss": 1.4165, + "theoretical_loss": 3.828325040304877, + "tokens_seen": 664010752 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040342026078234703, + "loss": 1.3586, + "theoretical_loss": 3.8282866031626925, + "tokens_seen": 664076288 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004034102306920762, + "loss": 1.4007, + "theoretical_loss": 3.828248170875579, + "tokens_seen": 664141824 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040340020060180545, + "loss": 1.3578, + "theoretical_loss": 3.828209743442444, + "tokens_seen": 664207360 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004033901705115346, + "loss": 1.3606, + "theoretical_loss": 3.828171320862196, + "tokens_seen": 664272896 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004033801404212638, + "loss": 1.2983, + "theoretical_loss": 3.8281329031337434, + "tokens_seen": 664338432 + }, + { + "epoch": 0.2, + "learning_rate": 0.000403370110330993, + "loss": 1.3667, + "theoretical_loss": 3.828094490255995, + "tokens_seen": 664403968 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004033600802407222, + "loss": 1.4772, + "theoretical_loss": 3.8280560822278598, + "tokens_seen": 664469504 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040335005015045136, + "loss": 1.3521, + "theoretical_loss": 3.8280176790482474, + "tokens_seen": 664535040 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040334002006018054, + "loss": 1.4624, + "theoretical_loss": 3.827979280716068, + "tokens_seen": 664600576 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004033299899699097, + "loss": 1.333, + "theoretical_loss": 3.8279408872302314, + "tokens_seen": 664666112 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040331995987963895, + "loss": 1.4773, + "theoretical_loss": 3.827902498589648, + "tokens_seen": 664731648 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004033099297893681, + "loss": 1.4961, + "theoretical_loss": 3.827864114793229, + "tokens_seen": 664797184 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004032998996990973, + "loss": 1.4112, + "theoretical_loss": 3.827825735839886, + "tokens_seen": 664862720 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040328986960882644, + "loss": 1.4005, + "theoretical_loss": 3.8277873617285305, + "tokens_seen": 664928256 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004032798395185557, + "loss": 1.4272, + "theoretical_loss": 3.8277489924580745, + "tokens_seen": 664993792 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040326980942828486, + "loss": 1.3854, + "theoretical_loss": 3.82771062802743, + "tokens_seen": 665059328 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040325977933801404, + "loss": 1.3268, + "theoretical_loss": 3.827672268435509, + "tokens_seen": 665124864 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 226202, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5618467330932617, + "objective/train/theoretical_loss": 3.8276339136812263, + "objective/train/tokens_used": 665190400, + "theoretical_loss": 3.8276339136812263, + "tokens_seen": 665190400 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004032497492477432, + "loss": 1.5127, + "theoretical_loss": 3.8276339136812263, + "tokens_seen": 665190400 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004032397191574724, + "loss": 1.3781, + "theoretical_loss": 3.8275955637634933, + "tokens_seen": 665255936 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004032296890672016, + "loss": 1.4338, + "theoretical_loss": 3.8275572186812257, + "tokens_seen": 665321472 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004032196589769308, + "loss": 1.3648, + "theoretical_loss": 3.8275188784333363, + "tokens_seen": 665387008 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040320962888665995, + "loss": 1.5034, + "theoretical_loss": 3.8274805430187406, + "tokens_seen": 665452544 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004031995987963892, + "loss": 1.4289, + "theoretical_loss": 3.8274422124363516, + "tokens_seen": 665518080 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040318956870611836, + "loss": 1.4267, + "theoretical_loss": 3.8274038866850866, + "tokens_seen": 665583616 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040317953861584754, + "loss": 1.4305, + "theoretical_loss": 3.8273655657638597, + "tokens_seen": 665649152 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004031695085255767, + "loss": 1.3136, + "theoretical_loss": 3.827327249671587, + "tokens_seen": 665714688 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004031594784353059, + "loss": 1.4741, + "theoretical_loss": 3.8272889384071846, + "tokens_seen": 665780224 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040314944834503514, + "loss": 1.4366, + "theoretical_loss": 3.8272506319695694, + "tokens_seen": 665845760 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004031394182547643, + "loss": 1.3935, + "theoretical_loss": 3.827212330357658, + "tokens_seen": 665911296 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004031293881644935, + "loss": 1.2989, + "theoretical_loss": 3.827174033570368, + "tokens_seen": 665976832 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004031193580742227, + "loss": 1.344, + "theoretical_loss": 3.8271357416066167, + "tokens_seen": 666042368 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040310932798395187, + "loss": 1.4409, + "theoretical_loss": 3.827097454465322, + "tokens_seen": 666107904 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040309929789368105, + "loss": 1.3424, + "theoretical_loss": 3.827059172145402, + "tokens_seen": 666173440 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004030892678034103, + "loss": 1.4026, + "theoretical_loss": 3.827020894645776, + "tokens_seen": 666238976 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004030792377131394, + "loss": 1.4997, + "theoretical_loss": 3.826982621965363, + "tokens_seen": 666304512 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040306920762286864, + "loss": 1.4583, + "theoretical_loss": 3.826944354103081, + "tokens_seen": 666370048 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040305917753259777, + "loss": 1.3361, + "theoretical_loss": 3.826906091057851, + "tokens_seen": 666435584 + }, + { + "epoch": 0.2, + "learning_rate": 0.000403049147442327, + "loss": 1.3503, + "theoretical_loss": 3.8268678328285928, + "tokens_seen": 666501120 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004030391173520562, + "loss": 1.3147, + "theoretical_loss": 3.8268295794142264, + "tokens_seen": 666566656 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040302908726178537, + "loss": 1.4097, + "theoretical_loss": 3.826791330813673, + "tokens_seen": 666632192 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040301905717151455, + "loss": 1.3944, + "theoretical_loss": 3.8267530870258533, + "tokens_seen": 666697728 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004030090270812438, + "loss": 1.4342, + "theoretical_loss": 3.826714848049689, + "tokens_seen": 666763264 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004029989969909729, + "loss": 1.3044, + "theoretical_loss": 3.8266766138841013, + "tokens_seen": 666828800 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040298896690070215, + "loss": 1.4533, + "theoretical_loss": 3.826638384528013, + "tokens_seen": 666894336 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004029789368104313, + "loss": 1.3494, + "theoretical_loss": 3.8266001599803467, + "tokens_seen": 666959872 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004029689067201605, + "loss": 1.4572, + "theoretical_loss": 3.8265619402400244, + "tokens_seen": 667025408 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004029588766298897, + "loss": 1.4341, + "theoretical_loss": 3.82652372530597, + "tokens_seen": 667090944 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040294884653961887, + "loss": 1.3932, + "theoretical_loss": 3.8264855151771067, + "tokens_seen": 667156480 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040293881644934805, + "loss": 1.4224, + "theoretical_loss": 3.826447309852358, + "tokens_seen": 667222016 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040292878635907723, + "loss": 1.414, + "theoretical_loss": 3.826409109330649, + "tokens_seen": 667287552 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004029187562688064, + "loss": 1.4093, + "theoretical_loss": 3.826370913610903, + "tokens_seen": 667353088 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040290872617853565, + "loss": 1.4026, + "theoretical_loss": 3.8263327226920465, + "tokens_seen": 667418624 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004028986960882648, + "loss": 1.4286, + "theoretical_loss": 3.8262945365730032, + "tokens_seen": 667484160 + }, + { + "epoch": 0.2, + "learning_rate": 0.000402888665997994, + "loss": 1.4655, + "theoretical_loss": 3.8262563552526996, + "tokens_seen": 667549696 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004028786359077232, + "loss": 1.4434, + "theoretical_loss": 3.8262181787300618, + "tokens_seen": 667615232 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004028686058174524, + "loss": 1.3815, + "theoretical_loss": 3.8261800070040146, + "tokens_seen": 667680768 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040285857572718156, + "loss": 1.498, + "theoretical_loss": 3.8261418400734866, + "tokens_seen": 667746304 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040284854563691074, + "loss": 1.398, + "theoretical_loss": 3.826103677937403, + "tokens_seen": 667811840 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004028385155466399, + "loss": 1.4189, + "theoretical_loss": 3.826065520594693, + "tokens_seen": 667877376 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040282848545636915, + "loss": 1.5389, + "theoretical_loss": 3.8260273680442825, + "tokens_seen": 667942912 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004028184553660983, + "loss": 1.4158, + "theoretical_loss": 3.8259892202851002, + "tokens_seen": 668008448 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004028084252758275, + "loss": 1.3198, + "theoretical_loss": 3.825951077316075, + "tokens_seen": 668073984 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040279839518555664, + "loss": 1.3984, + "theoretical_loss": 3.825912939136135, + "tokens_seen": 668139520 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004027883650952859, + "loss": 1.4768, + "theoretical_loss": 3.825874805744209, + "tokens_seen": 668205056 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040277833500501506, + "loss": 1.3324, + "theoretical_loss": 3.825836677139227, + "tokens_seen": 668270592 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040276830491474424, + "loss": 1.5212, + "theoretical_loss": 3.8257985533201184, + "tokens_seen": 668336128 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004027582748244734, + "loss": 1.3266, + "theoretical_loss": 3.8257604342858134, + "tokens_seen": 668401664 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 227136, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4975183010101318, + "objective/train/theoretical_loss": 3.8257223200352417, + "objective/train/tokens_used": 668467200, + "theoretical_loss": 3.8257223200352417, + "tokens_seen": 668467200 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004027482447342026, + "loss": 1.4835, + "theoretical_loss": 3.8257223200352417, + "tokens_seen": 668467200 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004027382146439318, + "loss": 1.4605, + "theoretical_loss": 3.825684210567335, + "tokens_seen": 668532736 + }, + { + "epoch": 0.2, + "learning_rate": 0.000402728184553661, + "loss": 1.4676, + "theoretical_loss": 3.8256461058810247, + "tokens_seen": 668598272 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040271815446339015, + "loss": 1.4755, + "theoretical_loss": 3.825608005975241, + "tokens_seen": 668663808 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004027081243731194, + "loss": 1.4042, + "theoretical_loss": 3.8255699108489165, + "tokens_seen": 668729344 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040269809428284856, + "loss": 1.3782, + "theoretical_loss": 3.8255318205009834, + "tokens_seen": 668794880 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040268806419257774, + "loss": 1.4058, + "theoretical_loss": 3.825493734930374, + "tokens_seen": 668860416 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004026780341023069, + "loss": 1.3554, + "theoretical_loss": 3.8254556541360207, + "tokens_seen": 668925952 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004026680040120361, + "loss": 1.4372, + "theoretical_loss": 3.8254175781168573, + "tokens_seen": 668991488 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004026579739217653, + "loss": 1.3454, + "theoretical_loss": 3.825379506871817, + "tokens_seen": 669057024 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004026479438314945, + "loss": 1.4517, + "theoretical_loss": 3.8253414403998334, + "tokens_seen": 669122560 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040263791374122365, + "loss": 1.2951, + "theoretical_loss": 3.825303378699841, + "tokens_seen": 669188096 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004026278836509529, + "loss": 1.3891, + "theoretical_loss": 3.8252653217707753, + "tokens_seen": 669253632 + }, + { + "epoch": 0.2, + "learning_rate": 0.000402617853560682, + "loss": 1.3662, + "theoretical_loss": 3.8252272696115694, + "tokens_seen": 669319168 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040260782347041125, + "loss": 1.4623, + "theoretical_loss": 3.8251892222211596, + "tokens_seen": 669384704 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040259779338014043, + "loss": 1.4033, + "theoretical_loss": 3.825151179598481, + "tokens_seen": 669450240 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004025877632898696, + "loss": 1.4222, + "theoretical_loss": 3.8251131417424693, + "tokens_seen": 669515776 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004025777331995988, + "loss": 1.3579, + "theoretical_loss": 3.8250751086520616, + "tokens_seen": 669581312 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040256770310932797, + "loss": 1.4919, + "theoretical_loss": 3.825037080326194, + "tokens_seen": 669646848 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040255767301905715, + "loss": 1.3577, + "theoretical_loss": 3.8249990567638035, + "tokens_seen": 669712384 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004025476429287864, + "loss": 1.397, + "theoretical_loss": 3.8249610379638277, + "tokens_seen": 669777920 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004025376128385155, + "loss": 1.542, + "theoretical_loss": 3.8249230239252032, + "tokens_seen": 669843456 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040252758274824475, + "loss": 1.4799, + "theoretical_loss": 3.8248850146468687, + "tokens_seen": 669908992 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040251755265797393, + "loss": 1.4323, + "theoretical_loss": 3.8248470101277627, + "tokens_seen": 669974528 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004025075225677031, + "loss": 1.3788, + "theoretical_loss": 3.824809010366823, + "tokens_seen": 670040064 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004024974924774323, + "loss": 1.3916, + "theoretical_loss": 3.8247710153629892, + "tokens_seen": 670105600 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004024874623871615, + "loss": 1.3202, + "theoretical_loss": 3.8247330251152007, + "tokens_seen": 670171136 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040247743229689066, + "loss": 1.4824, + "theoretical_loss": 3.824695039622397, + "tokens_seen": 670236672 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004024674022066199, + "loss": 1.3937, + "theoretical_loss": 3.824657058883518, + "tokens_seen": 670302208 + }, + { + "epoch": 0.2, + "learning_rate": 0.000402457372116349, + "loss": 1.3077, + "theoretical_loss": 3.824619082897504, + "tokens_seen": 670367744 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040244734202607825, + "loss": 1.4779, + "theoretical_loss": 3.824581111663296, + "tokens_seen": 670433280 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004024373119358074, + "loss": 1.4511, + "theoretical_loss": 3.8245431451798346, + "tokens_seen": 670498816 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004024272818455366, + "loss": 1.4685, + "theoretical_loss": 3.8245051834460613, + "tokens_seen": 670564352 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004024172517552658, + "loss": 1.4213, + "theoretical_loss": 3.824467226460918, + "tokens_seen": 670629888 + }, + { + "epoch": 0.2, + "learning_rate": 0.000402407221664995, + "loss": 1.4801, + "theoretical_loss": 3.8244292742233457, + "tokens_seen": 670695424 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004023971915747242, + "loss": 1.3995, + "theoretical_loss": 3.8243913267322887, + "tokens_seen": 670760960 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004023871614844534, + "loss": 1.3761, + "theoretical_loss": 3.824353383986688, + "tokens_seen": 670826496 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004023771313941826, + "loss": 1.4239, + "theoretical_loss": 3.824315445985487, + "tokens_seen": 670892032 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040236710130391176, + "loss": 1.3791, + "theoretical_loss": 3.8242775127276296, + "tokens_seen": 670957568 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040235707121364094, + "loss": 1.2534, + "theoretical_loss": 3.8242395842120596, + "tokens_seen": 671023104 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004023470411233701, + "loss": 1.5376, + "theoretical_loss": 3.824201660437721, + "tokens_seen": 671088640 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040233701103309935, + "loss": 1.3329, + "theoretical_loss": 3.824163741403557, + "tokens_seen": 671154176 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004023269809428285, + "loss": 1.3606, + "theoretical_loss": 3.8241258271085137, + "tokens_seen": 671219712 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004023169508525577, + "loss": 1.3775, + "theoretical_loss": 3.824087917551536, + "tokens_seen": 671285248 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040230692076228684, + "loss": 1.3282, + "theoretical_loss": 3.8240500127315693, + "tokens_seen": 671350784 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004022968906720161, + "loss": 1.48, + "theoretical_loss": 3.8240121126475586, + "tokens_seen": 671416320 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040228686058174526, + "loss": 1.3525, + "theoretical_loss": 3.82397421729845, + "tokens_seen": 671481856 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040227683049147444, + "loss": 1.3316, + "theoretical_loss": 3.8239363266831914, + "tokens_seen": 671547392 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004022668004012036, + "loss": 1.3982, + "theoretical_loss": 3.823898440800728, + "tokens_seen": 671612928 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004022567703109328, + "loss": 1.3427, + "theoretical_loss": 3.823860559650008, + "tokens_seen": 671678464 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 228328, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.469504475593567, + "objective/train/theoretical_loss": 3.823822683229978, + "objective/train/tokens_used": 671744000, + "theoretical_loss": 3.823822683229978, + "tokens_seen": 671744000 + }, + { + "epoch": 0.2, + "learning_rate": 0.000402246740220662, + "loss": 1.4106, + "theoretical_loss": 3.823822683229978, + "tokens_seen": 671744000 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004022367101303912, + "loss": 1.3731, + "theoretical_loss": 3.8237848115395865, + "tokens_seen": 671809536 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040222668004012035, + "loss": 1.4249, + "theoretical_loss": 3.823746944577781, + "tokens_seen": 671875072 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004022166499498496, + "loss": 1.321, + "theoretical_loss": 3.82370908234351, + "tokens_seen": 671940608 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040220661985957876, + "loss": 1.3603, + "theoretical_loss": 3.8236712248357225, + "tokens_seen": 672006144 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040219658976930794, + "loss": 1.3733, + "theoretical_loss": 3.8236333720533677, + "tokens_seen": 672071680 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004021865596790371, + "loss": 1.3874, + "theoretical_loss": 3.823595523995395, + "tokens_seen": 672137216 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004021765295887663, + "loss": 1.5194, + "theoretical_loss": 3.823557680660754, + "tokens_seen": 672202752 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004021664994984955, + "loss": 1.3614, + "theoretical_loss": 3.823519842048395, + "tokens_seen": 672268288 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004021564694082247, + "loss": 1.3734, + "theoretical_loss": 3.823482008157269, + "tokens_seen": 672333824 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040214643931795385, + "loss": 1.3594, + "theoretical_loss": 3.8234441789863256, + "tokens_seen": 672399360 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004021364092276831, + "loss": 1.3907, + "theoretical_loss": 3.8234063545345167, + "tokens_seen": 672464896 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004021263791374122, + "loss": 1.2903, + "theoretical_loss": 3.8233685348007933, + "tokens_seen": 672530432 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040211634904714145, + "loss": 1.3849, + "theoretical_loss": 3.823330719784108, + "tokens_seen": 672595968 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040210631895687063, + "loss": 1.3296, + "theoretical_loss": 3.8232929094834125, + "tokens_seen": 672661504 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004020962888665998, + "loss": 1.3526, + "theoretical_loss": 3.823255103897659, + "tokens_seen": 672727040 + }, + { + "epoch": 0.2, + "learning_rate": 0.000402086258776329, + "loss": 1.3206, + "theoretical_loss": 3.8232173030258005, + "tokens_seen": 672792576 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040207622868605817, + "loss": 1.3925, + "theoretical_loss": 3.823179506866791, + "tokens_seen": 672858112 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040206619859578735, + "loss": 1.582, + "theoretical_loss": 3.823141715419582, + "tokens_seen": 672923648 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004020561685055166, + "loss": 1.4165, + "theoretical_loss": 3.823103928683129, + "tokens_seen": 672989184 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004020461384152457, + "loss": 1.3228, + "theoretical_loss": 3.8230661466563864, + "tokens_seen": 673054720 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040203610832497495, + "loss": 1.421, + "theoretical_loss": 3.8230283693383074, + "tokens_seen": 673120256 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040202607823470413, + "loss": 1.4181, + "theoretical_loss": 3.822990596727847, + "tokens_seen": 673185792 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004020160481444333, + "loss": 1.3737, + "theoretical_loss": 3.822952828823961, + "tokens_seen": 673251328 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004020060180541625, + "loss": 1.4966, + "theoretical_loss": 3.8229150656256046, + "tokens_seen": 673316864 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004019959879638917, + "loss": 1.3787, + "theoretical_loss": 3.8228773071317335, + "tokens_seen": 673382400 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040198595787362086, + "loss": 1.4611, + "theoretical_loss": 3.822839553341304, + "tokens_seen": 673447936 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004019759277833501, + "loss": 1.3994, + "theoretical_loss": 3.822801804253273, + "tokens_seen": 673513472 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004019658976930792, + "loss": 1.3706, + "theoretical_loss": 3.8227640598665964, + "tokens_seen": 673579008 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040195586760280845, + "loss": 1.3642, + "theoretical_loss": 3.822726320180232, + "tokens_seen": 673644544 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004019458375125376, + "loss": 1.4026, + "theoretical_loss": 3.822688585193137, + "tokens_seen": 673710080 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004019358074222668, + "loss": 1.3219, + "theoretical_loss": 3.822650854904269, + "tokens_seen": 673775616 + }, + { + "epoch": 0.2, + "learning_rate": 0.000401925777331996, + "loss": 1.4663, + "theoretical_loss": 3.822613129312587, + "tokens_seen": 673841152 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004019157472417252, + "loss": 1.3867, + "theoretical_loss": 3.8225754084170482, + "tokens_seen": 673906688 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040190571715145436, + "loss": 1.3934, + "theoretical_loss": 3.8225376922166125, + "tokens_seen": 673972224 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004018956870611836, + "loss": 1.318, + "theoretical_loss": 3.822499980710239, + "tokens_seen": 674037760 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004018856569709127, + "loss": 1.4088, + "theoretical_loss": 3.8224622738968863, + "tokens_seen": 674103296 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040187562688064196, + "loss": 1.4116, + "theoretical_loss": 3.822424571775515, + "tokens_seen": 674168832 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004018655967903711, + "loss": 1.3034, + "theoretical_loss": 3.822386874345085, + "tokens_seen": 674234368 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004018555667001003, + "loss": 1.2909, + "theoretical_loss": 3.822349181604557, + "tokens_seen": 674299904 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004018455366098295, + "loss": 1.4528, + "theoretical_loss": 3.8223114935528915, + "tokens_seen": 674365440 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004018355065195587, + "loss": 1.4273, + "theoretical_loss": 3.82227381018905, + "tokens_seen": 674430976 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040182547642928786, + "loss": 1.4762, + "theoretical_loss": 3.822236131511993, + "tokens_seen": 674496512 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040181544633901704, + "loss": 1.4194, + "theoretical_loss": 3.8221984575206833, + "tokens_seen": 674562048 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004018054162487462, + "loss": 1.3923, + "theoretical_loss": 3.8221607882140827, + "tokens_seen": 674627584 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040179538615847546, + "loss": 1.3629, + "theoretical_loss": 3.8221231235911537, + "tokens_seen": 674693120 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004017853560682046, + "loss": 1.3815, + "theoretical_loss": 3.822085463650859, + "tokens_seen": 674758656 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004017753259779338, + "loss": 1.3391, + "theoretical_loss": 3.8220478083921616, + "tokens_seen": 674824192 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040176529588766295, + "loss": 1.45, + "theoretical_loss": 3.8220101578140255, + "tokens_seen": 674889728 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004017552657973922, + "loss": 1.4753, + "theoretical_loss": 3.8219725119154138, + "tokens_seen": 674955264 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 229330, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5177536010742188, + "objective/train/theoretical_loss": 3.821934870695291, + "objective/train/tokens_used": 675020800, + "theoretical_loss": 3.821934870695291, + "tokens_seen": 675020800 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040174523570712137, + "loss": 1.4435, + "theoretical_loss": 3.821934870695291, + "tokens_seen": 675020800 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040173520561685055, + "loss": 1.5058, + "theoretical_loss": 3.8218972341526216, + "tokens_seen": 675086336 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040172517552657973, + "loss": 1.5412, + "theoretical_loss": 3.8218596022863696, + "tokens_seen": 675151872 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040171514543630896, + "loss": 1.3937, + "theoretical_loss": 3.821821975095501, + "tokens_seen": 675217408 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004017051153460381, + "loss": 1.4432, + "theoretical_loss": 3.821784352578981, + "tokens_seen": 675282944 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004016950852557673, + "loss": 1.3341, + "theoretical_loss": 3.8217467347357754, + "tokens_seen": 675348480 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040168505516549645, + "loss": 1.4335, + "theoretical_loss": 3.82170912156485, + "tokens_seen": 675414016 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004016750250752257, + "loss": 1.4452, + "theoretical_loss": 3.8216715130651715, + "tokens_seen": 675479552 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040166499498495487, + "loss": 1.392, + "theoretical_loss": 3.8216339092357066, + "tokens_seen": 675545088 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040165496489468405, + "loss": 1.4618, + "theoretical_loss": 3.8215963100754218, + "tokens_seen": 675610624 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004016449348044133, + "loss": 1.3509, + "theoretical_loss": 3.8215587155832855, + "tokens_seen": 675676160 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004016349047141424, + "loss": 1.4002, + "theoretical_loss": 3.8215211257582644, + "tokens_seen": 675741696 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040162487462387165, + "loss": 1.4832, + "theoretical_loss": 3.8214835405993273, + "tokens_seen": 675807232 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040161484453360083, + "loss": 1.5766, + "theoretical_loss": 3.8214459601054425, + "tokens_seen": 675872768 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040160481444333, + "loss": 1.4355, + "theoretical_loss": 3.8214083842755784, + "tokens_seen": 675938304 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004015947843530592, + "loss": 1.3266, + "theoretical_loss": 3.8213708131087043, + "tokens_seen": 676003840 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040158475426278837, + "loss": 1.3723, + "theoretical_loss": 3.8213332466037895, + "tokens_seen": 676069376 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040157472417251755, + "loss": 1.4256, + "theoretical_loss": 3.8212956847598036, + "tokens_seen": 676134912 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004015646940822468, + "loss": 1.3648, + "theoretical_loss": 3.8212581275757165, + "tokens_seen": 676200448 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004015546639919759, + "loss": 1.4253, + "theoretical_loss": 3.821220575050499, + "tokens_seen": 676265984 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040154463390170515, + "loss": 1.4525, + "theoretical_loss": 3.821183027183122, + "tokens_seen": 676331520 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040153460381143433, + "loss": 1.4044, + "theoretical_loss": 3.821145483972556, + "tokens_seen": 676397056 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004015245737211635, + "loss": 1.4825, + "theoretical_loss": 3.821107945417771, + "tokens_seen": 676462592 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004015145436308927, + "loss": 1.3978, + "theoretical_loss": 3.8210704115177414, + "tokens_seen": 676528128 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004015045135406219, + "loss": 1.2707, + "theoretical_loss": 3.8210328822714374, + "tokens_seen": 676593664 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040149448345035106, + "loss": 1.3317, + "theoretical_loss": 3.820995357677832, + "tokens_seen": 676659200 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004014844533600803, + "loss": 1.2349, + "theoretical_loss": 3.8209578377358975, + "tokens_seen": 676724736 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004014744232698094, + "loss": 1.4171, + "theoretical_loss": 3.8209203224446067, + "tokens_seen": 676790272 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040146439317953865, + "loss": 1.4867, + "theoretical_loss": 3.820882811802933, + "tokens_seen": 676855808 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004014543630892678, + "loss": 1.4514, + "theoretical_loss": 3.82084530580985, + "tokens_seen": 676921344 + }, + { + "epoch": 0.21, + "learning_rate": 0.000401444332998997, + "loss": 1.467, + "theoretical_loss": 3.8208078044643328, + "tokens_seen": 676986880 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004014343029087262, + "loss": 1.3878, + "theoretical_loss": 3.8207703077653536, + "tokens_seen": 677052416 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004014242728184554, + "loss": 1.3413, + "theoretical_loss": 3.8207328157118887, + "tokens_seen": 677117952 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040141424272818456, + "loss": 1.404, + "theoretical_loss": 3.820695328302912, + "tokens_seen": 677183488 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004014042126379138, + "loss": 1.5257, + "theoretical_loss": 3.8206578455373994, + "tokens_seen": 677249024 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004013941825476429, + "loss": 1.3234, + "theoretical_loss": 3.820620367414326, + "tokens_seen": 677314560 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040138415245737216, + "loss": 1.4877, + "theoretical_loss": 3.8205828939326683, + "tokens_seen": 677380096 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004013741223671013, + "loss": 1.5204, + "theoretical_loss": 3.8205454250914013, + "tokens_seen": 677445632 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004013640922768305, + "loss": 1.4535, + "theoretical_loss": 3.820507960889503, + "tokens_seen": 677511168 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004013540621865597, + "loss": 1.3591, + "theoretical_loss": 3.82047050132595, + "tokens_seen": 677576704 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004013440320962889, + "loss": 1.4659, + "theoretical_loss": 3.8204330463997183, + "tokens_seen": 677642240 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040133400200601806, + "loss": 1.4089, + "theoretical_loss": 3.820395596109787, + "tokens_seen": 677707776 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040132397191574724, + "loss": 1.3966, + "theoretical_loss": 3.8203581504551325, + "tokens_seen": 677773312 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004013139418254764, + "loss": 1.3353, + "theoretical_loss": 3.820320709434734, + "tokens_seen": 677838848 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040130391173520566, + "loss": 1.4832, + "theoretical_loss": 3.8202832730475693, + "tokens_seen": 677904384 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004012938816449348, + "loss": 1.3712, + "theoretical_loss": 3.820245841292618, + "tokens_seen": 677969920 + }, + { + "epoch": 0.21, + "learning_rate": 0.000401283851554664, + "loss": 1.3377, + "theoretical_loss": 3.820208414168859, + "tokens_seen": 678035456 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040127382146439315, + "loss": 1.3763, + "theoretical_loss": 3.820170991675271, + "tokens_seen": 678100992 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004012637913741224, + "loss": 1.3952, + "theoretical_loss": 3.8201335738108346, + "tokens_seen": 678166528 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040125376128385157, + "loss": 1.4275, + "theoretical_loss": 3.82009616057453, + "tokens_seen": 678232064 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 230490, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.342788577079773, + "objective/train/theoretical_loss": 3.8200587519653366, + "objective/train/tokens_used": 678297600, + "theoretical_loss": 3.8200587519653366, + "tokens_seen": 678297600 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040124373119358075, + "loss": 1.3947, + "theoretical_loss": 3.8200587519653366, + "tokens_seen": 678297600 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040123370110330993, + "loss": 1.2743, + "theoretical_loss": 3.8200213479822365, + "tokens_seen": 678363136 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040122367101303916, + "loss": 1.4462, + "theoretical_loss": 3.81998394862421, + "tokens_seen": 678428672 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004012136409227683, + "loss": 1.4299, + "theoretical_loss": 3.8199465538902384, + "tokens_seen": 678494208 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004012036108324975, + "loss": 1.3505, + "theoretical_loss": 3.819909163779304, + "tokens_seen": 678559744 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040119358074222665, + "loss": 1.3146, + "theoretical_loss": 3.8198717782903886, + "tokens_seen": 678625280 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004011835506519559, + "loss": 1.3845, + "theoretical_loss": 3.8198343974224747, + "tokens_seen": 678690816 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040117352056168507, + "loss": 1.4413, + "theoretical_loss": 3.8197970211745442, + "tokens_seen": 678756352 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040116349047141425, + "loss": 1.4463, + "theoretical_loss": 3.819759649545581, + "tokens_seen": 678821888 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040115346038114343, + "loss": 1.4294, + "theoretical_loss": 3.819722282534568, + "tokens_seen": 678887424 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004011434302908726, + "loss": 1.4064, + "theoretical_loss": 3.8196849201404888, + "tokens_seen": 678952960 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004011334002006018, + "loss": 1.4331, + "theoretical_loss": 3.819647562362328, + "tokens_seen": 679018496 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040112337011033103, + "loss": 1.3166, + "theoretical_loss": 3.819610209199069, + "tokens_seen": 679084032 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040111334002006016, + "loss": 1.3804, + "theoretical_loss": 3.8195728606496973, + "tokens_seen": 679149568 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004011033099297894, + "loss": 1.424, + "theoretical_loss": 3.8195355167131977, + "tokens_seen": 679215104 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004010932798395185, + "loss": 1.3907, + "theoretical_loss": 3.819498177388555, + "tokens_seen": 679280640 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040108324974924775, + "loss": 1.3765, + "theoretical_loss": 3.8194608426747543, + "tokens_seen": 679346176 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040107321965897693, + "loss": 1.439, + "theoretical_loss": 3.819423512570783, + "tokens_seen": 679411712 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004010631895687061, + "loss": 1.4418, + "theoretical_loss": 3.8193861870756267, + "tokens_seen": 679477248 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004010531594784353, + "loss": 1.3506, + "theoretical_loss": 3.819348866188271, + "tokens_seen": 679542784 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040104312938816453, + "loss": 1.3874, + "theoretical_loss": 3.819311549907704, + "tokens_seen": 679608320 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040103309929789366, + "loss": 1.3775, + "theoretical_loss": 3.8192742382329126, + "tokens_seen": 679673856 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004010230692076229, + "loss": 1.4553, + "theoretical_loss": 3.819236931162884, + "tokens_seen": 679739392 + }, + { + "epoch": 0.21, + "learning_rate": 0.000401013039117352, + "loss": 1.3951, + "theoretical_loss": 3.8191996286966066, + "tokens_seen": 679804928 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040100300902708126, + "loss": 1.512, + "theoretical_loss": 3.819162330833068, + "tokens_seen": 679870464 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040099297893681044, + "loss": 1.2783, + "theoretical_loss": 3.8191250375712564, + "tokens_seen": 679936000 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004009829488465396, + "loss": 1.3356, + "theoretical_loss": 3.819087748910161, + "tokens_seen": 680001536 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004009729187562688, + "loss": 1.3649, + "theoretical_loss": 3.8190504648487718, + "tokens_seen": 680067072 + }, + { + "epoch": 0.21, + "learning_rate": 0.000400962888665998, + "loss": 1.3804, + "theoretical_loss": 3.8190131853860763, + "tokens_seen": 680132608 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040095285857572716, + "loss": 1.4469, + "theoretical_loss": 3.8189759105210657, + "tokens_seen": 680198144 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004009428284854564, + "loss": 1.2729, + "theoretical_loss": 3.8189386402527297, + "tokens_seen": 680263680 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004009327983951855, + "loss": 1.4944, + "theoretical_loss": 3.818901374580059, + "tokens_seen": 680329216 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040092276830491476, + "loss": 1.4305, + "theoretical_loss": 3.818864113502044, + "tokens_seen": 680394752 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004009127382146439, + "loss": 1.4532, + "theoretical_loss": 3.818826857017675, + "tokens_seen": 680460288 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004009027081243731, + "loss": 1.3922, + "theoretical_loss": 3.8187896051259442, + "tokens_seen": 680525824 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040089267803410236, + "loss": 1.3311, + "theoretical_loss": 3.818752357825843, + "tokens_seen": 680591360 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004008826479438315, + "loss": 1.3917, + "theoretical_loss": 3.8187151151163636, + "tokens_seen": 680656896 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004008726178535607, + "loss": 1.3223, + "theoretical_loss": 3.8186778769964986, + "tokens_seen": 680722432 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004008625877632899, + "loss": 1.4392, + "theoretical_loss": 3.81864064346524, + "tokens_seen": 680787968 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004008525576730191, + "loss": 1.4185, + "theoretical_loss": 3.818603414521581, + "tokens_seen": 680853504 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040084252758274826, + "loss": 1.3467, + "theoretical_loss": 3.818566190164515, + "tokens_seen": 680919040 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040083249749247744, + "loss": 1.3209, + "theoretical_loss": 3.8185289703930345, + "tokens_seen": 680984576 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004008224674022066, + "loss": 1.4398, + "theoretical_loss": 3.818491755206135, + "tokens_seen": 681050112 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040081243731193586, + "loss": 1.3817, + "theoretical_loss": 3.8184545446028095, + "tokens_seen": 681115648 + }, + { + "epoch": 0.21, + "learning_rate": 0.000400802407221665, + "loss": 1.4227, + "theoretical_loss": 3.8184173385820532, + "tokens_seen": 681181184 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004007923771313942, + "loss": 1.4292, + "theoretical_loss": 3.818380137142861, + "tokens_seen": 681246720 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040078234704112335, + "loss": 1.5084, + "theoretical_loss": 3.8183429402842277, + "tokens_seen": 681312256 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004007723169508526, + "loss": 1.4357, + "theoretical_loss": 3.8183057480051485, + "tokens_seen": 681377792 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040076228686058177, + "loss": 1.4301, + "theoretical_loss": 3.81826856030462, + "tokens_seen": 681443328 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040075225677031095, + "loss": 1.3677, + "theoretical_loss": 3.818231377181638, + "tokens_seen": 681508864 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 231714, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5107736587524414, + "objective/train/theoretical_loss": 3.818194198635198, + "objective/train/tokens_used": 681574400, + "theoretical_loss": 3.818194198635198, + "tokens_seen": 681574400 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040074222668004013, + "loss": 1.4733, + "theoretical_loss": 3.818194198635198, + "tokens_seen": 681574400 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040073219658976936, + "loss": 1.3802, + "theoretical_loss": 3.818157024664298, + "tokens_seen": 681639936 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004007221664994985, + "loss": 1.3258, + "theoretical_loss": 3.8181198552679345, + "tokens_seen": 681705472 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004007121364092277, + "loss": 1.4235, + "theoretical_loss": 3.818082690445105, + "tokens_seen": 681771008 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040070210631895685, + "loss": 1.3832, + "theoretical_loss": 3.818045530194807, + "tokens_seen": 681836544 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004006920762286861, + "loss": 1.3213, + "theoretical_loss": 3.8180083745160394, + "tokens_seen": 681902080 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040068204613841527, + "loss": 1.4102, + "theoretical_loss": 3.817971223407799, + "tokens_seen": 681967616 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040067201604814445, + "loss": 1.4118, + "theoretical_loss": 3.8179340768690855, + "tokens_seen": 682033152 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040066198595787363, + "loss": 1.4645, + "theoretical_loss": 3.817896934898897, + "tokens_seen": 682098688 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004006519558676028, + "loss": 1.4955, + "theoretical_loss": 3.8178597974962343, + "tokens_seen": 682164224 + }, + { + "epoch": 0.21, + "learning_rate": 0.000400641925777332, + "loss": 1.3883, + "theoretical_loss": 3.817822664660096, + "tokens_seen": 682229760 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040063189568706123, + "loss": 1.5413, + "theoretical_loss": 3.817785536389481, + "tokens_seen": 682295296 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040062186559679036, + "loss": 1.4185, + "theoretical_loss": 3.8177484126833914, + "tokens_seen": 682360832 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004006118355065196, + "loss": 1.3407, + "theoretical_loss": 3.8177112935408273, + "tokens_seen": 682426368 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004006018054162487, + "loss": 1.3892, + "theoretical_loss": 3.8176741789607886, + "tokens_seen": 682491904 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040059177532597795, + "loss": 1.3678, + "theoretical_loss": 3.8176370689422767, + "tokens_seen": 682557440 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040058174523570713, + "loss": 1.4807, + "theoretical_loss": 3.8175999634842936, + "tokens_seen": 682622976 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004005717151454363, + "loss": 1.3935, + "theoretical_loss": 3.8175628625858407, + "tokens_seen": 682688512 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004005616850551655, + "loss": 1.3903, + "theoretical_loss": 3.8175257662459208, + "tokens_seen": 682754048 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040055165496489473, + "loss": 1.3362, + "theoretical_loss": 3.8174886744635352, + "tokens_seen": 682819584 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040054162487462386, + "loss": 1.4632, + "theoretical_loss": 3.8174515872376875, + "tokens_seen": 682885120 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004005315947843531, + "loss": 1.4408, + "theoretical_loss": 3.81741450456738, + "tokens_seen": 682950656 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004005215646940822, + "loss": 1.2688, + "theoretical_loss": 3.817377426451617, + "tokens_seen": 683016192 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040051153460381146, + "loss": 1.4107, + "theoretical_loss": 3.817340352889402, + "tokens_seen": 683081728 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040050150451354064, + "loss": 1.4287, + "theoretical_loss": 3.8173032838797383, + "tokens_seen": 683147264 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004004914744232698, + "loss": 1.3908, + "theoretical_loss": 3.8172662194216302, + "tokens_seen": 683212800 + }, + { + "epoch": 0.21, + "learning_rate": 0.000400481444332999, + "loss": 1.4063, + "theoretical_loss": 3.817229159514083, + "tokens_seen": 683278336 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004004714142427282, + "loss": 1.3559, + "theoretical_loss": 3.817192104156101, + "tokens_seen": 683343872 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040046138415245736, + "loss": 1.4759, + "theoretical_loss": 3.8171550533466903, + "tokens_seen": 683409408 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004004513540621866, + "loss": 1.5002, + "theoretical_loss": 3.817118007084856, + "tokens_seen": 683474944 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004004413239719157, + "loss": 1.3519, + "theoretical_loss": 3.8170809653696036, + "tokens_seen": 683540480 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040043129388164496, + "loss": 1.4303, + "theoretical_loss": 3.817043928199939, + "tokens_seen": 683606016 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004004212637913741, + "loss": 1.3399, + "theoretical_loss": 3.8170068955748695, + "tokens_seen": 683671552 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004004112337011033, + "loss": 1.4805, + "theoretical_loss": 3.8169698674934023, + "tokens_seen": 683737088 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004004012036108325, + "loss": 1.2959, + "theoretical_loss": 3.816932843954543, + "tokens_seen": 683802624 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004003911735205617, + "loss": 1.455, + "theoretical_loss": 3.8168958249573004, + "tokens_seen": 683868160 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040038114343029087, + "loss": 1.3817, + "theoretical_loss": 3.8168588105006815, + "tokens_seen": 683933696 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004003711133400201, + "loss": 1.432, + "theoretical_loss": 3.816821800583695, + "tokens_seen": 683999232 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040036108324974923, + "loss": 1.3723, + "theoretical_loss": 3.816784795205348, + "tokens_seen": 684064768 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040035105315947846, + "loss": 1.4262, + "theoretical_loss": 3.816747794364651, + "tokens_seen": 684130304 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004003410230692076, + "loss": 1.3712, + "theoretical_loss": 3.816710798060611, + "tokens_seen": 684195840 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004003309929789368, + "loss": 1.3979, + "theoretical_loss": 3.816673806292239, + "tokens_seen": 684261376 + }, + { + "epoch": 0.21, + "learning_rate": 0.000400320962888666, + "loss": 1.3314, + "theoretical_loss": 3.8166368190585436, + "tokens_seen": 684326912 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004003109327983952, + "loss": 1.3954, + "theoretical_loss": 3.816599836358535, + "tokens_seen": 684392448 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040030090270812437, + "loss": 1.469, + "theoretical_loss": 3.816562858191223, + "tokens_seen": 684457984 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040029087261785355, + "loss": 1.3326, + "theoretical_loss": 3.8165258845556194, + "tokens_seen": 684523520 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040028084252758273, + "loss": 1.3794, + "theoretical_loss": 3.8164889154507335, + "tokens_seen": 684589056 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040027081243731197, + "loss": 1.3997, + "theoretical_loss": 3.816451950875578, + "tokens_seen": 684654592 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004002607823470411, + "loss": 1.4078, + "theoretical_loss": 3.8164149908291627, + "tokens_seen": 684720128 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040025075225677033, + "loss": 1.3812, + "theoretical_loss": 3.8163780353105006, + "tokens_seen": 684785664 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 232685, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.453451156616211, + "objective/train/theoretical_loss": 3.816341084318604, + "objective/train/tokens_used": 684851200, + "theoretical_loss": 3.816341084318604, + "tokens_seen": 684851200 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040024072216649946, + "loss": 1.4279, + "theoretical_loss": 3.816341084318604, + "tokens_seen": 684851200 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004002306920762287, + "loss": 1.4297, + "theoretical_loss": 3.816304137852484, + "tokens_seen": 684916736 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040022066198595787, + "loss": 1.3247, + "theoretical_loss": 3.8162671959111543, + "tokens_seen": 684982272 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040021063189568705, + "loss": 1.3485, + "theoretical_loss": 3.8162302584936274, + "tokens_seen": 685047808 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040020060180541623, + "loss": 1.3918, + "theoretical_loss": 3.8161933255989173, + "tokens_seen": 685113344 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040019057171514547, + "loss": 1.5459, + "theoretical_loss": 3.8161563972260373, + "tokens_seen": 685178880 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004001805416248746, + "loss": 1.468, + "theoretical_loss": 3.816119473374001, + "tokens_seen": 685244416 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040017051153460383, + "loss": 1.3626, + "theoretical_loss": 3.816082554041823, + "tokens_seen": 685309952 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040016048144433296, + "loss": 1.4682, + "theoretical_loss": 3.816045639228518, + "tokens_seen": 685375488 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004001504513540622, + "loss": 1.5004, + "theoretical_loss": 3.8160087289331006, + "tokens_seen": 685441024 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040014042126379143, + "loss": 1.4691, + "theoretical_loss": 3.815971823154586, + "tokens_seen": 685506560 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040013039117352056, + "loss": 1.4762, + "theoretical_loss": 3.81593492189199, + "tokens_seen": 685572096 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004001203610832498, + "loss": 1.395, + "theoretical_loss": 3.8158980251443277, + "tokens_seen": 685637632 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004001103309929789, + "loss": 1.4936, + "theoretical_loss": 3.8158611329106162, + "tokens_seen": 685703168 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040010030090270815, + "loss": 1.48, + "theoretical_loss": 3.815824245189871, + "tokens_seen": 685768704 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040009027081243733, + "loss": 1.4976, + "theoretical_loss": 3.81578736198111, + "tokens_seen": 685834240 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004000802407221665, + "loss": 1.3688, + "theoretical_loss": 3.815750483283349, + "tokens_seen": 685899776 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004000702106318957, + "loss": 1.4005, + "theoretical_loss": 3.8157136090956056, + "tokens_seen": 685965312 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040006018054162493, + "loss": 1.3939, + "theoretical_loss": 3.8156767394168982, + "tokens_seen": 686030848 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040005015045135406, + "loss": 1.3508, + "theoretical_loss": 3.8156398742462443, + "tokens_seen": 686096384 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004000401203610833, + "loss": 1.3352, + "theoretical_loss": 3.8156030135826615, + "tokens_seen": 686161920 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004000300902708124, + "loss": 1.4011, + "theoretical_loss": 3.81556615742517, + "tokens_seen": 686227456 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040002006018054166, + "loss": 1.4414, + "theoretical_loss": 3.815529305772787, + "tokens_seen": 686292992 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040001003009027084, + "loss": 1.5418, + "theoretical_loss": 3.815492458624532, + "tokens_seen": 686358528 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004, + "loss": 1.4135, + "theoretical_loss": 3.8154556159794257, + "tokens_seen": 686424064 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003999899699097292, + "loss": 1.4573, + "theoretical_loss": 3.815418777836487, + "tokens_seen": 686489600 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003999799398194584, + "loss": 1.3952, + "theoretical_loss": 3.8153819441947356, + "tokens_seen": 686555136 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039996990972918756, + "loss": 1.3966, + "theoretical_loss": 3.815345115053193, + "tokens_seen": 686620672 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003999598796389168, + "loss": 1.4254, + "theoretical_loss": 3.815308290410879, + "tokens_seen": 686686208 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003999498495486459, + "loss": 1.3001, + "theoretical_loss": 3.815271470266815, + "tokens_seen": 686751744 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039993981945837516, + "loss": 1.3514, + "theoretical_loss": 3.8152346546200224, + "tokens_seen": 686817280 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003999297893681043, + "loss": 1.5396, + "theoretical_loss": 3.815197843469523, + "tokens_seen": 686882816 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003999197592778335, + "loss": 1.5412, + "theoretical_loss": 3.815161036814338, + "tokens_seen": 686948352 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003999097291875627, + "loss": 1.4294, + "theoretical_loss": 3.8151242346534904, + "tokens_seen": 687013888 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003998996990972919, + "loss": 1.4274, + "theoretical_loss": 3.8150874369860026, + "tokens_seen": 687079424 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039988966900702107, + "loss": 1.4819, + "theoretical_loss": 3.8150506438108973, + "tokens_seen": 687144960 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003998796389167503, + "loss": 1.3794, + "theoretical_loss": 3.815013855127198, + "tokens_seen": 687210496 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039986960882647943, + "loss": 1.433, + "theoretical_loss": 3.814977070933928, + "tokens_seen": 687276032 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039985957873620866, + "loss": 1.4603, + "theoretical_loss": 3.814940291230111, + "tokens_seen": 687341568 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003998495486459378, + "loss": 1.4022, + "theoretical_loss": 3.8149035160147706, + "tokens_seen": 687407104 + }, + { + "epoch": 0.21, + "learning_rate": 0.000399839518555667, + "loss": 1.4777, + "theoretical_loss": 3.8148667452869325, + "tokens_seen": 687472640 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003998294884653962, + "loss": 1.3895, + "theoretical_loss": 3.81482997904562, + "tokens_seen": 687538176 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003998194583751254, + "loss": 1.4831, + "theoretical_loss": 3.8147932172898598, + "tokens_seen": 687603712 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039980942828485457, + "loss": 1.4802, + "theoretical_loss": 3.814756460018675, + "tokens_seen": 687669248 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039979939819458375, + "loss": 1.4378, + "theoretical_loss": 3.814719707231093, + "tokens_seen": 687734784 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039978936810431293, + "loss": 1.387, + "theoretical_loss": 3.814682958926139, + "tokens_seen": 687800320 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039977933801404217, + "loss": 1.4098, + "theoretical_loss": 3.8146462151028393, + "tokens_seen": 687865856 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003997693079237713, + "loss": 1.496, + "theoretical_loss": 3.8146094757602205, + "tokens_seen": 687931392 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039975927783350053, + "loss": 1.5616, + "theoretical_loss": 3.814572740897309, + "tokens_seen": 687996928 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039974924774322966, + "loss": 1.4077, + "theoretical_loss": 3.814536010513133, + "tokens_seen": 688062464 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 233890, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.452341079711914, + "objective/train/theoretical_loss": 3.8144992846067187, + "objective/train/tokens_used": 688128000, + "theoretical_loss": 3.8144992846067187, + "tokens_seen": 688128000 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003997392176529589, + "loss": 1.4297, + "theoretical_loss": 3.8144992846067187, + "tokens_seen": 688128000 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039972918756268807, + "loss": 1.447, + "theoretical_loss": 3.8144625631770945, + "tokens_seen": 688193536 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039971915747241725, + "loss": 1.4231, + "theoretical_loss": 3.8144258462232887, + "tokens_seen": 688259072 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039970912738214643, + "loss": 1.4521, + "theoretical_loss": 3.8143891337443288, + "tokens_seen": 688324608 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039969909729187567, + "loss": 1.4232, + "theoretical_loss": 3.8143524257392443, + "tokens_seen": 688390144 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003996890672016048, + "loss": 1.5113, + "theoretical_loss": 3.814315722207064, + "tokens_seen": 688455680 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039967903711133403, + "loss": 1.4359, + "theoretical_loss": 3.814279023146817, + "tokens_seen": 688521216 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039966900702106316, + "loss": 1.504, + "theoretical_loss": 3.8142423285575324, + "tokens_seen": 688586752 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003996589769307924, + "loss": 1.4109, + "theoretical_loss": 3.814205638438241, + "tokens_seen": 688652288 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003996489468405216, + "loss": 1.4108, + "theoretical_loss": 3.814168952787972, + "tokens_seen": 688717824 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039963891675025076, + "loss": 1.4634, + "theoretical_loss": 3.8141322716057564, + "tokens_seen": 688783360 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039962888665997994, + "loss": 1.5034, + "theoretical_loss": 3.814095594890625, + "tokens_seen": 688848896 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003996188565697091, + "loss": 1.4114, + "theoretical_loss": 3.8140589226416095, + "tokens_seen": 688914432 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003996088264794383, + "loss": 1.5117, + "theoretical_loss": 3.81402225485774, + "tokens_seen": 688979968 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039959879638916754, + "loss": 1.4134, + "theoretical_loss": 3.8139855915380485, + "tokens_seen": 689045504 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039958876629889666, + "loss": 1.3088, + "theoretical_loss": 3.813948932681568, + "tokens_seen": 689111040 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003995787362086259, + "loss": 1.32, + "theoretical_loss": 3.8139122782873294, + "tokens_seen": 689176576 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003995687061183551, + "loss": 1.43, + "theoretical_loss": 3.813875628354366, + "tokens_seen": 689242112 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039955867602808426, + "loss": 1.3834, + "theoretical_loss": 3.8138389828817116, + "tokens_seen": 689307648 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039954864593781344, + "loss": 1.4495, + "theoretical_loss": 3.8138023418683975, + "tokens_seen": 689373184 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003995386158475426, + "loss": 1.4601, + "theoretical_loss": 3.8137657053134584, + "tokens_seen": 689438720 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003995285857572718, + "loss": 1.4569, + "theoretical_loss": 3.813729073215928, + "tokens_seen": 689504256 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039951855566700104, + "loss": 1.5282, + "theoretical_loss": 3.8136924455748398, + "tokens_seen": 689569792 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039950852557673017, + "loss": 1.3894, + "theoretical_loss": 3.813655822389229, + "tokens_seen": 689635328 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003994984954864594, + "loss": 1.469, + "theoretical_loss": 3.8136192036581296, + "tokens_seen": 689700864 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039948846539618853, + "loss": 1.3862, + "theoretical_loss": 3.813582589380577, + "tokens_seen": 689766400 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039947843530591776, + "loss": 1.3801, + "theoretical_loss": 3.8135459795556064, + "tokens_seen": 689831936 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039946840521564694, + "loss": 1.4455, + "theoretical_loss": 3.8135093741822534, + "tokens_seen": 689897472 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003994583751253761, + "loss": 1.5038, + "theoretical_loss": 3.813472773259554, + "tokens_seen": 689963008 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003994483450351053, + "loss": 1.3016, + "theoretical_loss": 3.8134361767865443, + "tokens_seen": 690028544 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003994383149448345, + "loss": 1.5671, + "theoretical_loss": 3.8133995847622613, + "tokens_seen": 690094080 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039942828485456367, + "loss": 1.3301, + "theoretical_loss": 3.8133629971857403, + "tokens_seen": 690159616 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003994182547642929, + "loss": 1.4419, + "theoretical_loss": 3.8133264140560197, + "tokens_seen": 690225152 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039940822467402203, + "loss": 1.3841, + "theoretical_loss": 3.813289835372137, + "tokens_seen": 690290688 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039939819458375127, + "loss": 1.4424, + "theoretical_loss": 3.81325326113313, + "tokens_seen": 690356224 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003993881644934805, + "loss": 1.4869, + "theoretical_loss": 3.8132166913380354, + "tokens_seen": 690421760 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039937813440320963, + "loss": 1.4635, + "theoretical_loss": 3.8131801259858924, + "tokens_seen": 690487296 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039936810431293886, + "loss": 1.3209, + "theoretical_loss": 3.8131435650757397, + "tokens_seen": 690552832 + }, + { + "epoch": 0.21, + "learning_rate": 0.000399358074222668, + "loss": 1.429, + "theoretical_loss": 3.8131070086066154, + "tokens_seen": 690618368 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003993480441323972, + "loss": 1.3755, + "theoretical_loss": 3.8130704565775604, + "tokens_seen": 690683904 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003993380140421264, + "loss": 1.3139, + "theoretical_loss": 3.8130339089876126, + "tokens_seen": 690749440 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003993279839518556, + "loss": 1.4204, + "theoretical_loss": 3.8129973658358125, + "tokens_seen": 690814976 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039931795386158477, + "loss": 1.3152, + "theoretical_loss": 3.8129608271211994, + "tokens_seen": 690880512 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039930792377131395, + "loss": 1.422, + "theoretical_loss": 3.8129242928428146, + "tokens_seen": 690946048 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039929789368104313, + "loss": 1.4607, + "theoretical_loss": 3.812887762999699, + "tokens_seen": 691011584 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039928786359077237, + "loss": 1.4444, + "theoretical_loss": 3.8128512375908925, + "tokens_seen": 691077120 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003992778335005015, + "loss": 1.3724, + "theoretical_loss": 3.8128147166154376, + "tokens_seen": 691142656 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039926780341023073, + "loss": 1.4281, + "theoretical_loss": 3.8127782000723744, + "tokens_seen": 691208192 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039925777331995986, + "loss": 1.4361, + "theoretical_loss": 3.812741687960746, + "tokens_seen": 691273728 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003992477432296891, + "loss": 1.393, + "theoretical_loss": 3.8127051802795946, + "tokens_seen": 691339264 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 234824, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4199223518371582, + "objective/train/theoretical_loss": 3.812668677027962, + "objective/train/tokens_used": 691404800, + "theoretical_loss": 3.812668677027962, + "tokens_seen": 691404800 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039923771313941827, + "loss": 1.3806, + "theoretical_loss": 3.812668677027962, + "tokens_seen": 691404800 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039922768304914745, + "loss": 1.4014, + "theoretical_loss": 3.8126321782048915, + "tokens_seen": 691470336 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039921765295887663, + "loss": 1.4504, + "theoretical_loss": 3.8125956838094255, + "tokens_seen": 691535872 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039920762286860587, + "loss": 1.2865, + "theoretical_loss": 3.812559193840608, + "tokens_seen": 691601408 + }, + { + "epoch": 0.21, + "learning_rate": 0.000399197592778335, + "loss": 1.4414, + "theoretical_loss": 3.8125227082974824, + "tokens_seen": 691666944 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039918756268806423, + "loss": 1.3983, + "theoretical_loss": 3.812486227179093, + "tokens_seen": 691732480 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039917753259779336, + "loss": 1.399, + "theoretical_loss": 3.812449750484484, + "tokens_seen": 691798016 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003991675025075226, + "loss": 1.4602, + "theoretical_loss": 3.8124132782126994, + "tokens_seen": 691863552 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003991574724172518, + "loss": 1.4695, + "theoretical_loss": 3.812376810362785, + "tokens_seen": 691929088 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039914744232698096, + "loss": 1.3491, + "theoretical_loss": 3.8123403469337847, + "tokens_seen": 691994624 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039913741223671014, + "loss": 1.4684, + "theoretical_loss": 3.812303887924745, + "tokens_seen": 692060160 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003991273821464393, + "loss": 1.4204, + "theoretical_loss": 3.8122674333347115, + "tokens_seen": 692125696 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003991173520561685, + "loss": 1.3664, + "theoretical_loss": 3.8122309831627295, + "tokens_seen": 692191232 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039910732196589774, + "loss": 1.6205, + "theoretical_loss": 3.8121945374078465, + "tokens_seen": 692256768 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039909729187562686, + "loss": 1.3008, + "theoretical_loss": 3.8121580960691084, + "tokens_seen": 692322304 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003990872617853561, + "loss": 1.2905, + "theoretical_loss": 3.812121659145562, + "tokens_seen": 692387840 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003990772316950853, + "loss": 1.3651, + "theoretical_loss": 3.8120852266362553, + "tokens_seen": 692453376 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039906720160481446, + "loss": 1.2849, + "theoretical_loss": 3.8120487985402347, + "tokens_seen": 692518912 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039905717151454364, + "loss": 1.4669, + "theoretical_loss": 3.8120123748565486, + "tokens_seen": 692584448 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003990471414242728, + "loss": 1.3246, + "theoretical_loss": 3.811975955584246, + "tokens_seen": 692649984 + }, + { + "epoch": 0.21, + "learning_rate": 0.000399037111334002, + "loss": 1.3442, + "theoretical_loss": 3.8119395407223737, + "tokens_seen": 692715520 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039902708124373124, + "loss": 1.3604, + "theoretical_loss": 3.8119031302699815, + "tokens_seen": 692781056 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039901705115346037, + "loss": 1.2831, + "theoretical_loss": 3.8118667242261175, + "tokens_seen": 692846592 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003990070210631896, + "loss": 1.3587, + "theoretical_loss": 3.8118303225898327, + "tokens_seen": 692912128 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039899699097291873, + "loss": 1.351, + "theoretical_loss": 3.8117939253601745, + "tokens_seen": 692977664 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039898696088264796, + "loss": 1.444, + "theoretical_loss": 3.811757532536194, + "tokens_seen": 693043200 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039897693079237714, + "loss": 1.3405, + "theoretical_loss": 3.811721144116942, + "tokens_seen": 693108736 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003989669007021063, + "loss": 1.4164, + "theoretical_loss": 3.8116847601014676, + "tokens_seen": 693174272 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003989568706118355, + "loss": 1.3994, + "theoretical_loss": 3.811648380488822, + "tokens_seen": 693239808 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003989468405215647, + "loss": 1.503, + "theoretical_loss": 3.811612005278057, + "tokens_seen": 693305344 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039893681043129387, + "loss": 1.5199, + "theoretical_loss": 3.8115756344682232, + "tokens_seen": 693370880 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003989267803410231, + "loss": 1.4761, + "theoretical_loss": 3.811539268058372, + "tokens_seen": 693436416 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039891675025075223, + "loss": 1.3989, + "theoretical_loss": 3.811502906047556, + "tokens_seen": 693501952 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039890672016048147, + "loss": 1.4253, + "theoretical_loss": 3.8114665484348276, + "tokens_seen": 693567488 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039889669007021065, + "loss": 1.4313, + "theoretical_loss": 3.8114301952192386, + "tokens_seen": 693633024 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039888665997993983, + "loss": 1.4515, + "theoretical_loss": 3.8113938463998425, + "tokens_seen": 693698560 + }, + { + "epoch": 0.21, + "learning_rate": 0.000398876629889669, + "loss": 1.3322, + "theoretical_loss": 3.811357501975692, + "tokens_seen": 693764096 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003988665997993982, + "loss": 1.4543, + "theoretical_loss": 3.8113211619458403, + "tokens_seen": 693829632 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039885656970912737, + "loss": 1.4254, + "theoretical_loss": 3.811284826309342, + "tokens_seen": 693895168 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003988465396188566, + "loss": 1.4448, + "theoretical_loss": 3.81124849506525, + "tokens_seen": 693960704 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039883650952858573, + "loss": 1.3069, + "theoretical_loss": 3.811212168212619, + "tokens_seen": 694026240 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039882647943831497, + "loss": 1.3031, + "theoretical_loss": 3.8111758457505047, + "tokens_seen": 694091776 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003988164493480441, + "loss": 1.407, + "theoretical_loss": 3.8111395276779607, + "tokens_seen": 694157312 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039880641925777333, + "loss": 1.5263, + "theoretical_loss": 3.8111032139940417, + "tokens_seen": 694222848 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003987963891675025, + "loss": 1.4303, + "theoretical_loss": 3.811066904697805, + "tokens_seen": 694288384 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003987863590772317, + "loss": 1.3856, + "theoretical_loss": 3.811030599788304, + "tokens_seen": 694353920 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003987763289869609, + "loss": 1.4261, + "theoretical_loss": 3.810994299264597, + "tokens_seen": 694419456 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039876629889669006, + "loss": 1.3342, + "theoretical_loss": 3.810958003125739, + "tokens_seen": 694484992 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039875626880641924, + "loss": 1.4205, + "theoretical_loss": 3.8109217113707876, + "tokens_seen": 694550528 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039874623871614847, + "loss": 1.4085, + "theoretical_loss": 3.8108854239987986, + "tokens_seen": 694616064 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 235950, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3620924949645996, + "objective/train/theoretical_loss": 3.81084914100883, + "objective/train/tokens_used": 694681600, + "theoretical_loss": 3.81084914100883, + "tokens_seen": 694681600 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003987362086258776, + "loss": 1.3774, + "theoretical_loss": 3.81084914100883, + "tokens_seen": 694681600 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039872617853560683, + "loss": 1.3813, + "theoretical_loss": 3.810812862399939, + "tokens_seen": 694747136 + }, + { + "epoch": 0.21, + "learning_rate": 0.000398716148445336, + "loss": 1.4376, + "theoretical_loss": 3.8107765881711835, + "tokens_seen": 694812672 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003987061183550652, + "loss": 1.3082, + "theoretical_loss": 3.8107403183216215, + "tokens_seen": 694878208 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003986960882647944, + "loss": 1.481, + "theoretical_loss": 3.810704052850311, + "tokens_seen": 694943744 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039868605817452356, + "loss": 1.5313, + "theoretical_loss": 3.8106677917563117, + "tokens_seen": 695009280 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039867602808425274, + "loss": 1.3784, + "theoretical_loss": 3.810631535038682, + "tokens_seen": 695074816 + }, + { + "epoch": 0.21, + "learning_rate": 0.000398665997993982, + "loss": 1.4158, + "theoretical_loss": 3.810595282696481, + "tokens_seen": 695140352 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003986559679037111, + "loss": 1.4148, + "theoretical_loss": 3.810559034728769, + "tokens_seen": 695205888 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039864593781344034, + "loss": 1.4184, + "theoretical_loss": 3.8105227911346042, + "tokens_seen": 695271424 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003986359077231695, + "loss": 1.4416, + "theoretical_loss": 3.810486551913048, + "tokens_seen": 695336960 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003986258776328987, + "loss": 1.4038, + "theoretical_loss": 3.810450317063161, + "tokens_seen": 695402496 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039861584754262794, + "loss": 1.363, + "theoretical_loss": 3.8104140865840033, + "tokens_seen": 695468032 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039860581745235706, + "loss": 1.4684, + "theoretical_loss": 3.810377860474636, + "tokens_seen": 695533568 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003985957873620863, + "loss": 1.3676, + "theoretical_loss": 3.8103416387341205, + "tokens_seen": 695599104 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003985857572718155, + "loss": 1.4388, + "theoretical_loss": 3.8103054213615186, + "tokens_seen": 695664640 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039857572718154466, + "loss": 1.3096, + "theoretical_loss": 3.8102692083558916, + "tokens_seen": 695730176 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039856569709127384, + "loss": 1.3415, + "theoretical_loss": 3.810232999716302, + "tokens_seen": 695795712 + }, + { + "epoch": 0.21, + "learning_rate": 0.000398555667001003, + "loss": 1.3856, + "theoretical_loss": 3.8101967954418123, + "tokens_seen": 695861248 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003985456369107322, + "loss": 1.475, + "theoretical_loss": 3.8101605955314857, + "tokens_seen": 695926784 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039853560682046144, + "loss": 1.4434, + "theoretical_loss": 3.8101243999843843, + "tokens_seen": 695992320 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039852557673019057, + "loss": 1.327, + "theoretical_loss": 3.8100882087995718, + "tokens_seen": 696057856 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003985155466399198, + "loss": 1.3736, + "theoretical_loss": 3.810052021976112, + "tokens_seen": 696123392 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039850551654964893, + "loss": 1.4393, + "theoretical_loss": 3.8100158395130683, + "tokens_seen": 696188928 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039849548645937816, + "loss": 1.4021, + "theoretical_loss": 3.809979661409505, + "tokens_seen": 696254464 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039848545636910734, + "loss": 1.4432, + "theoretical_loss": 3.809943487664488, + "tokens_seen": 696320000 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003984754262788365, + "loss": 1.5024, + "theoretical_loss": 3.8099073182770797, + "tokens_seen": 696385536 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003984653961885657, + "loss": 1.3458, + "theoretical_loss": 3.809871153246347, + "tokens_seen": 696451072 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003984553660982949, + "loss": 1.5004, + "theoretical_loss": 3.8098349925713544, + "tokens_seen": 696516608 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039844533600802407, + "loss": 1.3149, + "theoretical_loss": 3.809798836251167, + "tokens_seen": 696582144 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003984353059177533, + "loss": 1.3394, + "theoretical_loss": 3.809762684284852, + "tokens_seen": 696647680 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039842527582748243, + "loss": 1.364, + "theoretical_loss": 3.809726536671475, + "tokens_seen": 696713216 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039841524573721167, + "loss": 1.4338, + "theoretical_loss": 3.8096903934101025, + "tokens_seen": 696778752 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039840521564694085, + "loss": 1.417, + "theoretical_loss": 3.8096542544998018, + "tokens_seen": 696844288 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039839518555667003, + "loss": 1.5427, + "theoretical_loss": 3.8096181199396386, + "tokens_seen": 696909824 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003983851554663992, + "loss": 1.3967, + "theoretical_loss": 3.8095819897286813, + "tokens_seen": 696975360 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003983751253761284, + "loss": 1.5553, + "theoretical_loss": 3.8095458638659974, + "tokens_seen": 697040896 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039836509528585757, + "loss": 1.4301, + "theoretical_loss": 3.809509742350655, + "tokens_seen": 697106432 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003983550651955868, + "loss": 1.48, + "theoretical_loss": 3.809473625181722, + "tokens_seen": 697171968 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039834503510531593, + "loss": 1.3682, + "theoretical_loss": 3.809437512358267, + "tokens_seen": 697237504 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039833500501504517, + "loss": 1.3861, + "theoretical_loss": 3.809401403879359, + "tokens_seen": 697303040 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003983249749247743, + "loss": 1.4128, + "theoretical_loss": 3.8093652997440666, + "tokens_seen": 697368576 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039831494483450353, + "loss": 1.4919, + "theoretical_loss": 3.80932919995146, + "tokens_seen": 697434112 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003983049147442327, + "loss": 1.4028, + "theoretical_loss": 3.8092931045006075, + "tokens_seen": 697499648 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003982948846539619, + "loss": 1.3954, + "theoretical_loss": 3.8092570133905808, + "tokens_seen": 697565184 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003982848545636911, + "loss": 1.3945, + "theoretical_loss": 3.809220926620448, + "tokens_seen": 697630720 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039827482447342026, + "loss": 1.4207, + "theoretical_loss": 3.809184844189282, + "tokens_seen": 697696256 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039826479438314944, + "loss": 1.3653, + "theoretical_loss": 3.8091487660961514, + "tokens_seen": 697761792 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039825476429287867, + "loss": 1.5015, + "theoretical_loss": 3.809112692340129, + "tokens_seen": 697827328 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003982447342026078, + "loss": 1.4662, + "theoretical_loss": 3.8090766229202853, + "tokens_seen": 697892864 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 236749, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.347040057182312, + "objective/train/theoretical_loss": 3.8090405578356927, + "objective/train/tokens_used": 697958400, + "theoretical_loss": 3.8090405578356927, + "tokens_seen": 697958400 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039823470411233703, + "loss": 1.4002, + "theoretical_loss": 3.8090405578356927, + "tokens_seen": 697958400 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003982246740220662, + "loss": 1.3612, + "theoretical_loss": 3.8090044970854215, + "tokens_seen": 698023936 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003982146439317954, + "loss": 1.5231, + "theoretical_loss": 3.8089684406685453, + "tokens_seen": 698089472 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003982046138415246, + "loss": 1.3939, + "theoretical_loss": 3.808932388584137, + "tokens_seen": 698155008 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039819458375125376, + "loss": 1.4471, + "theoretical_loss": 3.808896340831268, + "tokens_seen": 698220544 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039818455366098294, + "loss": 1.3467, + "theoretical_loss": 3.808860297409012, + "tokens_seen": 698286080 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003981745235707122, + "loss": 1.463, + "theoretical_loss": 3.8088242583164433, + "tokens_seen": 698351616 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003981644934804413, + "loss": 1.4208, + "theoretical_loss": 3.808788223552634, + "tokens_seen": 698417152 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039815446339017054, + "loss": 1.2881, + "theoretical_loss": 3.8087521931166592, + "tokens_seen": 698482688 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039814443329989966, + "loss": 1.4968, + "theoretical_loss": 3.8087161670075926, + "tokens_seen": 698548224 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003981344032096289, + "loss": 1.3971, + "theoretical_loss": 3.8086801452245087, + "tokens_seen": 698613760 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003981243731193581, + "loss": 1.3494, + "theoretical_loss": 3.808644127766483, + "tokens_seen": 698679296 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039811434302908726, + "loss": 1.3945, + "theoretical_loss": 3.808608114632589, + "tokens_seen": 698744832 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039810431293881644, + "loss": 1.2914, + "theoretical_loss": 3.808572105821904, + "tokens_seen": 698810368 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003980942828485457, + "loss": 1.4244, + "theoretical_loss": 3.8085361013335026, + "tokens_seen": 698875904 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003980842527582748, + "loss": 1.4478, + "theoretical_loss": 3.8085001011664605, + "tokens_seen": 698941440 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039807422266800404, + "loss": 1.3703, + "theoretical_loss": 3.8084641053198545, + "tokens_seen": 699006976 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039806419257773317, + "loss": 1.3798, + "theoretical_loss": 3.808428113792761, + "tokens_seen": 699072512 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003980541624874624, + "loss": 1.414, + "theoretical_loss": 3.8083921265842564, + "tokens_seen": 699138048 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003980441323971916, + "loss": 1.4085, + "theoretical_loss": 3.8083561436934183, + "tokens_seen": 699203584 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039803410230692077, + "loss": 1.4458, + "theoretical_loss": 3.8083201651193237, + "tokens_seen": 699269120 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039802407221664995, + "loss": 1.322, + "theoretical_loss": 3.80828419086105, + "tokens_seen": 699334656 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039801404212637913, + "loss": 1.4769, + "theoretical_loss": 3.808248220917676, + "tokens_seen": 699400192 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003980040120361083, + "loss": 1.3503, + "theoretical_loss": 3.8082122552882796, + "tokens_seen": 699465728 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039799398194583754, + "loss": 1.4159, + "theoretical_loss": 3.808176293971939, + "tokens_seen": 699531264 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039798395185556667, + "loss": 1.4763, + "theoretical_loss": 3.8081403369677322, + "tokens_seen": 699596800 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003979739217652959, + "loss": 1.4775, + "theoretical_loss": 3.8081043842747397, + "tokens_seen": 699662336 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039796389167502503, + "loss": 1.4532, + "theoretical_loss": 3.8080684358920402, + "tokens_seen": 699727872 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039795386158475427, + "loss": 1.4152, + "theoretical_loss": 3.8080324918187136, + "tokens_seen": 699793408 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039794383149448345, + "loss": 1.5256, + "theoretical_loss": 3.8079965520538392, + "tokens_seen": 699858944 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039793380140421263, + "loss": 1.4243, + "theoretical_loss": 3.8079606165964974, + "tokens_seen": 699924480 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003979237713139418, + "loss": 1.4417, + "theoretical_loss": 3.8079246854457693, + "tokens_seen": 699990016 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039791374122367105, + "loss": 1.3978, + "theoretical_loss": 3.807888758600735, + "tokens_seen": 700055552 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003979037111334002, + "loss": 1.6061, + "theoretical_loss": 3.8078528360604755, + "tokens_seen": 700121088 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003978936810431294, + "loss": 1.2795, + "theoretical_loss": 3.8078169178240726, + "tokens_seen": 700186624 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003978836509528586, + "loss": 1.414, + "theoretical_loss": 3.8077810038906073, + "tokens_seen": 700252160 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039787362086258777, + "loss": 1.3598, + "theoretical_loss": 3.807745094259162, + "tokens_seen": 700317696 + }, + { + "epoch": 0.21, + "learning_rate": 0.000397863590772317, + "loss": 1.461, + "theoretical_loss": 3.807709188928819, + "tokens_seen": 700383232 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039785356068204613, + "loss": 1.3232, + "theoretical_loss": 3.80767328789866, + "tokens_seen": 700448768 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039784353059177537, + "loss": 1.3762, + "theoretical_loss": 3.807637391167768, + "tokens_seen": 700514304 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003978335005015045, + "loss": 1.3459, + "theoretical_loss": 3.8076014987352265, + "tokens_seen": 700579840 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039782347041123373, + "loss": 1.4284, + "theoretical_loss": 3.8075656106001183, + "tokens_seen": 700645376 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003978134403209629, + "loss": 1.3726, + "theoretical_loss": 3.807529726761527, + "tokens_seen": 700710912 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003978034102306921, + "loss": 1.3797, + "theoretical_loss": 3.807493847218537, + "tokens_seen": 700776448 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003977933801404213, + "loss": 1.4091, + "theoretical_loss": 3.8074579719702317, + "tokens_seen": 700841984 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039778335005015046, + "loss": 1.3572, + "theoretical_loss": 3.8074221010156957, + "tokens_seen": 700907520 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039777331995987964, + "loss": 1.3609, + "theoretical_loss": 3.8073862343540137, + "tokens_seen": 700973056 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039776328986960887, + "loss": 1.4371, + "theoretical_loss": 3.807350371984271, + "tokens_seen": 701038592 + }, + { + "epoch": 0.21, + "learning_rate": 0.000397753259779338, + "loss": 1.4403, + "theoretical_loss": 3.8073145139055526, + "tokens_seen": 701104128 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039774322968906723, + "loss": 1.5327, + "theoretical_loss": 3.8072786601169435, + "tokens_seen": 701169664 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 238109, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4006081819534302, + "objective/train/theoretical_loss": 3.807242810617531, + "objective/train/tokens_used": 701235200, + "theoretical_loss": 3.807242810617531, + "tokens_seen": 701235200 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003977331995987964, + "loss": 1.3973, + "theoretical_loss": 3.807242810617531, + "tokens_seen": 701235200 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003977231695085256, + "loss": 1.4298, + "theoretical_loss": 3.8072069654063996, + "tokens_seen": 701300736 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003977131394182548, + "loss": 1.3649, + "theoretical_loss": 3.8071711244826365, + "tokens_seen": 701366272 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039770310932798396, + "loss": 1.2685, + "theoretical_loss": 3.807135287845328, + "tokens_seen": 701431808 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039769307923771314, + "loss": 1.4224, + "theoretical_loss": 3.807099455493561, + "tokens_seen": 701497344 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003976830491474424, + "loss": 1.374, + "theoretical_loss": 3.807063627426424, + "tokens_seen": 701562880 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003976730190571715, + "loss": 1.3847, + "theoretical_loss": 3.8070278036430025, + "tokens_seen": 701628416 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039766298896690074, + "loss": 1.3435, + "theoretical_loss": 3.8069919841423854, + "tokens_seen": 701693952 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039765295887662987, + "loss": 1.3234, + "theoretical_loss": 3.8069561689236604, + "tokens_seen": 701759488 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003976429287863591, + "loss": 1.4347, + "theoretical_loss": 3.8069203579859163, + "tokens_seen": 701825024 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003976328986960883, + "loss": 1.3773, + "theoretical_loss": 3.806884551328242, + "tokens_seen": 701890560 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039762286860581746, + "loss": 1.4343, + "theoretical_loss": 3.806848748949725, + "tokens_seen": 701956096 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039761283851554664, + "loss": 1.3556, + "theoretical_loss": 3.806812950849455, + "tokens_seen": 702021632 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003976028084252759, + "loss": 1.3744, + "theoretical_loss": 3.806777157026522, + "tokens_seen": 702087168 + }, + { + "epoch": 0.21, + "learning_rate": 0.000397592778335005, + "loss": 1.4883, + "theoretical_loss": 3.806741367480016, + "tokens_seen": 702152704 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039758274824473424, + "loss": 1.4026, + "theoretical_loss": 3.806705582209026, + "tokens_seen": 702218240 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039757271815446337, + "loss": 1.4469, + "theoretical_loss": 3.8066698012126423, + "tokens_seen": 702283776 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003975626880641926, + "loss": 1.5057, + "theoretical_loss": 3.8066340244899566, + "tokens_seen": 702349312 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003975526579739218, + "loss": 1.446, + "theoretical_loss": 3.8065982520400583, + "tokens_seen": 702414848 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039754262788365097, + "loss": 1.4543, + "theoretical_loss": 3.80656248386204, + "tokens_seen": 702480384 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039753259779338015, + "loss": 1.4959, + "theoretical_loss": 3.8065267199549924, + "tokens_seen": 702545920 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039752256770310933, + "loss": 1.4127, + "theoretical_loss": 3.8064909603180066, + "tokens_seen": 702611456 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003975125376128385, + "loss": 1.4503, + "theoretical_loss": 3.8064552049501756, + "tokens_seen": 702676992 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039750250752256774, + "loss": 1.4012, + "theoretical_loss": 3.806419453850591, + "tokens_seen": 702742528 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039749247743229687, + "loss": 1.3045, + "theoretical_loss": 3.806383707018345, + "tokens_seen": 702808064 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003974824473420261, + "loss": 1.4334, + "theoretical_loss": 3.806347964452531, + "tokens_seen": 702873600 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039747241725175523, + "loss": 1.3682, + "theoretical_loss": 3.806312226152242, + "tokens_seen": 702939136 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039746238716148447, + "loss": 1.4014, + "theoretical_loss": 3.806276492116571, + "tokens_seen": 703004672 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039745235707121365, + "loss": 1.4408, + "theoretical_loss": 3.806240762344612, + "tokens_seen": 703070208 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039744232698094283, + "loss": 1.3574, + "theoretical_loss": 3.806205036835459, + "tokens_seen": 703135744 + }, + { + "epoch": 0.21, + "learning_rate": 0.000397432296890672, + "loss": 1.4672, + "theoretical_loss": 3.806169315588206, + "tokens_seen": 703201280 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039742226680040125, + "loss": 1.4001, + "theoretical_loss": 3.8061335986019467, + "tokens_seen": 703266816 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003974122367101304, + "loss": 1.3695, + "theoretical_loss": 3.806097885875777, + "tokens_seen": 703332352 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003974022066198596, + "loss": 1.3311, + "theoretical_loss": 3.8060621774087915, + "tokens_seen": 703397888 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039739217652958874, + "loss": 1.4537, + "theoretical_loss": 3.8060264732000846, + "tokens_seen": 703463424 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039738214643931797, + "loss": 1.4145, + "theoretical_loss": 3.8059907732487535, + "tokens_seen": 703528960 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039737211634904715, + "loss": 1.4066, + "theoretical_loss": 3.8059550775538926, + "tokens_seen": 703594496 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039736208625877633, + "loss": 1.3881, + "theoretical_loss": 3.805919386114599, + "tokens_seen": 703660032 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003973520561685055, + "loss": 1.2774, + "theoretical_loss": 3.8058836989299683, + "tokens_seen": 703725568 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003973420260782347, + "loss": 1.3505, + "theoretical_loss": 3.8058480159990977, + "tokens_seen": 703791104 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003973319959879639, + "loss": 1.4921, + "theoretical_loss": 3.805812337321084, + "tokens_seen": 703856640 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003973219658976931, + "loss": 1.3416, + "theoretical_loss": 3.805776662895024, + "tokens_seen": 703922176 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039731193580742224, + "loss": 1.4594, + "theoretical_loss": 3.8057409927200156, + "tokens_seen": 703987712 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003973019057171515, + "loss": 1.4439, + "theoretical_loss": 3.805705326795157, + "tokens_seen": 704053248 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003972918756268806, + "loss": 1.4466, + "theoretical_loss": 3.805669665119545, + "tokens_seen": 704118784 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039728184553660984, + "loss": 1.41, + "theoretical_loss": 3.8056340076922797, + "tokens_seen": 704184320 + }, + { + "epoch": 0.21, + "learning_rate": 0.000397271815446339, + "loss": 1.344, + "theoretical_loss": 3.8055983545124583, + "tokens_seen": 704249856 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003972617853560682, + "loss": 1.4428, + "theoretical_loss": 3.805562705579179, + "tokens_seen": 704315392 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003972517552657974, + "loss": 1.3769, + "theoretical_loss": 3.805527060891543, + "tokens_seen": 704380928 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003972417251755266, + "loss": 1.4232, + "theoretical_loss": 3.8054914204486487, + "tokens_seen": 704446464 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 239273, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.464916467666626, + "objective/train/theoretical_loss": 3.8054557842495957, + "objective/train/tokens_used": 704512000, + "theoretical_loss": 3.8054557842495957, + "tokens_seen": 704512000 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039723169508525574, + "loss": 1.4611, + "theoretical_loss": 3.8054557842495957, + "tokens_seen": 704512000 + }, + { + "epoch": 0.21, + "learning_rate": 0.000397221664994985, + "loss": 1.4299, + "theoretical_loss": 3.8054201522934843, + "tokens_seen": 704577536 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003972116349047141, + "loss": 1.4076, + "theoretical_loss": 3.805384524579414, + "tokens_seen": 704643072 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039720160481444334, + "loss": 1.4536, + "theoretical_loss": 3.805348901106486, + "tokens_seen": 704708608 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003971915747241725, + "loss": 1.4199, + "theoretical_loss": 3.8053132818738002, + "tokens_seen": 704774144 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003971815446339017, + "loss": 1.46, + "theoretical_loss": 3.805277666880459, + "tokens_seen": 704839680 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003971715145436309, + "loss": 1.4616, + "theoretical_loss": 3.8052420561255627, + "tokens_seen": 704905216 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039716148445336007, + "loss": 1.4197, + "theoretical_loss": 3.805206449608214, + "tokens_seen": 704970752 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039715145436308925, + "loss": 1.3994, + "theoretical_loss": 3.8051708473275134, + "tokens_seen": 705036288 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003971414242728185, + "loss": 1.4578, + "theoretical_loss": 3.805135249282564, + "tokens_seen": 705101824 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039713139418254766, + "loss": 1.3023, + "theoretical_loss": 3.8050996554724676, + "tokens_seen": 705167360 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039712136409227684, + "loss": 1.3999, + "theoretical_loss": 3.8050640658963277, + "tokens_seen": 705232896 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003971113340020061, + "loss": 1.4783, + "theoretical_loss": 3.8050284805532466, + "tokens_seen": 705298432 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003971013039117352, + "loss": 1.4864, + "theoretical_loss": 3.804992899442328, + "tokens_seen": 705363968 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039709127382146444, + "loss": 1.4266, + "theoretical_loss": 3.804957322562675, + "tokens_seen": 705429504 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039708124373119357, + "loss": 1.5129, + "theoretical_loss": 3.804921749913392, + "tokens_seen": 705495040 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003970712136409228, + "loss": 1.3602, + "theoretical_loss": 3.804886181493582, + "tokens_seen": 705560576 + }, + { + "epoch": 0.21, + "learning_rate": 0.000397061183550652, + "loss": 1.4111, + "theoretical_loss": 3.8048506173023506, + "tokens_seen": 705626112 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039705115346038117, + "loss": 1.4916, + "theoretical_loss": 3.8048150573388018, + "tokens_seen": 705691648 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039704112337011035, + "loss": 1.4593, + "theoretical_loss": 3.8047795016020407, + "tokens_seen": 705757184 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039703109327983953, + "loss": 1.4276, + "theoretical_loss": 3.804743950091172, + "tokens_seen": 705822720 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003970210631895687, + "loss": 1.4735, + "theoretical_loss": 3.8047084028053013, + "tokens_seen": 705888256 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039701103309929794, + "loss": 1.4154, + "theoretical_loss": 3.8046728597435346, + "tokens_seen": 705953792 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039700100300902707, + "loss": 1.4201, + "theoretical_loss": 3.8046373209049777, + "tokens_seen": 706019328 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003969909729187563, + "loss": 1.3944, + "theoretical_loss": 3.804601786288737, + "tokens_seen": 706084864 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039698094282848543, + "loss": 1.4327, + "theoretical_loss": 3.804566255893919, + "tokens_seen": 706150400 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039697091273821467, + "loss": 1.3822, + "theoretical_loss": 3.8045307297196302, + "tokens_seen": 706215936 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039696088264794385, + "loss": 1.4077, + "theoretical_loss": 3.804495207764978, + "tokens_seen": 706281472 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039695085255767303, + "loss": 1.418, + "theoretical_loss": 3.804459690029069, + "tokens_seen": 706347008 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003969408224674022, + "loss": 1.4074, + "theoretical_loss": 3.804424176511012, + "tokens_seen": 706412544 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039693079237713145, + "loss": 1.3093, + "theoretical_loss": 3.8043886672099148, + "tokens_seen": 706478080 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003969207622868606, + "loss": 1.4071, + "theoretical_loss": 3.804353162124884, + "tokens_seen": 706543616 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003969107321965898, + "loss": 1.4422, + "theoretical_loss": 3.8043176612550296, + "tokens_seen": 706609152 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039690070210631894, + "loss": 1.2877, + "theoretical_loss": 3.80428216459946, + "tokens_seen": 706674688 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039689067201604817, + "loss": 1.3866, + "theoretical_loss": 3.8042466721572836, + "tokens_seen": 706740224 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039688064192577735, + "loss": 1.3003, + "theoretical_loss": 3.80421118392761, + "tokens_seen": 706805760 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039687061183550653, + "loss": 1.4212, + "theoretical_loss": 3.804175699909549, + "tokens_seen": 706871296 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003968605817452357, + "loss": 1.4908, + "theoretical_loss": 3.80414022010221, + "tokens_seen": 706936832 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003968505516549649, + "loss": 1.439, + "theoretical_loss": 3.804104744504703, + "tokens_seen": 707002368 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003968405215646941, + "loss": 1.454, + "theoretical_loss": 3.8040692731161383, + "tokens_seen": 707067904 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003968304914744233, + "loss": 1.3787, + "theoretical_loss": 3.8040338059356267, + "tokens_seen": 707133440 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039682046138415244, + "loss": 1.5277, + "theoretical_loss": 3.803998342962279, + "tokens_seen": 707198976 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003968104312938817, + "loss": 1.3718, + "theoretical_loss": 3.8039628841952067, + "tokens_seen": 707264512 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003968004012036108, + "loss": 1.415, + "theoretical_loss": 3.8039274296335206, + "tokens_seen": 707330048 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039679037111334004, + "loss": 1.4218, + "theoretical_loss": 3.803891979276332, + "tokens_seen": 707395584 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003967803410230692, + "loss": 1.4148, + "theoretical_loss": 3.8038565331227545, + "tokens_seen": 707461120 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003967703109327984, + "loss": 1.3246, + "theoretical_loss": 3.8038210911718986, + "tokens_seen": 707526656 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003967602808425276, + "loss": 1.357, + "theoretical_loss": 3.803785653422878, + "tokens_seen": 707592192 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003967502507522568, + "loss": 1.2858, + "theoretical_loss": 3.8037502198748046, + "tokens_seen": 707657728 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039674022066198594, + "loss": 1.4578, + "theoretical_loss": 3.8037147905267914, + "tokens_seen": 707723264 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 240273, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5194541215896606, + "objective/train/theoretical_loss": 3.8036793653779526, + "objective/train/tokens_used": 707788800, + "theoretical_loss": 3.8036793653779526, + "tokens_seen": 707788800 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003967301905717152, + "loss": 1.4574, + "theoretical_loss": 3.8036793653779526, + "tokens_seen": 707788800 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003967201604814443, + "loss": 1.3582, + "theoretical_loss": 3.8036439444274004, + "tokens_seen": 707854336 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039671013039117354, + "loss": 1.4233, + "theoretical_loss": 3.80360852767425, + "tokens_seen": 707919872 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003967001003009027, + "loss": 1.374, + "theoretical_loss": 3.803573115117615, + "tokens_seen": 707985408 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003966900702106319, + "loss": 1.2997, + "theoretical_loss": 3.803537706756609, + "tokens_seen": 708050944 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003966800401203611, + "loss": 1.3347, + "theoretical_loss": 3.8035023025903483, + "tokens_seen": 708116480 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039667001003009027, + "loss": 1.4608, + "theoretical_loss": 3.803466902617946, + "tokens_seen": 708182016 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039665997993981945, + "loss": 1.434, + "theoretical_loss": 3.8034315068385185, + "tokens_seen": 708247552 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003966499498495487, + "loss": 1.3546, + "theoretical_loss": 3.8033961152511804, + "tokens_seen": 708313088 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003966399197592778, + "loss": 1.402, + "theoretical_loss": 3.8033607278550483, + "tokens_seen": 708378624 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039662988966900704, + "loss": 1.3665, + "theoretical_loss": 3.803325344649237, + "tokens_seen": 708444160 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039661985957873617, + "loss": 1.4529, + "theoretical_loss": 3.8032899656328647, + "tokens_seen": 708509696 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003966098294884654, + "loss": 1.4327, + "theoretical_loss": 3.8032545908050457, + "tokens_seen": 708575232 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003965997993981946, + "loss": 1.5004, + "theoretical_loss": 3.803219220164898, + "tokens_seen": 708640768 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039658976930792377, + "loss": 1.4064, + "theoretical_loss": 3.8031838537115386, + "tokens_seen": 708706304 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039657973921765295, + "loss": 1.4513, + "theoretical_loss": 3.8031484914440847, + "tokens_seen": 708771840 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003965697091273822, + "loss": 1.3349, + "theoretical_loss": 3.803113133361654, + "tokens_seen": 708837376 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003965596790371113, + "loss": 1.5501, + "theoretical_loss": 3.8030777794633632, + "tokens_seen": 708902912 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039654964894684055, + "loss": 1.3998, + "theoretical_loss": 3.8030424297483325, + "tokens_seen": 708968448 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003965396188565697, + "loss": 1.3842, + "theoretical_loss": 3.803007084215678, + "tokens_seen": 709033984 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003965295887662989, + "loss": 1.3425, + "theoretical_loss": 3.8029717428645204, + "tokens_seen": 709099520 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003965195586760281, + "loss": 1.3807, + "theoretical_loss": 3.802936405693978, + "tokens_seen": 709165056 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039650952858575727, + "loss": 1.3888, + "theoretical_loss": 3.802901072703169, + "tokens_seen": 709230592 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039649949849548645, + "loss": 1.3605, + "theoretical_loss": 3.802865743891214, + "tokens_seen": 709296128 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039648946840521563, + "loss": 1.3533, + "theoretical_loss": 3.8028304192572326, + "tokens_seen": 709361664 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003964794383149448, + "loss": 1.5635, + "theoretical_loss": 3.802795098800344, + "tokens_seen": 709427200 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039646940822467405, + "loss": 1.3826, + "theoretical_loss": 3.8027597825196695, + "tokens_seen": 709492736 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003964593781344032, + "loss": 1.4254, + "theoretical_loss": 3.802724470414329, + "tokens_seen": 709558272 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003964493480441324, + "loss": 1.3832, + "theoretical_loss": 3.8026891624834427, + "tokens_seen": 709623808 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003964393179538616, + "loss": 1.4123, + "theoretical_loss": 3.802653858726133, + "tokens_seen": 709689344 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003964292878635908, + "loss": 1.4739, + "theoretical_loss": 3.8026185591415205, + "tokens_seen": 709754880 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039641925777331996, + "loss": 1.4189, + "theoretical_loss": 3.802583263728727, + "tokens_seen": 709820416 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039640922768304914, + "loss": 1.5093, + "theoretical_loss": 3.802547972486874, + "tokens_seen": 709885952 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003963991975927783, + "loss": 1.3288, + "theoretical_loss": 3.802512685415084, + "tokens_seen": 709951488 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039638916750250755, + "loss": 1.5318, + "theoretical_loss": 3.802477402512479, + "tokens_seen": 710017024 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039637913741223673, + "loss": 1.3599, + "theoretical_loss": 3.8024421237781825, + "tokens_seen": 710082560 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003963691073219659, + "loss": 1.4495, + "theoretical_loss": 3.8024068492113163, + "tokens_seen": 710148096 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003963590772316951, + "loss": 1.5136, + "theoretical_loss": 3.802371578811004, + "tokens_seen": 710213632 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003963490471414243, + "loss": 1.3538, + "theoretical_loss": 3.8023363125763696, + "tokens_seen": 710279168 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003963390170511535, + "loss": 1.4398, + "theoretical_loss": 3.8023010505065358, + "tokens_seen": 710344704 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039632898696088264, + "loss": 1.4558, + "theoretical_loss": 3.802265792600627, + "tokens_seen": 710410240 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003963189568706119, + "loss": 1.4263, + "theoretical_loss": 3.8022305388577684, + "tokens_seen": 710475776 + }, + { + "epoch": 0.22, + "learning_rate": 0.000396308926780341, + "loss": 1.4128, + "theoretical_loss": 3.8021952892770834, + "tokens_seen": 710541312 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039629889669007024, + "loss": 1.4586, + "theoretical_loss": 3.802160043857697, + "tokens_seen": 710606848 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003962888665997994, + "loss": 1.5594, + "theoretical_loss": 3.802124802598734, + "tokens_seen": 710672384 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003962788365095286, + "loss": 1.4374, + "theoretical_loss": 3.8020895654993203, + "tokens_seen": 710737920 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003962688064192578, + "loss": 1.3568, + "theoretical_loss": 3.8020543325585807, + "tokens_seen": 710803456 + }, + { + "epoch": 0.22, + "learning_rate": 0.000396258776328987, + "loss": 1.4129, + "theoretical_loss": 3.802019103775642, + "tokens_seen": 710868992 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039624874623871614, + "loss": 1.3237, + "theoretical_loss": 3.8019838791496303, + "tokens_seen": 710934528 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003962387161484454, + "loss": 1.4582, + "theoretical_loss": 3.8019486586796707, + "tokens_seen": 711000064 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 241525, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4488269090652466, + "objective/train/theoretical_loss": 3.8019134423648904, + "objective/train/tokens_used": 711065600, + "theoretical_loss": 3.8019134423648904, + "tokens_seen": 711065600 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003962286860581745, + "loss": 1.4323, + "theoretical_loss": 3.8019134423648904, + "tokens_seen": 711065600 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039621865596790374, + "loss": 1.4838, + "theoretical_loss": 3.8018782302044167, + "tokens_seen": 711131136 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003962086258776329, + "loss": 1.539, + "theoretical_loss": 3.801843022197377, + "tokens_seen": 711196672 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003961985957873621, + "loss": 1.3839, + "theoretical_loss": 3.801807818342898, + "tokens_seen": 711262208 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003961885656970913, + "loss": 1.5267, + "theoretical_loss": 3.8017726186401077, + "tokens_seen": 711327744 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039617853560682047, + "loss": 1.403, + "theoretical_loss": 3.8017374230881336, + "tokens_seen": 711393280 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039616850551654965, + "loss": 1.4573, + "theoretical_loss": 3.8017022316861047, + "tokens_seen": 711458816 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003961584754262789, + "loss": 1.4463, + "theoretical_loss": 3.801667044433149, + "tokens_seen": 711524352 + }, + { + "epoch": 0.22, + "learning_rate": 0.000396148445336008, + "loss": 1.4755, + "theoretical_loss": 3.801631861328395, + "tokens_seen": 711589888 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039613841524573724, + "loss": 1.3301, + "theoretical_loss": 3.801596682370972, + "tokens_seen": 711655424 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039612838515546637, + "loss": 1.4679, + "theoretical_loss": 3.8015615075600095, + "tokens_seen": 711720960 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003961183550651956, + "loss": 1.4342, + "theoretical_loss": 3.8015263368946366, + "tokens_seen": 711786496 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003961083249749248, + "loss": 1.4933, + "theoretical_loss": 3.8014911703739833, + "tokens_seen": 711852032 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039609829488465397, + "loss": 1.4228, + "theoretical_loss": 3.8014560079971798, + "tokens_seen": 711917568 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039608826479438315, + "loss": 1.4706, + "theoretical_loss": 3.801420849763356, + "tokens_seen": 711983104 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003960782347041124, + "loss": 1.3534, + "theoretical_loss": 3.8013856956716428, + "tokens_seen": 712048640 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003960682046138415, + "loss": 1.4746, + "theoretical_loss": 3.8013505457211707, + "tokens_seen": 712114176 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039605817452357075, + "loss": 1.4249, + "theoretical_loss": 3.801315399911071, + "tokens_seen": 712179712 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003960481444332999, + "loss": 1.3946, + "theoretical_loss": 3.8012802582404754, + "tokens_seen": 712245248 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003960381143430291, + "loss": 1.3763, + "theoretical_loss": 3.801245120708515, + "tokens_seen": 712310784 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003960280842527583, + "loss": 1.4283, + "theoretical_loss": 3.801209987314322, + "tokens_seen": 712376320 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039601805416248747, + "loss": 1.4478, + "theoretical_loss": 3.8011748580570286, + "tokens_seen": 712441856 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039600802407221665, + "loss": 1.3672, + "theoretical_loss": 3.8011397329357672, + "tokens_seen": 712507392 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039599799398194583, + "loss": 1.3816, + "theoretical_loss": 3.80110461194967, + "tokens_seen": 712572928 + }, + { + "epoch": 0.22, + "learning_rate": 0.000395987963891675, + "loss": 1.4083, + "theoretical_loss": 3.8010694950978703, + "tokens_seen": 712638464 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039597793380140425, + "loss": 1.3695, + "theoretical_loss": 3.801034382379501, + "tokens_seen": 712704000 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003959679037111334, + "loss": 1.4004, + "theoretical_loss": 3.800999273793696, + "tokens_seen": 712769536 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003959578736208626, + "loss": 1.3498, + "theoretical_loss": 3.800964169339589, + "tokens_seen": 712835072 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003959478435305918, + "loss": 1.4518, + "theoretical_loss": 3.8009290690163136, + "tokens_seen": 712900608 + }, + { + "epoch": 0.22, + "learning_rate": 0.000395937813440321, + "loss": 1.4546, + "theoretical_loss": 3.8008939728230047, + "tokens_seen": 712966144 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039592778335005016, + "loss": 1.4658, + "theoretical_loss": 3.8008588807587955, + "tokens_seen": 713031680 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039591775325977934, + "loss": 1.5065, + "theoretical_loss": 3.8008237928228223, + "tokens_seen": 713097216 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003959077231695085, + "loss": 1.4477, + "theoretical_loss": 3.8007887090142196, + "tokens_seen": 713162752 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039589769307923775, + "loss": 1.4069, + "theoretical_loss": 3.800753629332122, + "tokens_seen": 713228288 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003958876629889669, + "loss": 1.4764, + "theoretical_loss": 3.800718553775666, + "tokens_seen": 713293824 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003958776328986961, + "loss": 1.4546, + "theoretical_loss": 3.8006834823439863, + "tokens_seen": 713359360 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039586760280842524, + "loss": 1.5328, + "theoretical_loss": 3.80064841503622, + "tokens_seen": 713424896 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003958575727181545, + "loss": 1.41, + "theoretical_loss": 3.800613351851503, + "tokens_seen": 713490432 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039584754262788366, + "loss": 1.408, + "theoretical_loss": 3.800578292788972, + "tokens_seen": 713555968 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039583751253761284, + "loss": 1.4763, + "theoretical_loss": 3.800543237847764, + "tokens_seen": 713621504 + }, + { + "epoch": 0.22, + "learning_rate": 0.000395827482447342, + "loss": 1.3925, + "theoretical_loss": 3.8005081870270154, + "tokens_seen": 713687040 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003958174523570712, + "loss": 1.4827, + "theoretical_loss": 3.8004731403258645, + "tokens_seen": 713752576 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003958074222668004, + "loss": 1.4168, + "theoretical_loss": 3.800438097743448, + "tokens_seen": 713818112 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003957973921765296, + "loss": 1.4131, + "theoretical_loss": 3.8004030592789046, + "tokens_seen": 713883648 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039578736208625875, + "loss": 1.3346, + "theoretical_loss": 3.8003680249313723, + "tokens_seen": 713949184 + }, + { + "epoch": 0.22, + "learning_rate": 0.000395777331995988, + "loss": 1.4204, + "theoretical_loss": 3.800332994699989, + "tokens_seen": 714014720 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039576730190571716, + "loss": 1.4265, + "theoretical_loss": 3.8002979685838936, + "tokens_seen": 714080256 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039575727181544634, + "loss": 1.486, + "theoretical_loss": 3.800262946582225, + "tokens_seen": 714145792 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003957472417251755, + "loss": 1.3778, + "theoretical_loss": 3.800227928694123, + "tokens_seen": 714211328 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003957372116349047, + "loss": 1.443, + "theoretical_loss": 3.800192914918726, + "tokens_seen": 714276864 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 242820, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4017164707183838, + "objective/train/theoretical_loss": 3.800157905255175, + "objective/train/tokens_used": 714342400, + "theoretical_loss": 3.800157905255175, + "tokens_seen": 714342400 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003957271815446339, + "loss": 1.4055, + "theoretical_loss": 3.800157905255175, + "tokens_seen": 714342400 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003957171514543631, + "loss": 1.4039, + "theoretical_loss": 3.8001228997026084, + "tokens_seen": 714407936 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039570712136409225, + "loss": 1.375, + "theoretical_loss": 3.800087898260167, + "tokens_seen": 714473472 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003956970912738215, + "loss": 1.4658, + "theoretical_loss": 3.800052900926992, + "tokens_seen": 714539008 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003956870611835506, + "loss": 1.4355, + "theoretical_loss": 3.8000179077022236, + "tokens_seen": 714604544 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039567703109327985, + "loss": 1.4457, + "theoretical_loss": 3.7999829185850027, + "tokens_seen": 714670080 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039566700100300903, + "loss": 1.5087, + "theoretical_loss": 3.7999479335744706, + "tokens_seen": 714735616 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003956569709127382, + "loss": 1.4767, + "theoretical_loss": 3.799912952669769, + "tokens_seen": 714801152 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003956469408224674, + "loss": 1.4097, + "theoretical_loss": 3.7998779758700394, + "tokens_seen": 714866688 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039563691073219657, + "loss": 1.3334, + "theoretical_loss": 3.7998430031744235, + "tokens_seen": 714932224 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003956268806419258, + "loss": 1.5123, + "theoretical_loss": 3.7998080345820644, + "tokens_seen": 714997760 + }, + { + "epoch": 0.22, + "learning_rate": 0.000395616850551655, + "loss": 1.3525, + "theoretical_loss": 3.7997730700921046, + "tokens_seen": 715063296 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039560682046138417, + "loss": 1.4065, + "theoretical_loss": 3.7997381097036858, + "tokens_seen": 715128832 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039559679037111335, + "loss": 1.3321, + "theoretical_loss": 3.7997031534159524, + "tokens_seen": 715194368 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003955867602808426, + "loss": 1.3619, + "theoretical_loss": 3.799668201228047, + "tokens_seen": 715259904 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003955767301905717, + "loss": 1.4991, + "theoretical_loss": 3.7996332531391133, + "tokens_seen": 715325440 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039556670010030095, + "loss": 1.4955, + "theoretical_loss": 3.7995983091482954, + "tokens_seen": 715390976 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003955566700100301, + "loss": 1.4641, + "theoretical_loss": 3.799563369254737, + "tokens_seen": 715456512 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003955466399197593, + "loss": 1.3567, + "theoretical_loss": 3.7995284334575823, + "tokens_seen": 715522048 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003955366098294885, + "loss": 1.4289, + "theoretical_loss": 3.7994935017559763, + "tokens_seen": 715587584 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039552657973921767, + "loss": 1.418, + "theoretical_loss": 3.799458574149064, + "tokens_seen": 715653120 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039551654964894685, + "loss": 1.3971, + "theoretical_loss": 3.7994236506359895, + "tokens_seen": 715718656 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039550651955867603, + "loss": 1.3894, + "theoretical_loss": 3.7993887312158994, + "tokens_seen": 715784192 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003954964894684052, + "loss": 1.3631, + "theoretical_loss": 3.799353815887939, + "tokens_seen": 715849728 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039548645937813445, + "loss": 1.3295, + "theoretical_loss": 3.799318904651254, + "tokens_seen": 715915264 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003954764292878636, + "loss": 1.4803, + "theoretical_loss": 3.7992839975049906, + "tokens_seen": 715980800 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003954663991975928, + "loss": 1.2767, + "theoretical_loss": 3.799249094448295, + "tokens_seen": 716046336 + }, + { + "epoch": 0.22, + "learning_rate": 0.000395456369107322, + "loss": 1.4069, + "theoretical_loss": 3.799214195480314, + "tokens_seen": 716111872 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003954463390170512, + "loss": 1.3537, + "theoretical_loss": 3.799179300600194, + "tokens_seen": 716177408 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039543630892678036, + "loss": 1.3138, + "theoretical_loss": 3.7991444098070835, + "tokens_seen": 716242944 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039542627883650954, + "loss": 1.4631, + "theoretical_loss": 3.7991095231001286, + "tokens_seen": 716308480 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003954162487462387, + "loss": 1.298, + "theoretical_loss": 3.7990746404784774, + "tokens_seen": 716374016 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039540621865596795, + "loss": 1.4757, + "theoretical_loss": 3.7990397619412786, + "tokens_seen": 716439552 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003953961885656971, + "loss": 1.3974, + "theoretical_loss": 3.799004887487679, + "tokens_seen": 716505088 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003953861584754263, + "loss": 1.3926, + "theoretical_loss": 3.798970017116828, + "tokens_seen": 716570624 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039537612838515544, + "loss": 1.3926, + "theoretical_loss": 3.7989351508278744, + "tokens_seen": 716636160 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003953660982948847, + "loss": 1.3773, + "theoretical_loss": 3.7989002886199663, + "tokens_seen": 716701696 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039535606820461386, + "loss": 1.3053, + "theoretical_loss": 3.7988654304922536, + "tokens_seen": 716767232 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039534603811434304, + "loss": 1.4648, + "theoretical_loss": 3.798830576443886, + "tokens_seen": 716832768 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003953360080240722, + "loss": 1.4428, + "theoretical_loss": 3.7987957264740118, + "tokens_seen": 716898304 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003953259779338014, + "loss": 1.4126, + "theoretical_loss": 3.7987608805817827, + "tokens_seen": 716963840 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003953159478435306, + "loss": 1.405, + "theoretical_loss": 3.7987260387663486, + "tokens_seen": 717029376 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003953059177532598, + "loss": 1.3112, + "theoretical_loss": 3.7986912010268585, + "tokens_seen": 717094912 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039529588766298895, + "loss": 1.3853, + "theoretical_loss": 3.798656367362465, + "tokens_seen": 717160448 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003952858575727182, + "loss": 1.3904, + "theoretical_loss": 3.798621537772318, + "tokens_seen": 717225984 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039527582748244736, + "loss": 1.3877, + "theoretical_loss": 3.7985867122555694, + "tokens_seen": 717291520 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039526579739217654, + "loss": 1.4294, + "theoretical_loss": 3.79855189081137, + "tokens_seen": 717357056 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003952557673019057, + "loss": 1.4731, + "theoretical_loss": 3.7985170734388722, + "tokens_seen": 717422592 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003952457372116349, + "loss": 1.466, + "theoretical_loss": 3.7984822601372272, + "tokens_seen": 717488128 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003952357071213641, + "loss": 1.4063, + "theoretical_loss": 3.7984474509055888, + "tokens_seen": 717553664 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 243710, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3566665649414062, + "objective/train/theoretical_loss": 3.7984126457431078, + "objective/train/tokens_used": 717619200, + "theoretical_loss": 3.7984126457431078, + "tokens_seen": 717619200 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003952256770310933, + "loss": 1.3522, + "theoretical_loss": 3.7984126457431078, + "tokens_seen": 717619200 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039521564694082245, + "loss": 1.4265, + "theoretical_loss": 3.798377844648938, + "tokens_seen": 717684736 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003952056168505517, + "loss": 1.4618, + "theoretical_loss": 3.7983430476222315, + "tokens_seen": 717750272 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003951955867602808, + "loss": 1.3959, + "theoretical_loss": 3.7983082546621425, + "tokens_seen": 717815808 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039518555667001005, + "loss": 1.3764, + "theoretical_loss": 3.7982734657678243, + "tokens_seen": 717881344 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039517552657973923, + "loss": 1.3718, + "theoretical_loss": 3.798238680938431, + "tokens_seen": 717946880 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003951654964894684, + "loss": 1.4571, + "theoretical_loss": 3.798203900173116, + "tokens_seen": 718012416 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003951554663991976, + "loss": 1.359, + "theoretical_loss": 3.798169123471034, + "tokens_seen": 718077952 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039514543630892677, + "loss": 1.3928, + "theoretical_loss": 3.798134350831339, + "tokens_seen": 718143488 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039513540621865595, + "loss": 1.4632, + "theoretical_loss": 3.7980995822531862, + "tokens_seen": 718209024 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003951253761283852, + "loss": 1.4309, + "theoretical_loss": 3.7980648177357312, + "tokens_seen": 718274560 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003951153460381143, + "loss": 1.3777, + "theoretical_loss": 3.7980300572781283, + "tokens_seen": 718340096 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039510531594784355, + "loss": 1.3824, + "theoretical_loss": 3.797995300879534, + "tokens_seen": 718405632 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039509528585757273, + "loss": 1.3957, + "theoretical_loss": 3.797960548539103, + "tokens_seen": 718471168 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003950852557673019, + "loss": 1.4225, + "theoretical_loss": 3.7979258002559924, + "tokens_seen": 718536704 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003950752256770311, + "loss": 1.4747, + "theoretical_loss": 3.797891056029358, + "tokens_seen": 718602240 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003950651955867603, + "loss": 1.3833, + "theoretical_loss": 3.7978563158583567, + "tokens_seen": 718667776 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039505516549648946, + "loss": 1.4119, + "theoretical_loss": 3.797821579742145, + "tokens_seen": 718733312 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003950451354062187, + "loss": 1.4298, + "theoretical_loss": 3.7977868476798795, + "tokens_seen": 718798848 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003950351053159478, + "loss": 1.4801, + "theoretical_loss": 3.7977521196707187, + "tokens_seen": 718864384 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039502507522567705, + "loss": 1.4265, + "theoretical_loss": 3.7977173957138195, + "tokens_seen": 718929920 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003950150451354062, + "loss": 1.3007, + "theoretical_loss": 3.79768267580834, + "tokens_seen": 718995456 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003950050150451354, + "loss": 1.4688, + "theoretical_loss": 3.797647959953438, + "tokens_seen": 719060992 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003949949849548646, + "loss": 1.3897, + "theoretical_loss": 3.7976132481482714, + "tokens_seen": 719126528 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003949849548645938, + "loss": 1.5009, + "theoretical_loss": 3.7975785403919997, + "tokens_seen": 719192064 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039497492477432296, + "loss": 1.5064, + "theoretical_loss": 3.797543836683781, + "tokens_seen": 719257600 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003949648946840522, + "loss": 1.3335, + "theoretical_loss": 3.797509137022775, + "tokens_seen": 719323136 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003949548645937813, + "loss": 1.5105, + "theoretical_loss": 3.797474441408141, + "tokens_seen": 719388672 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039494483450351056, + "loss": 1.3414, + "theoretical_loss": 3.797439749839038, + "tokens_seen": 719454208 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003949348044132397, + "loss": 1.3822, + "theoretical_loss": 3.7974050623146267, + "tokens_seen": 719519744 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003949247743229689, + "loss": 1.2561, + "theoretical_loss": 3.7973703788340663, + "tokens_seen": 719585280 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003949147442326981, + "loss": 1.3877, + "theoretical_loss": 3.797335699396517, + "tokens_seen": 719650816 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003949047141424273, + "loss": 1.4451, + "theoretical_loss": 3.7973010240011407, + "tokens_seen": 719716352 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003948946840521565, + "loss": 1.4148, + "theoretical_loss": 3.7972663526470973, + "tokens_seen": 719781888 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039488465396188564, + "loss": 1.3817, + "theoretical_loss": 3.797231685333548, + "tokens_seen": 719847424 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003948746238716149, + "loss": 1.3937, + "theoretical_loss": 3.7971970220596543, + "tokens_seen": 719912960 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039486459378134406, + "loss": 1.4514, + "theoretical_loss": 3.7971623628245776, + "tokens_seen": 719978496 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039485456369107324, + "loss": 1.3812, + "theoretical_loss": 3.79712770762748, + "tokens_seen": 720044032 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003948445336008024, + "loss": 1.4014, + "theoretical_loss": 3.797093056467523, + "tokens_seen": 720109568 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003948345035105316, + "loss": 1.4275, + "theoretical_loss": 3.7970584093438697, + "tokens_seen": 720175104 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003948244734202608, + "loss": 1.3613, + "theoretical_loss": 3.7970237662556823, + "tokens_seen": 720240640 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039481444332999, + "loss": 1.4794, + "theoretical_loss": 3.7969891272021234, + "tokens_seen": 720306176 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039480441323971915, + "loss": 1.3757, + "theoretical_loss": 3.796954492182357, + "tokens_seen": 720371712 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003947943831494484, + "loss": 1.4211, + "theoretical_loss": 3.796919861195546, + "tokens_seen": 720437248 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039478435305917756, + "loss": 1.2972, + "theoretical_loss": 3.796885234240853, + "tokens_seen": 720502784 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039477432296890674, + "loss": 1.4153, + "theoretical_loss": 3.796850611317443, + "tokens_seen": 720568320 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003947642928786359, + "loss": 1.3795, + "theoretical_loss": 3.79681599242448, + "tokens_seen": 720633856 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003947542627883651, + "loss": 1.3442, + "theoretical_loss": 3.7967813775611283, + "tokens_seen": 720699392 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003947442326980943, + "loss": 1.5216, + "theoretical_loss": 3.796746766726552, + "tokens_seen": 720764928 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003947342026078235, + "loss": 1.4785, + "theoretical_loss": 3.7967121599199167, + "tokens_seen": 720830464 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 244991, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3190022706985474, + "objective/train/theoretical_loss": 3.7966775571403866, + "objective/train/tokens_used": 720896000, + "theoretical_loss": 3.7966775571403866, + "tokens_seen": 720896000 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039472417251755265, + "loss": 1.3429, + "theoretical_loss": 3.7966775571403866, + "tokens_seen": 720896000 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003947141424272819, + "loss": 1.3739, + "theoretical_loss": 3.7966429583871277, + "tokens_seen": 720961536 + }, + { + "epoch": 0.22, + "learning_rate": 0.000394704112337011, + "loss": 1.3882, + "theoretical_loss": 3.7966083636593053, + "tokens_seen": 721027072 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039469408224674025, + "loss": 1.4208, + "theoretical_loss": 3.7965737729560853, + "tokens_seen": 721092608 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039468405215646943, + "loss": 1.4731, + "theoretical_loss": 3.7965391862766342, + "tokens_seen": 721158144 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003946740220661986, + "loss": 1.4425, + "theoretical_loss": 3.7965046036201175, + "tokens_seen": 721223680 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003946639919759278, + "loss": 1.4411, + "theoretical_loss": 3.7964700249857026, + "tokens_seen": 721289216 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003946639919759278, + "loss": 1.3833, + "theoretical_loss": 3.796435450372556, + "tokens_seen": 721354752 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039465396188565697, + "loss": 1.4488, + "theoretical_loss": 3.7964008797798443, + "tokens_seen": 721420288 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039464393179538615, + "loss": 1.3803, + "theoretical_loss": 3.796366313206735, + "tokens_seen": 721485824 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003946339017051154, + "loss": 1.4337, + "theoretical_loss": 3.7963317506523966, + "tokens_seen": 721551360 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003946238716148445, + "loss": 1.5076, + "theoretical_loss": 3.796297192115996, + "tokens_seen": 721616896 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039461384152457375, + "loss": 1.4226, + "theoretical_loss": 3.796262637596702, + "tokens_seen": 721682432 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039460381143430293, + "loss": 1.4338, + "theoretical_loss": 3.7962280870936818, + "tokens_seen": 721747968 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003945937813440321, + "loss": 1.3636, + "theoretical_loss": 3.7961935406061054, + "tokens_seen": 721813504 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003945837512537613, + "loss": 1.5842, + "theoretical_loss": 3.79615899813314, + "tokens_seen": 721879040 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003945737211634905, + "loss": 1.3807, + "theoretical_loss": 3.7961244596739556, + "tokens_seen": 721944576 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039456369107321966, + "loss": 1.2956, + "theoretical_loss": 3.7960899252277214, + "tokens_seen": 722010112 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003945536609829489, + "loss": 1.3291, + "theoretical_loss": 3.7960553947936075, + "tokens_seen": 722075648 + }, + { + "epoch": 0.22, + "learning_rate": 0.000394543630892678, + "loss": 1.2335, + "theoretical_loss": 3.796020868370783, + "tokens_seen": 722141184 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039453360080240725, + "loss": 1.3501, + "theoretical_loss": 3.795986345958418, + "tokens_seen": 722206720 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003945235707121364, + "loss": 1.4181, + "theoretical_loss": 3.7959518275556827, + "tokens_seen": 722272256 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003945135406218656, + "loss": 1.4543, + "theoretical_loss": 3.7959173131617483, + "tokens_seen": 722337792 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003945035105315948, + "loss": 1.3495, + "theoretical_loss": 3.795882802775785, + "tokens_seen": 722403328 + }, + { + "epoch": 0.22, + "learning_rate": 0.000394493480441324, + "loss": 1.4233, + "theoretical_loss": 3.7958482963969638, + "tokens_seen": 722468864 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039448345035105316, + "loss": 1.3835, + "theoretical_loss": 3.7958137940244567, + "tokens_seen": 722534400 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003944734202607824, + "loss": 1.4425, + "theoretical_loss": 3.795779295657434, + "tokens_seen": 722599936 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003944633901705115, + "loss": 1.4753, + "theoretical_loss": 3.7957448012950685, + "tokens_seen": 722665472 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039445336008024076, + "loss": 1.4811, + "theoretical_loss": 3.795710310936532, + "tokens_seen": 722731008 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003944433299899699, + "loss": 1.3355, + "theoretical_loss": 3.7956758245809965, + "tokens_seen": 722796544 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003944332998996991, + "loss": 1.4882, + "theoretical_loss": 3.795641342227635, + "tokens_seen": 722862080 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003944232698094283, + "loss": 1.3793, + "theoretical_loss": 3.7956068638756197, + "tokens_seen": 722927616 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003944132397191575, + "loss": 1.4506, + "theoretical_loss": 3.795572389524124, + "tokens_seen": 722993152 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039440320962888666, + "loss": 1.4771, + "theoretical_loss": 3.795537919172321, + "tokens_seen": 723058688 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039439317953861584, + "loss": 1.4252, + "theoretical_loss": 3.795503452819384, + "tokens_seen": 723124224 + }, + { + "epoch": 0.22, + "learning_rate": 0.000394383149448345, + "loss": 1.3932, + "theoretical_loss": 3.795468990464487, + "tokens_seen": 723189760 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039437311935807426, + "loss": 1.5195, + "theoretical_loss": 3.795434532106804, + "tokens_seen": 723255296 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003943630892678034, + "loss": 1.4791, + "theoretical_loss": 3.79540007774551, + "tokens_seen": 723320832 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003943530591775326, + "loss": 1.4144, + "theoretical_loss": 3.795365627379778, + "tokens_seen": 723386368 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039434302908726175, + "loss": 1.3847, + "theoretical_loss": 3.795331181008783, + "tokens_seen": 723451904 + }, + { + "epoch": 0.22, + "learning_rate": 0.000394332998996991, + "loss": 1.4803, + "theoretical_loss": 3.795296738631701, + "tokens_seen": 723517440 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039432296890672017, + "loss": 1.3757, + "theoretical_loss": 3.795262300247707, + "tokens_seen": 723582976 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039431293881644935, + "loss": 1.3567, + "theoretical_loss": 3.795227865855975, + "tokens_seen": 723648512 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039430290872617853, + "loss": 1.4123, + "theoretical_loss": 3.7951934354556824, + "tokens_seen": 723714048 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039429287863590776, + "loss": 1.413, + "theoretical_loss": 3.7951590090460043, + "tokens_seen": 723779584 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003942828485456369, + "loss": 1.3822, + "theoretical_loss": 3.795124586626117, + "tokens_seen": 723845120 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003942728184553661, + "loss": 1.4416, + "theoretical_loss": 3.7950901681951974, + "tokens_seen": 723910656 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039426278836509525, + "loss": 1.3534, + "theoretical_loss": 3.795055753752422, + "tokens_seen": 723976192 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003942527582748245, + "loss": 1.441, + "theoretical_loss": 3.795021343296967, + "tokens_seen": 724041728 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039424272818455367, + "loss": 1.3346, + "theoretical_loss": 3.7949869368280105, + "tokens_seen": 724107264 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 245956, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4522697925567627, + "objective/train/theoretical_loss": 3.7949525343447297, + "objective/train/tokens_used": 724172800, + "theoretical_loss": 3.7949525343447297, + "tokens_seen": 724172800 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039423269809428285, + "loss": 1.3897, + "theoretical_loss": 3.7949525343447297, + "tokens_seen": 724172800 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039422266800401203, + "loss": 1.3861, + "theoretical_loss": 3.794918135846302, + "tokens_seen": 724238336 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003942126379137412, + "loss": 1.4609, + "theoretical_loss": 3.7948837413319056, + "tokens_seen": 724303872 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003942026078234704, + "loss": 1.4036, + "theoretical_loss": 3.7948493508007184, + "tokens_seen": 724369408 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039419257773319963, + "loss": 1.4357, + "theoretical_loss": 3.794814964251919, + "tokens_seen": 724434944 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039418254764292876, + "loss": 1.3925, + "theoretical_loss": 3.7947805816846856, + "tokens_seen": 724500480 + }, + { + "epoch": 0.22, + "learning_rate": 0.000394172517552658, + "loss": 1.4432, + "theoretical_loss": 3.7947462030981978, + "tokens_seen": 724566016 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003941624874623871, + "loss": 1.344, + "theoretical_loss": 3.794711828491634, + "tokens_seen": 724631552 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039415245737211635, + "loss": 1.3706, + "theoretical_loss": 3.794677457864174, + "tokens_seen": 724697088 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003941424272818456, + "loss": 1.4378, + "theoretical_loss": 3.7946430912149975, + "tokens_seen": 724762624 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003941323971915747, + "loss": 1.4181, + "theoretical_loss": 3.794608728543284, + "tokens_seen": 724828160 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039412236710130395, + "loss": 1.4249, + "theoretical_loss": 3.794574369848214, + "tokens_seen": 724893696 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039411233701103313, + "loss": 1.4099, + "theoretical_loss": 3.794540015128968, + "tokens_seen": 724959232 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003941023069207623, + "loss": 1.4756, + "theoretical_loss": 3.7945056643847255, + "tokens_seen": 725024768 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003940922768304915, + "loss": 1.3731, + "theoretical_loss": 3.7944713176146685, + "tokens_seen": 725090304 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003940822467402207, + "loss": 1.3936, + "theoretical_loss": 3.794436974817977, + "tokens_seen": 725155840 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039407221664994986, + "loss": 1.4207, + "theoretical_loss": 3.7944026359938334, + "tokens_seen": 725221376 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003940621865596791, + "loss": 1.4313, + "theoretical_loss": 3.794368301141419, + "tokens_seen": 725286912 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003940521564694082, + "loss": 1.3784, + "theoretical_loss": 3.794333970259915, + "tokens_seen": 725352448 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039404212637913745, + "loss": 1.3699, + "theoretical_loss": 3.794299643348504, + "tokens_seen": 725417984 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003940320962888666, + "loss": 1.3928, + "theoretical_loss": 3.7942653204063683, + "tokens_seen": 725483520 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003940220661985958, + "loss": 1.3557, + "theoretical_loss": 3.79423100143269, + "tokens_seen": 725549056 + }, + { + "epoch": 0.22, + "learning_rate": 0.000394012036108325, + "loss": 1.4298, + "theoretical_loss": 3.794196686426652, + "tokens_seen": 725614592 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003940020060180542, + "loss": 1.4671, + "theoretical_loss": 3.7941623753874376, + "tokens_seen": 725680128 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039399197592778336, + "loss": 1.42, + "theoretical_loss": 3.79412806831423, + "tokens_seen": 725745664 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003939819458375126, + "loss": 1.3718, + "theoretical_loss": 3.7940937652062123, + "tokens_seen": 725811200 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003939719157472417, + "loss": 1.3128, + "theoretical_loss": 3.794059466062569, + "tokens_seen": 725876736 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039396188565697096, + "loss": 1.384, + "theoretical_loss": 3.7940251708824833, + "tokens_seen": 725942272 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003939518555667001, + "loss": 1.3948, + "theoretical_loss": 3.79399087966514, + "tokens_seen": 726007808 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003939418254764293, + "loss": 1.459, + "theoretical_loss": 3.7939565924097227, + "tokens_seen": 726073344 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003939317953861585, + "loss": 1.3482, + "theoretical_loss": 3.793922309115417, + "tokens_seen": 726138880 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003939217652958877, + "loss": 1.3987, + "theoretical_loss": 3.7938880297814075, + "tokens_seen": 726204416 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039391173520561686, + "loss": 1.2803, + "theoretical_loss": 3.793853754406879, + "tokens_seen": 726269952 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039390170511534604, + "loss": 1.3935, + "theoretical_loss": 3.793819482991018, + "tokens_seen": 726335488 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003938916750250752, + "loss": 1.381, + "theoretical_loss": 3.793785215533009, + "tokens_seen": 726401024 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039388164493480446, + "loss": 1.2597, + "theoretical_loss": 3.7937509520320387, + "tokens_seen": 726466560 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003938716148445336, + "loss": 1.4703, + "theoretical_loss": 3.793716692487293, + "tokens_seen": 726532096 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003938615847542628, + "loss": 1.3451, + "theoretical_loss": 3.7936824368979583, + "tokens_seen": 726597632 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039385155466399195, + "loss": 1.436, + "theoretical_loss": 3.793648185263221, + "tokens_seen": 726663168 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003938415245737212, + "loss": 1.4762, + "theoretical_loss": 3.7936139375822684, + "tokens_seen": 726728704 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039383149448345037, + "loss": 1.4747, + "theoretical_loss": 3.793579693854287, + "tokens_seen": 726794240 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039382146439317955, + "loss": 1.3477, + "theoretical_loss": 3.7935454540784646, + "tokens_seen": 726859776 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039381143430290873, + "loss": 1.3656, + "theoretical_loss": 3.7935112182539887, + "tokens_seen": 726925312 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039380140421263796, + "loss": 1.4033, + "theoretical_loss": 3.7934769863800475, + "tokens_seen": 726990848 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003937913741223671, + "loss": 1.4848, + "theoretical_loss": 3.7934427584558286, + "tokens_seen": 727056384 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003937813440320963, + "loss": 1.4169, + "theoretical_loss": 3.7934085344805206, + "tokens_seen": 727121920 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039377131394182545, + "loss": 1.4923, + "theoretical_loss": 3.7933743144533114, + "tokens_seen": 727187456 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003937612838515547, + "loss": 1.4459, + "theoretical_loss": 3.7933400983733905, + "tokens_seen": 727252992 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039375125376128387, + "loss": 1.3681, + "theoretical_loss": 3.7933058862399465, + "tokens_seen": 727318528 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039374122367101305, + "loss": 1.3264, + "theoretical_loss": 3.7932716780521694, + "tokens_seen": 727384064 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 247152, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4575345516204834, + "objective/train/theoretical_loss": 3.793237473809248, + "objective/train/tokens_used": 727449600, + "theoretical_loss": 3.793237473809248, + "tokens_seen": 727449600 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039373119358074223, + "loss": 1.4138, + "theoretical_loss": 3.793237473809248, + "tokens_seen": 727449600 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003937211634904714, + "loss": 1.446, + "theoretical_loss": 3.7932032735103722, + "tokens_seen": 727515136 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003937111334002006, + "loss": 1.3345, + "theoretical_loss": 3.793169077154732, + "tokens_seen": 727580672 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039370110330992983, + "loss": 1.4409, + "theoretical_loss": 3.7931348847415176, + "tokens_seen": 727646208 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039369107321965896, + "loss": 1.4477, + "theoretical_loss": 3.79310069626992, + "tokens_seen": 727711744 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003936810431293882, + "loss": 1.5182, + "theoretical_loss": 3.793066511739129, + "tokens_seen": 727777280 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003936710130391173, + "loss": 1.4505, + "theoretical_loss": 3.793032331148336, + "tokens_seen": 727842816 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039366098294884655, + "loss": 1.4057, + "theoretical_loss": 3.7929981544967326, + "tokens_seen": 727908352 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039365095285857573, + "loss": 1.5325, + "theoretical_loss": 3.7929639817835095, + "tokens_seen": 727973888 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003936409227683049, + "loss": 1.3898, + "theoretical_loss": 3.7929298130078584, + "tokens_seen": 728039424 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003936308926780341, + "loss": 1.4453, + "theoretical_loss": 3.792895648168972, + "tokens_seen": 728104960 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039362086258776333, + "loss": 1.4669, + "theoretical_loss": 3.792861487266042, + "tokens_seen": 728170496 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039361083249749246, + "loss": 1.5329, + "theoretical_loss": 3.7928273302982602, + "tokens_seen": 728236032 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003936008024072217, + "loss": 1.4475, + "theoretical_loss": 3.7927931772648202, + "tokens_seen": 728301568 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003935907723169508, + "loss": 1.4076, + "theoretical_loss": 3.792759028164914, + "tokens_seen": 728367104 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039358074222668006, + "loss": 1.4353, + "theoretical_loss": 3.792724882997735, + "tokens_seen": 728432640 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039357071213640924, + "loss": 1.3976, + "theoretical_loss": 3.792690741762477, + "tokens_seen": 728498176 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003935606820461384, + "loss": 1.3431, + "theoretical_loss": 3.792656604458333, + "tokens_seen": 728563712 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003935506519558676, + "loss": 1.3411, + "theoretical_loss": 3.7926224710844973, + "tokens_seen": 728629248 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003935406218655968, + "loss": 1.3431, + "theoretical_loss": 3.7925883416401636, + "tokens_seen": 728694784 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039353059177532596, + "loss": 1.4844, + "theoretical_loss": 3.792554216124526, + "tokens_seen": 728760320 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003935205616850552, + "loss": 1.5158, + "theoretical_loss": 3.792520094536779, + "tokens_seen": 728825856 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003935105315947843, + "loss": 1.4456, + "theoretical_loss": 3.792485976876118, + "tokens_seen": 728891392 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039350050150451356, + "loss": 1.319, + "theoretical_loss": 3.7924518631417374, + "tokens_seen": 728956928 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003934904714142427, + "loss": 1.43, + "theoretical_loss": 3.792417753332833, + "tokens_seen": 729022464 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003934804413239719, + "loss": 1.4146, + "theoretical_loss": 3.7923836474485997, + "tokens_seen": 729088000 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003934704112337011, + "loss": 1.3147, + "theoretical_loss": 3.792349545488233, + "tokens_seen": 729153536 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003934603811434303, + "loss": 1.4712, + "theoretical_loss": 3.79231544745093, + "tokens_seen": 729219072 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039345035105315946, + "loss": 1.3011, + "theoretical_loss": 3.7922813533358855, + "tokens_seen": 729284608 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003934403209628887, + "loss": 1.4192, + "theoretical_loss": 3.792247263142297, + "tokens_seen": 729350144 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039343029087261783, + "loss": 1.3867, + "theoretical_loss": 3.7922131768693603, + "tokens_seen": 729415680 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039342026078234706, + "loss": 1.3622, + "theoretical_loss": 3.792179094516273, + "tokens_seen": 729481216 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003934102306920762, + "loss": 1.4688, + "theoretical_loss": 3.792145016082232, + "tokens_seen": 729546752 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003934002006018054, + "loss": 1.4621, + "theoretical_loss": 3.7921109415664347, + "tokens_seen": 729612288 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039339017051153466, + "loss": 1.3238, + "theoretical_loss": 3.792076870968078, + "tokens_seen": 729677824 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003933801404212638, + "loss": 1.3636, + "theoretical_loss": 3.792042804286361, + "tokens_seen": 729743360 + }, + { + "epoch": 0.22, + "learning_rate": 0.000393370110330993, + "loss": 1.4009, + "theoretical_loss": 3.792008741520481, + "tokens_seen": 729808896 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039336008024072215, + "loss": 1.3655, + "theoretical_loss": 3.791974682669636, + "tokens_seen": 729874432 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003933500501504514, + "loss": 1.3976, + "theoretical_loss": 3.791940627733026, + "tokens_seen": 729939968 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039334002006018057, + "loss": 1.4134, + "theoretical_loss": 3.791906576709848, + "tokens_seen": 730005504 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039332998996990975, + "loss": 1.4181, + "theoretical_loss": 3.791872529599302, + "tokens_seen": 730071040 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039331995987963893, + "loss": 1.4417, + "theoretical_loss": 3.791838486400587, + "tokens_seen": 730136576 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039330992978936816, + "loss": 1.3934, + "theoretical_loss": 3.7918044471129027, + "tokens_seen": 730202112 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003932998996990973, + "loss": 1.3388, + "theoretical_loss": 3.7917704117354485, + "tokens_seen": 730267648 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003932898696088265, + "loss": 1.4081, + "theoretical_loss": 3.7917363802674244, + "tokens_seen": 730333184 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039327983951855565, + "loss": 1.426, + "theoretical_loss": 3.791702352708031, + "tokens_seen": 730398720 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003932698094282849, + "loss": 1.3429, + "theoretical_loss": 3.7916683290564683, + "tokens_seen": 730464256 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039325977933801407, + "loss": 1.3688, + "theoretical_loss": 3.7916343093119376, + "tokens_seen": 730529792 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039324974924774325, + "loss": 1.34, + "theoretical_loss": 3.791600293473639, + "tokens_seen": 730595328 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039323971915747243, + "loss": 1.3876, + "theoretical_loss": 3.791566281540774, + "tokens_seen": 730660864 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 247952, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3238844871520996, + "objective/train/theoretical_loss": 3.791532273512544, + "objective/train/tokens_used": 730726400, + "theoretical_loss": 3.791532273512544, + "tokens_seen": 730726400 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003932296890672016, + "loss": 1.3387, + "theoretical_loss": 3.791532273512544, + "tokens_seen": 730726400 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003932196589769308, + "loss": 1.2857, + "theoretical_loss": 3.7914982693881503, + "tokens_seen": 730791936 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039320962888666003, + "loss": 1.3689, + "theoretical_loss": 3.791464269166795, + "tokens_seen": 730857472 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039319959879638916, + "loss": 1.4953, + "theoretical_loss": 3.7914302728476805, + "tokens_seen": 730923008 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003931895687061184, + "loss": 1.3882, + "theoretical_loss": 3.7913962804300088, + "tokens_seen": 730988544 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003931795386158475, + "loss": 1.3467, + "theoretical_loss": 3.791362291912982, + "tokens_seen": 731054080 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039316950852557675, + "loss": 1.3751, + "theoretical_loss": 3.791328307295804, + "tokens_seen": 731119616 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039315947843530593, + "loss": 1.3846, + "theoretical_loss": 3.791294326577677, + "tokens_seen": 731185152 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003931494483450351, + "loss": 1.455, + "theoretical_loss": 3.7912603497578043, + "tokens_seen": 731250688 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003931394182547643, + "loss": 1.4055, + "theoretical_loss": 3.7912263768353895, + "tokens_seen": 731316224 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039312938816449353, + "loss": 1.4481, + "theoretical_loss": 3.7911924078096364, + "tokens_seen": 731381760 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039311935807422266, + "loss": 1.4328, + "theoretical_loss": 3.791158442679748, + "tokens_seen": 731447296 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003931093279839519, + "loss": 1.3655, + "theoretical_loss": 3.7911244814449305, + "tokens_seen": 731512832 + }, + { + "epoch": 0.22, + "learning_rate": 0.000393099297893681, + "loss": 1.4975, + "theoretical_loss": 3.7910905241043866, + "tokens_seen": 731578368 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039308926780341026, + "loss": 1.3671, + "theoretical_loss": 3.7910565706573216, + "tokens_seen": 731643904 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039307923771313944, + "loss": 1.4776, + "theoretical_loss": 3.7910226211029405, + "tokens_seen": 731709440 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003930692076228686, + "loss": 1.3517, + "theoretical_loss": 3.790988675440448, + "tokens_seen": 731774976 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003930591775325978, + "loss": 1.4103, + "theoretical_loss": 3.79095473366905, + "tokens_seen": 731840512 + }, + { + "epoch": 0.22, + "learning_rate": 0.000393049147442327, + "loss": 1.3589, + "theoretical_loss": 3.790920795787952, + "tokens_seen": 731906048 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039303911735205616, + "loss": 1.3927, + "theoretical_loss": 3.7908868617963587, + "tokens_seen": 731971584 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003930290872617854, + "loss": 1.3659, + "theoretical_loss": 3.7908529316934776, + "tokens_seen": 732037120 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003930190571715145, + "loss": 1.5234, + "theoretical_loss": 3.7908190054785145, + "tokens_seen": 732102656 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039300902708124376, + "loss": 1.215, + "theoretical_loss": 3.7907850831506753, + "tokens_seen": 732168192 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003929989969909729, + "loss": 1.4212, + "theoretical_loss": 3.7907511647091683, + "tokens_seen": 732233728 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003929889669007021, + "loss": 1.4575, + "theoretical_loss": 3.7907172501531985, + "tokens_seen": 732299264 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003929789368104313, + "loss": 1.3656, + "theoretical_loss": 3.7906833394819746, + "tokens_seen": 732364800 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003929689067201605, + "loss": 1.4025, + "theoretical_loss": 3.790649432694704, + "tokens_seen": 732430336 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039295887662988967, + "loss": 1.3695, + "theoretical_loss": 3.7906155297905935, + "tokens_seen": 732495872 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003929488465396189, + "loss": 1.3544, + "theoretical_loss": 3.7905816307688514, + "tokens_seen": 732561408 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039293881644934803, + "loss": 1.3848, + "theoretical_loss": 3.790547735628686, + "tokens_seen": 732626944 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039292878635907726, + "loss": 1.3748, + "theoretical_loss": 3.790513844369306, + "tokens_seen": 732692480 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003929187562688064, + "loss": 1.3808, + "theoretical_loss": 3.7904799569899192, + "tokens_seen": 732758016 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003929087261785356, + "loss": 1.4546, + "theoretical_loss": 3.790446073489735, + "tokens_seen": 732823552 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003928986960882648, + "loss": 1.4187, + "theoretical_loss": 3.7904121938679625, + "tokens_seen": 732889088 + }, + { + "epoch": 0.22, + "learning_rate": 0.000392888665997994, + "loss": 1.3459, + "theoretical_loss": 3.7903783181238104, + "tokens_seen": 732954624 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039287863590772317, + "loss": 1.4057, + "theoretical_loss": 3.7903444462564893, + "tokens_seen": 733020160 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039286860581745235, + "loss": 1.388, + "theoretical_loss": 3.790310578265208, + "tokens_seen": 733085696 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039285857572718153, + "loss": 1.3203, + "theoretical_loss": 3.7902767141491776, + "tokens_seen": 733151232 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039284854563691077, + "loss": 1.284, + "theoretical_loss": 3.790242853907607, + "tokens_seen": 733216768 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003928385155466399, + "loss": 1.5068, + "theoretical_loss": 3.790208997539707, + "tokens_seen": 733282304 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039282848545636913, + "loss": 1.3518, + "theoretical_loss": 3.7901751450446897, + "tokens_seen": 733347840 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039281845536609825, + "loss": 1.2894, + "theoretical_loss": 3.790141296421764, + "tokens_seen": 733413376 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003928084252758275, + "loss": 1.4134, + "theoretical_loss": 3.790107451670142, + "tokens_seen": 733478912 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039279839518555667, + "loss": 1.4899, + "theoretical_loss": 3.790073610789036, + "tokens_seen": 733544448 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039278836509528585, + "loss": 1.4133, + "theoretical_loss": 3.7900397737776554, + "tokens_seen": 733609984 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039277833500501503, + "loss": 1.3166, + "theoretical_loss": 3.790005940635214, + "tokens_seen": 733675520 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039276830491474427, + "loss": 1.2746, + "theoretical_loss": 3.7899721113609237, + "tokens_seen": 733741056 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003927582748244734, + "loss": 1.396, + "theoretical_loss": 3.789938285953996, + "tokens_seen": 733806592 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039274824473420263, + "loss": 1.3556, + "theoretical_loss": 3.789904464413644, + "tokens_seen": 733872128 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039273821464393176, + "loss": 1.4012, + "theoretical_loss": 3.7898706467390806, + "tokens_seen": 733937664 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 249139, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.467482566833496, + "objective/train/theoretical_loss": 3.7898368329295176, + "objective/train/tokens_used": 734003200, + "theoretical_loss": 3.7898368329295176, + "tokens_seen": 734003200 + }, + { + "epoch": 0.22, + "learning_rate": 0.000392728184553661, + "loss": 1.4981, + "theoretical_loss": 3.7898368329295176, + "tokens_seen": 734003200 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003927181544633902, + "loss": 1.3563, + "theoretical_loss": 3.7898030229841697, + "tokens_seen": 734068736 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039270812437311936, + "loss": 1.3576, + "theoretical_loss": 3.78976921690225, + "tokens_seen": 734134272 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039269809428284854, + "loss": 1.3819, + "theoretical_loss": 3.789735414682972, + "tokens_seen": 734199808 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003926880641925777, + "loss": 1.4039, + "theoretical_loss": 3.7897016163255497, + "tokens_seen": 734265344 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003926780341023069, + "loss": 1.4137, + "theoretical_loss": 3.789667821829197, + "tokens_seen": 734330880 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039266800401203613, + "loss": 1.4139, + "theoretical_loss": 3.7896340311931285, + "tokens_seen": 734396416 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039265797392176526, + "loss": 1.4204, + "theoretical_loss": 3.789600244416559, + "tokens_seen": 734461952 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003926479438314945, + "loss": 1.3621, + "theoretical_loss": 3.789566461498703, + "tokens_seen": 734527488 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039263791374122373, + "loss": 1.3693, + "theoretical_loss": 3.789532682438776, + "tokens_seen": 734593024 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039262788365095286, + "loss": 1.3619, + "theoretical_loss": 3.789498907235993, + "tokens_seen": 734658560 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003926178535606821, + "loss": 1.4546, + "theoretical_loss": 3.7894651358895697, + "tokens_seen": 734724096 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003926078234704112, + "loss": 1.4317, + "theoretical_loss": 3.789431368398722, + "tokens_seen": 734789632 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039259779338014046, + "loss": 1.512, + "theoretical_loss": 3.789397604762666, + "tokens_seen": 734855168 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039258776328986964, + "loss": 1.4166, + "theoretical_loss": 3.789363844980617, + "tokens_seen": 734920704 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003925777331995988, + "loss": 1.3576, + "theoretical_loss": 3.7893300890517922, + "tokens_seen": 734986240 + }, + { + "epoch": 0.22, + "learning_rate": 0.000392567703109328, + "loss": 1.3987, + "theoretical_loss": 3.789296336975408, + "tokens_seen": 735051776 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003925576730190572, + "loss": 1.3913, + "theoretical_loss": 3.789262588750682, + "tokens_seen": 735117312 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039254764292878636, + "loss": 1.3452, + "theoretical_loss": 3.7892288443768303, + "tokens_seen": 735182848 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003925376128385156, + "loss": 1.348, + "theoretical_loss": 3.7891951038530713, + "tokens_seen": 735248384 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003925275827482447, + "loss": 1.3434, + "theoretical_loss": 3.7891613671786217, + "tokens_seen": 735313920 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039251755265797396, + "loss": 1.4409, + "theoretical_loss": 3.7891276343527, + "tokens_seen": 735379456 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003925075225677031, + "loss": 1.3987, + "theoretical_loss": 3.789093905374524, + "tokens_seen": 735444992 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003924974924774323, + "loss": 1.3208, + "theoretical_loss": 3.789060180243312, + "tokens_seen": 735510528 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003924874623871615, + "loss": 1.361, + "theoretical_loss": 3.7890264589582827, + "tokens_seen": 735576064 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003924774322968907, + "loss": 1.4898, + "theoretical_loss": 3.7889927415186544, + "tokens_seen": 735641600 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039246740220661987, + "loss": 1.318, + "theoretical_loss": 3.7889590279236467, + "tokens_seen": 735707136 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003924573721163491, + "loss": 1.3864, + "theoretical_loss": 3.788925318172478, + "tokens_seen": 735772672 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039244734202607823, + "loss": 1.3669, + "theoretical_loss": 3.7888916122643685, + "tokens_seen": 735838208 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039243731193580746, + "loss": 1.2807, + "theoretical_loss": 3.788857910198537, + "tokens_seen": 735903744 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003924272818455366, + "loss": 1.288, + "theoretical_loss": 3.7888242119742044, + "tokens_seen": 735969280 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003924172517552658, + "loss": 1.3563, + "theoretical_loss": 3.7887905175905905, + "tokens_seen": 736034816 + }, + { + "epoch": 0.22, + "learning_rate": 0.000392407221664995, + "loss": 1.5114, + "theoretical_loss": 3.7887568270469147, + "tokens_seen": 736100352 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003923971915747242, + "loss": 1.234, + "theoretical_loss": 3.788723140342399, + "tokens_seen": 736165888 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039238716148445337, + "loss": 1.4894, + "theoretical_loss": 3.7886894574762637, + "tokens_seen": 736231424 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039237713139418255, + "loss": 1.3829, + "theoretical_loss": 3.7886557784477293, + "tokens_seen": 736296960 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039236710130391173, + "loss": 1.3117, + "theoretical_loss": 3.7886221032560172, + "tokens_seen": 736362496 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039235707121364097, + "loss": 1.2713, + "theoretical_loss": 3.7885884319003496, + "tokens_seen": 736428032 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003923470411233701, + "loss": 1.3865, + "theoretical_loss": 3.7885547643799473, + "tokens_seen": 736493568 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039233701103309933, + "loss": 1.4063, + "theoretical_loss": 3.788521100694033, + "tokens_seen": 736559104 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039232698094282846, + "loss": 1.356, + "theoretical_loss": 3.7884874408418288, + "tokens_seen": 736624640 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003923169508525577, + "loss": 1.3111, + "theoretical_loss": 3.788453784822556, + "tokens_seen": 736690176 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039230692076228687, + "loss": 1.3962, + "theoretical_loss": 3.788420132635438, + "tokens_seen": 736755712 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039229689067201605, + "loss": 1.4022, + "theoretical_loss": 3.788386484279698, + "tokens_seen": 736821248 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039228686058174523, + "loss": 1.4418, + "theoretical_loss": 3.788352839754559, + "tokens_seen": 736886784 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039227683049147447, + "loss": 1.3951, + "theoretical_loss": 3.7883191990592433, + "tokens_seen": 736952320 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003922668004012036, + "loss": 1.3988, + "theoretical_loss": 3.788285562192976, + "tokens_seen": 737017856 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039225677031093283, + "loss": 1.446, + "theoretical_loss": 3.7882519291549794, + "tokens_seen": 737083392 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039224674022066196, + "loss": 1.3098, + "theoretical_loss": 3.788218299944478, + "tokens_seen": 737148928 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003922367101303912, + "loss": 1.3888, + "theoretical_loss": 3.7881846745606964, + "tokens_seen": 737214464 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 249996, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.385530948638916, + "objective/train/theoretical_loss": 3.788151053002858, + "objective/train/tokens_used": 737280000, + "theoretical_loss": 3.788151053002858, + "tokens_seen": 737280000 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003922266800401204, + "loss": 1.3681, + "theoretical_loss": 3.788151053002858, + "tokens_seen": 737280000 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039221664994984956, + "loss": 1.3515, + "theoretical_loss": 3.788117435270189, + "tokens_seen": 737345536 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039220661985957874, + "loss": 1.4308, + "theoretical_loss": 3.788083821361913, + "tokens_seen": 737411072 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003921965897693079, + "loss": 1.4227, + "theoretical_loss": 3.7880502112772554, + "tokens_seen": 737476608 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003921865596790371, + "loss": 1.3891, + "theoretical_loss": 3.788016605015442, + "tokens_seen": 737542144 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039217652958876633, + "loss": 1.473, + "theoretical_loss": 3.7879830025756975, + "tokens_seen": 737607680 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039216649949849546, + "loss": 1.403, + "theoretical_loss": 3.7879494039572483, + "tokens_seen": 737673216 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003921564694082247, + "loss": 1.4675, + "theoretical_loss": 3.787915809159321, + "tokens_seen": 737738752 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003921464393179539, + "loss": 1.3618, + "theoretical_loss": 3.7878822181811405, + "tokens_seen": 737804288 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039213640922768306, + "loss": 1.391, + "theoretical_loss": 3.7878486310219337, + "tokens_seen": 737869824 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039212637913741224, + "loss": 1.4359, + "theoretical_loss": 3.787815047680928, + "tokens_seen": 737935360 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003921163490471414, + "loss": 1.4099, + "theoretical_loss": 3.787781468157349, + "tokens_seen": 738000896 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003921063189568706, + "loss": 1.4213, + "theoretical_loss": 3.787747892450425, + "tokens_seen": 738066432 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039209628886659984, + "loss": 1.3283, + "theoretical_loss": 3.7877143205593837, + "tokens_seen": 738131968 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039208625877632896, + "loss": 1.3108, + "theoretical_loss": 3.787680752483451, + "tokens_seen": 738197504 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003920762286860582, + "loss": 1.4316, + "theoretical_loss": 3.787647188221856, + "tokens_seen": 738263040 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003920661985957873, + "loss": 1.3781, + "theoretical_loss": 3.787613627773826, + "tokens_seen": 738328576 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039205616850551656, + "loss": 1.4297, + "theoretical_loss": 3.7875800711385903, + "tokens_seen": 738394112 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039204613841524574, + "loss": 1.3776, + "theoretical_loss": 3.7875465183153767, + "tokens_seen": 738459648 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003920361083249749, + "loss": 1.504, + "theoretical_loss": 3.787512969303414, + "tokens_seen": 738525184 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003920260782347041, + "loss": 1.4061, + "theoretical_loss": 3.7874794241019307, + "tokens_seen": 738590720 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003920160481444333, + "loss": 1.4502, + "theoretical_loss": 3.7874458827101565, + "tokens_seen": 738656256 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039200601805416247, + "loss": 1.4464, + "theoretical_loss": 3.7874123451273203, + "tokens_seen": 738721792 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003919959879638917, + "loss": 1.4037, + "theoretical_loss": 3.787378811352652, + "tokens_seen": 738787328 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039198595787362083, + "loss": 1.408, + "theoretical_loss": 3.787345281385382, + "tokens_seen": 738852864 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039197592778335007, + "loss": 1.3825, + "theoretical_loss": 3.7873117552247395, + "tokens_seen": 738918400 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039196589769307925, + "loss": 1.5089, + "theoretical_loss": 3.7872782328699555, + "tokens_seen": 738983936 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039195586760280843, + "loss": 1.3792, + "theoretical_loss": 3.7872447143202597, + "tokens_seen": 739049472 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003919458375125376, + "loss": 1.3357, + "theoretical_loss": 3.787211199574883, + "tokens_seen": 739115008 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003919358074222668, + "loss": 1.3571, + "theoretical_loss": 3.787177688633057, + "tokens_seen": 739180544 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039192577733199597, + "loss": 1.3377, + "theoretical_loss": 3.7871441814940123, + "tokens_seen": 739246080 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003919157472417252, + "loss": 1.4104, + "theoretical_loss": 3.7871106781569805, + "tokens_seen": 739311616 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039190571715145433, + "loss": 1.4743, + "theoretical_loss": 3.7870771786211925, + "tokens_seen": 739377152 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039189568706118357, + "loss": 1.3114, + "theoretical_loss": 3.787043682885882, + "tokens_seen": 739442688 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039188565697091275, + "loss": 1.3917, + "theoretical_loss": 3.7870101909502787, + "tokens_seen": 739508224 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039187562688064193, + "loss": 1.39, + "theoretical_loss": 3.7869767028136163, + "tokens_seen": 739573760 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039186559679037117, + "loss": 1.4495, + "theoretical_loss": 3.7869432184751277, + "tokens_seen": 739639296 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003918555667001003, + "loss": 1.4224, + "theoretical_loss": 3.7869097379340446, + "tokens_seen": 739704832 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039184553660982953, + "loss": 1.3908, + "theoretical_loss": 3.7868762611896, + "tokens_seen": 739770368 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039183550651955866, + "loss": 1.3599, + "theoretical_loss": 3.786842788241028, + "tokens_seen": 739835904 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003918254764292879, + "loss": 1.4578, + "theoretical_loss": 3.7868093190875616, + "tokens_seen": 739901440 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039181544633901707, + "loss": 1.4355, + "theoretical_loss": 3.7867758537284337, + "tokens_seen": 739966976 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039180541624874625, + "loss": 1.2834, + "theoretical_loss": 3.786742392162879, + "tokens_seen": 740032512 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039179538615847543, + "loss": 1.394, + "theoretical_loss": 3.7867089343901315, + "tokens_seen": 740098048 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039178535606820467, + "loss": 1.312, + "theoretical_loss": 3.786675480409425, + "tokens_seen": 740163584 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003917753259779338, + "loss": 1.4162, + "theoretical_loss": 3.786642030219994, + "tokens_seen": 740229120 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039176529588766303, + "loss": 1.3481, + "theoretical_loss": 3.7866085838210735, + "tokens_seen": 740294656 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039175526579739216, + "loss": 1.4782, + "theoretical_loss": 3.786575141211899, + "tokens_seen": 740360192 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003917452357071214, + "loss": 1.3143, + "theoretical_loss": 3.7865417023917054, + "tokens_seen": 740425728 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003917352056168506, + "loss": 1.3834, + "theoretical_loss": 3.7865082673597277, + "tokens_seen": 740491264 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 251178, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3643934726715088, + "objective/train/theoretical_loss": 3.786474836115201, + "objective/train/tokens_used": 740556800, + "theoretical_loss": 3.786474836115201, + "tokens_seen": 740556800 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039172517552657976, + "loss": 1.3058, + "theoretical_loss": 3.786474836115201, + "tokens_seen": 740556800 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039171514543630894, + "loss": 1.3447, + "theoretical_loss": 3.786441408657363, + "tokens_seen": 740622336 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003917051153460381, + "loss": 1.3876, + "theoretical_loss": 3.7864079849854475, + "tokens_seen": 740687872 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003916950852557673, + "loss": 1.4281, + "theoretical_loss": 3.7863745650986926, + "tokens_seen": 740753408 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039168505516549653, + "loss": 1.3824, + "theoretical_loss": 3.7863411489963346, + "tokens_seen": 740818944 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039167502507522566, + "loss": 1.4265, + "theoretical_loss": 3.786307736677609, + "tokens_seen": 740884480 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003916649949849549, + "loss": 1.4244, + "theoretical_loss": 3.7862743281417535, + "tokens_seen": 740950016 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003916549648946841, + "loss": 1.383, + "theoretical_loss": 3.786240923388006, + "tokens_seen": 741015552 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039164493480441326, + "loss": 1.4171, + "theoretical_loss": 3.7862075224156024, + "tokens_seen": 741081088 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039163490471414244, + "loss": 1.5193, + "theoretical_loss": 3.7861741252237815, + "tokens_seen": 741146624 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003916248746238716, + "loss": 1.3546, + "theoretical_loss": 3.7861407318117815, + "tokens_seen": 741212160 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003916148445336008, + "loss": 1.4504, + "theoretical_loss": 3.7861073421788385, + "tokens_seen": 741277696 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039160481444333004, + "loss": 1.378, + "theoretical_loss": 3.786073956324193, + "tokens_seen": 741343232 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039159478435305916, + "loss": 1.3242, + "theoretical_loss": 3.786040574247082, + "tokens_seen": 741408768 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003915847542627884, + "loss": 1.4085, + "theoretical_loss": 3.7860071959467447, + "tokens_seen": 741474304 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003915747241725175, + "loss": 1.4203, + "theoretical_loss": 3.785973821422421, + "tokens_seen": 741539840 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039156469408224676, + "loss": 1.3918, + "theoretical_loss": 3.7859404506733485, + "tokens_seen": 741605376 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039155466399197594, + "loss": 1.2864, + "theoretical_loss": 3.7859070836987674, + "tokens_seen": 741670912 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003915446339017051, + "loss": 1.4595, + "theoretical_loss": 3.785873720497917, + "tokens_seen": 741736448 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003915346038114343, + "loss": 1.4221, + "theoretical_loss": 3.7858403610700373, + "tokens_seen": 741801984 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003915245737211635, + "loss": 1.4099, + "theoretical_loss": 3.7858070054143687, + "tokens_seen": 741867520 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039151454363089267, + "loss": 1.3991, + "theoretical_loss": 3.7857736535301507, + "tokens_seen": 741933056 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003915045135406219, + "loss": 1.4165, + "theoretical_loss": 3.785740305416624, + "tokens_seen": 741998592 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039149448345035103, + "loss": 1.4158, + "theoretical_loss": 3.7857069610730303, + "tokens_seen": 742064128 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039148445336008027, + "loss": 1.3598, + "theoretical_loss": 3.785673620498609, + "tokens_seen": 742129664 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039147442326980945, + "loss": 1.4072, + "theoretical_loss": 3.7856402836926026, + "tokens_seen": 742195200 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039146439317953863, + "loss": 1.4271, + "theoretical_loss": 3.785606950654252, + "tokens_seen": 742260736 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003914543630892678, + "loss": 1.3911, + "theoretical_loss": 3.785573621382798, + "tokens_seen": 742326272 + }, + { + "epoch": 0.22, + "learning_rate": 0.000391444332998997, + "loss": 1.3969, + "theoretical_loss": 3.7855402958774835, + "tokens_seen": 742391808 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039143430290872617, + "loss": 1.377, + "theoretical_loss": 3.78550697413755, + "tokens_seen": 742457344 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003914242728184554, + "loss": 1.3457, + "theoretical_loss": 3.7854736561622397, + "tokens_seen": 742522880 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039141424272818453, + "loss": 1.4568, + "theoretical_loss": 3.785440341950795, + "tokens_seen": 742588416 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039140421263791377, + "loss": 1.4001, + "theoretical_loss": 3.7854070315024595, + "tokens_seen": 742653952 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003913941825476429, + "loss": 1.4644, + "theoretical_loss": 3.7853737248164743, + "tokens_seen": 742719488 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039138415245737213, + "loss": 1.362, + "theoretical_loss": 3.7853404218920845, + "tokens_seen": 742785024 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003913741223671013, + "loss": 1.3302, + "theoretical_loss": 3.7853071227285326, + "tokens_seen": 742850560 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003913640922768305, + "loss": 1.3213, + "theoretical_loss": 3.785273827325062, + "tokens_seen": 742916096 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003913540621865597, + "loss": 1.4402, + "theoretical_loss": 3.7852405356809165, + "tokens_seen": 742981632 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039134403209628886, + "loss": 1.4125, + "theoretical_loss": 3.78520724779534, + "tokens_seen": 743047168 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039133400200601804, + "loss": 1.4453, + "theoretical_loss": 3.785173963667577, + "tokens_seen": 743112704 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039132397191574727, + "loss": 1.4303, + "theoretical_loss": 3.785140683296872, + "tokens_seen": 743178240 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003913139418254764, + "loss": 1.4748, + "theoretical_loss": 3.7851074066824695, + "tokens_seen": 743243776 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039130391173520563, + "loss": 1.4784, + "theoretical_loss": 3.7850741338236142, + "tokens_seen": 743309312 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003912938816449348, + "loss": 1.4292, + "theoretical_loss": 3.7850408647195515, + "tokens_seen": 743374848 + }, + { + "epoch": 0.23, + "learning_rate": 0.000391283851554664, + "loss": 1.4024, + "theoretical_loss": 3.7850075993695267, + "tokens_seen": 743440384 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003912738214643932, + "loss": 1.3962, + "theoretical_loss": 3.7849743377727854, + "tokens_seen": 743505920 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039126379137412236, + "loss": 1.459, + "theoretical_loss": 3.784941079928573, + "tokens_seen": 743571456 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039125376128385154, + "loss": 1.4527, + "theoretical_loss": 3.7849078258361355, + "tokens_seen": 743636992 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003912437311935808, + "loss": 1.5066, + "theoretical_loss": 3.784874575494719, + "tokens_seen": 743702528 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003912337011033099, + "loss": 1.4767, + "theoretical_loss": 3.7848413289035703, + "tokens_seen": 743768064 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 252451, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4141029119491577, + "objective/train/theoretical_loss": 3.784808086061936, + "objective/train/tokens_used": 743833600, + "theoretical_loss": 3.784808086061936, + "tokens_seen": 743833600 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039122367101303914, + "loss": 1.4301, + "theoretical_loss": 3.784808086061936, + "tokens_seen": 743833600 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039121364092276826, + "loss": 1.4715, + "theoretical_loss": 3.7847748469690625, + "tokens_seen": 743899136 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003912036108324975, + "loss": 1.4844, + "theoretical_loss": 3.784741611624198, + "tokens_seen": 743964672 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003911935807422267, + "loss": 1.43, + "theoretical_loss": 3.784708380026588, + "tokens_seen": 744030208 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039118355065195586, + "loss": 1.3889, + "theoretical_loss": 3.784675152175481, + "tokens_seen": 744095744 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039117352056168504, + "loss": 1.3305, + "theoretical_loss": 3.7846419280701245, + "tokens_seen": 744161280 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003911634904714143, + "loss": 1.3879, + "theoretical_loss": 3.7846087077097668, + "tokens_seen": 744226816 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003911534603811434, + "loss": 1.3324, + "theoretical_loss": 3.7845754910936558, + "tokens_seen": 744292352 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039114343029087264, + "loss": 1.4291, + "theoretical_loss": 3.784542278221039, + "tokens_seen": 744357888 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003911334002006018, + "loss": 1.3663, + "theoretical_loss": 3.784509069091167, + "tokens_seen": 744423424 + }, + { + "epoch": 0.23, + "learning_rate": 0.000391123370110331, + "loss": 1.4046, + "theoretical_loss": 3.7844758637032867, + "tokens_seen": 744488960 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039111334002006024, + "loss": 1.4257, + "theoretical_loss": 3.784442662056648, + "tokens_seen": 744554496 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039110330992978936, + "loss": 1.3288, + "theoretical_loss": 3.7844094641504995, + "tokens_seen": 744620032 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003910932798395186, + "loss": 1.3773, + "theoretical_loss": 3.784376269984092, + "tokens_seen": 744685568 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039108324974924773, + "loss": 1.4308, + "theoretical_loss": 3.784343079556673, + "tokens_seen": 744751104 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039107321965897696, + "loss": 1.335, + "theoretical_loss": 3.784309892867494, + "tokens_seen": 744816640 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039106318956870614, + "loss": 1.4062, + "theoretical_loss": 3.784276709915805, + "tokens_seen": 744882176 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003910531594784353, + "loss": 1.3953, + "theoretical_loss": 3.7842435307008557, + "tokens_seen": 744947712 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003910431293881645, + "loss": 1.4464, + "theoretical_loss": 3.784210355221897, + "tokens_seen": 745013248 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003910330992978937, + "loss": 1.3493, + "theoretical_loss": 3.78417718347818, + "tokens_seen": 745078784 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039102306920762287, + "loss": 1.261, + "theoretical_loss": 3.7841440154689545, + "tokens_seen": 745144320 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003910130391173521, + "loss": 1.3801, + "theoretical_loss": 3.7841108511934727, + "tokens_seen": 745209856 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039100300902708123, + "loss": 1.4483, + "theoretical_loss": 3.7840776906509856, + "tokens_seen": 745275392 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039099297893681047, + "loss": 1.4193, + "theoretical_loss": 3.784044533840745, + "tokens_seen": 745340928 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039098294884653965, + "loss": 1.4038, + "theoretical_loss": 3.784011380762003, + "tokens_seen": 745406464 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039097291875626883, + "loss": 1.27, + "theoretical_loss": 3.7839782314140105, + "tokens_seen": 745472000 + }, + { + "epoch": 0.23, + "learning_rate": 0.000390962888665998, + "loss": 1.514, + "theoretical_loss": 3.78394508579602, + "tokens_seen": 745537536 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003909528585757272, + "loss": 1.4472, + "theoretical_loss": 3.7839119439072855, + "tokens_seen": 745603072 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039094282848545637, + "loss": 1.3812, + "theoretical_loss": 3.7838788057470585, + "tokens_seen": 745668608 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003909327983951856, + "loss": 1.2963, + "theoretical_loss": 3.783845671314592, + "tokens_seen": 745734144 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039092276830491473, + "loss": 1.3042, + "theoretical_loss": 3.783812540609139, + "tokens_seen": 745799680 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039091273821464397, + "loss": 1.3459, + "theoretical_loss": 3.783779413629953, + "tokens_seen": 745865216 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003909027081243731, + "loss": 1.4605, + "theoretical_loss": 3.7837462903762873, + "tokens_seen": 745930752 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039089267803410233, + "loss": 1.4162, + "theoretical_loss": 3.7837131708473963, + "tokens_seen": 745996288 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003908826479438315, + "loss": 1.4311, + "theoretical_loss": 3.7836800550425336, + "tokens_seen": 746061824 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003908726178535607, + "loss": 1.313, + "theoretical_loss": 3.783646942960953, + "tokens_seen": 746127360 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003908625877632899, + "loss": 1.4363, + "theoretical_loss": 3.7836138346019093, + "tokens_seen": 746192896 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039085255767301906, + "loss": 1.3603, + "theoretical_loss": 3.783580729964657, + "tokens_seen": 746258432 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039084252758274824, + "loss": 1.3821, + "theoretical_loss": 3.783547629048451, + "tokens_seen": 746323968 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039083249749247747, + "loss": 1.3633, + "theoretical_loss": 3.7835145318525463, + "tokens_seen": 746389504 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003908224674022066, + "loss": 1.4103, + "theoretical_loss": 3.7834814383761985, + "tokens_seen": 746455040 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039081243731193583, + "loss": 1.3779, + "theoretical_loss": 3.7834483486186627, + "tokens_seen": 746520576 + }, + { + "epoch": 0.23, + "learning_rate": 0.000390802407221665, + "loss": 1.3387, + "theoretical_loss": 3.7834152625791946, + "tokens_seen": 746586112 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003907923771313942, + "loss": 1.3886, + "theoretical_loss": 3.7833821802570506, + "tokens_seen": 746651648 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003907823470411234, + "loss": 1.4619, + "theoretical_loss": 3.7833491016514857, + "tokens_seen": 746717184 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039077231695085256, + "loss": 1.4617, + "theoretical_loss": 3.7833160267617574, + "tokens_seen": 746782720 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039076228686058174, + "loss": 1.4977, + "theoretical_loss": 3.783282955587122, + "tokens_seen": 746848256 + }, + { + "epoch": 0.23, + "learning_rate": 0.000390752256770311, + "loss": 1.4715, + "theoretical_loss": 3.7832498881268357, + "tokens_seen": 746913792 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003907422266800401, + "loss": 1.4068, + "theoretical_loss": 3.7832168243801556, + "tokens_seen": 746979328 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039073219658976934, + "loss": 1.3814, + "theoretical_loss": 3.7831837643463393, + "tokens_seen": 747044864 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 253356, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5134073495864868, + "objective/train/theoretical_loss": 3.7831507080246443, + "objective/train/tokens_used": 747110400, + "theoretical_loss": 3.7831507080246443, + "tokens_seen": 747110400 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039072216649949846, + "loss": 1.4266, + "theoretical_loss": 3.7831507080246443, + "tokens_seen": 747110400 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003907121364092277, + "loss": 1.4299, + "theoretical_loss": 3.783117655414328, + "tokens_seen": 747175936 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003907021063189569, + "loss": 1.3358, + "theoretical_loss": 3.783084606514648, + "tokens_seen": 747241472 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039069207622868606, + "loss": 1.4638, + "theoretical_loss": 3.7830515613248625, + "tokens_seen": 747307008 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039068204613841524, + "loss": 1.4485, + "theoretical_loss": 3.7830185198442297, + "tokens_seen": 747372544 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003906720160481445, + "loss": 1.4193, + "theoretical_loss": 3.782985482072008, + "tokens_seen": 747438080 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003906619859578736, + "loss": 1.3998, + "theoretical_loss": 3.782952448007457, + "tokens_seen": 747503616 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039065195586760284, + "loss": 1.4945, + "theoretical_loss": 3.7829194176498335, + "tokens_seen": 747569152 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039064192577733197, + "loss": 1.3377, + "theoretical_loss": 3.7828863909983994, + "tokens_seen": 747634688 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003906318956870612, + "loss": 1.3899, + "theoretical_loss": 3.7828533680524115, + "tokens_seen": 747700224 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003906218655967904, + "loss": 1.5016, + "theoretical_loss": 3.782820348811131, + "tokens_seen": 747765760 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039061183550651957, + "loss": 1.3929, + "theoretical_loss": 3.7827873332738164, + "tokens_seen": 747831296 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039060180541624875, + "loss": 1.3225, + "theoretical_loss": 3.782754321439729, + "tokens_seen": 747896832 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039059177532597793, + "loss": 1.3595, + "theoretical_loss": 3.7827213133081283, + "tokens_seen": 747962368 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003905817452357071, + "loss": 1.3391, + "theoretical_loss": 3.7826883088782743, + "tokens_seen": 748027904 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039057171514543634, + "loss": 1.4961, + "theoretical_loss": 3.782655308149428, + "tokens_seen": 748093440 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039056168505516547, + "loss": 1.4564, + "theoretical_loss": 3.78262231112085, + "tokens_seen": 748158976 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003905516549648947, + "loss": 1.5219, + "theoretical_loss": 3.7825893177918015, + "tokens_seen": 748224512 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039054162487462383, + "loss": 1.3934, + "theoretical_loss": 3.7825563281615446, + "tokens_seen": 748290048 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039053159478435307, + "loss": 1.4822, + "theoretical_loss": 3.7825233422293394, + "tokens_seen": 748355584 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039052156469408225, + "loss": 1.3994, + "theoretical_loss": 3.7824903599944477, + "tokens_seen": 748421120 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039051153460381143, + "loss": 1.4383, + "theoretical_loss": 3.7824573814561324, + "tokens_seen": 748486656 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003905015045135406, + "loss": 1.4094, + "theoretical_loss": 3.782424406613654, + "tokens_seen": 748552192 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039049147442326985, + "loss": 1.3989, + "theoretical_loss": 3.782391435466277, + "tokens_seen": 748617728 + }, + { + "epoch": 0.23, + "learning_rate": 0.000390481444332999, + "loss": 1.4175, + "theoretical_loss": 3.7823584680132614, + "tokens_seen": 748683264 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003904714142427282, + "loss": 1.3936, + "theoretical_loss": 3.7823255042538717, + "tokens_seen": 748748800 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039046138415245734, + "loss": 1.3856, + "theoretical_loss": 3.78229254418737, + "tokens_seen": 748814336 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039045135406218657, + "loss": 1.4559, + "theoretical_loss": 3.7822595878130203, + "tokens_seen": 748879872 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039044132397191575, + "loss": 1.4306, + "theoretical_loss": 3.782226635130085, + "tokens_seen": 748945408 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039043129388164493, + "loss": 1.343, + "theoretical_loss": 3.782193686137828, + "tokens_seen": 749010944 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003904212637913741, + "loss": 1.3748, + "theoretical_loss": 3.782160740835513, + "tokens_seen": 749076480 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003904112337011033, + "loss": 1.4101, + "theoretical_loss": 3.7821277992224043, + "tokens_seen": 749142016 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003904012036108325, + "loss": 1.3787, + "theoretical_loss": 3.7820948612977663, + "tokens_seen": 749207552 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003903911735205617, + "loss": 1.3918, + "theoretical_loss": 3.7820619270608624, + "tokens_seen": 749273088 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003903811434302909, + "loss": 1.3435, + "theoretical_loss": 3.782028996510958, + "tokens_seen": 749338624 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003903711133400201, + "loss": 1.2771, + "theoretical_loss": 3.781996069647318, + "tokens_seen": 749404160 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039036108324974926, + "loss": 1.1772, + "theoretical_loss": 3.781963146469207, + "tokens_seen": 749469696 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039035105315947844, + "loss": 1.3901, + "theoretical_loss": 3.7819302269758905, + "tokens_seen": 749535232 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039034102306920767, + "loss": 1.4192, + "theoretical_loss": 3.7818973111666345, + "tokens_seen": 749600768 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003903309929789368, + "loss": 1.4613, + "theoretical_loss": 3.7818643990407037, + "tokens_seen": 749666304 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039032096288866603, + "loss": 1.3513, + "theoretical_loss": 3.7818314905973645, + "tokens_seen": 749731840 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003903109327983952, + "loss": 1.3124, + "theoretical_loss": 3.7817985858358827, + "tokens_seen": 749797376 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003903009027081244, + "loss": 1.3367, + "theoretical_loss": 3.781765684755525, + "tokens_seen": 749862912 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003902908726178536, + "loss": 1.4357, + "theoretical_loss": 3.7817327873555575, + "tokens_seen": 749928448 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039028084252758276, + "loss": 1.3825, + "theoretical_loss": 3.7816998936352473, + "tokens_seen": 749993984 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039027081243731194, + "loss": 1.4117, + "theoretical_loss": 3.7816670035938618, + "tokens_seen": 750059520 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003902607823470412, + "loss": 1.4043, + "theoretical_loss": 3.7816341172306664, + "tokens_seen": 750125056 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003902507522567703, + "loss": 1.4642, + "theoretical_loss": 3.781601234544931, + "tokens_seen": 750190592 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039024072216649954, + "loss": 1.4127, + "theoretical_loss": 3.7815683555359207, + "tokens_seen": 750256128 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039023069207622866, + "loss": 1.3207, + "theoretical_loss": 3.781535480202905, + "tokens_seen": 750321664 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 254550, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3803294897079468, + "objective/train/theoretical_loss": 3.781502608545151, + "objective/train/tokens_used": 750387200, + "theoretical_loss": 3.781502608545151, + "tokens_seen": 750387200 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003902206619859579, + "loss": 1.4654, + "theoretical_loss": 3.781502608545151, + "tokens_seen": 750387200 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003902106318956871, + "loss": 1.3578, + "theoretical_loss": 3.781469740561927, + "tokens_seen": 750452736 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039020060180541626, + "loss": 1.5046, + "theoretical_loss": 3.7814368762525015, + "tokens_seen": 750518272 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039019057171514544, + "loss": 1.4223, + "theoretical_loss": 3.7814040156161433, + "tokens_seen": 750583808 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003901805416248747, + "loss": 1.4477, + "theoretical_loss": 3.7813711586521217, + "tokens_seen": 750649344 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003901705115346038, + "loss": 1.5326, + "theoretical_loss": 3.7813383053597045, + "tokens_seen": 750714880 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039016048144433304, + "loss": 1.4122, + "theoretical_loss": 3.7813054557381616, + "tokens_seen": 750780416 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039015045135406217, + "loss": 1.3212, + "theoretical_loss": 3.7812726097867624, + "tokens_seen": 750845952 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003901404212637914, + "loss": 1.5419, + "theoretical_loss": 3.781239767504777, + "tokens_seen": 750911488 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003901303911735206, + "loss": 1.3476, + "theoretical_loss": 3.781206928891475, + "tokens_seen": 750977024 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039012036108324977, + "loss": 1.4243, + "theoretical_loss": 3.7811740939461256, + "tokens_seen": 751042560 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039011033099297895, + "loss": 1.406, + "theoretical_loss": 3.781141262668, + "tokens_seen": 751108096 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039010030090270813, + "loss": 1.3292, + "theoretical_loss": 3.7811084350563693, + "tokens_seen": 751173632 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003900902708124373, + "loss": 1.4732, + "theoretical_loss": 3.781075611110503, + "tokens_seen": 751239168 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039008024072216654, + "loss": 1.4164, + "theoretical_loss": 3.781042790829672, + "tokens_seen": 751304704 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039007021063189567, + "loss": 1.4026, + "theoretical_loss": 3.7810099742131484, + "tokens_seen": 751370240 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003900601805416249, + "loss": 1.4474, + "theoretical_loss": 3.7809771612602034, + "tokens_seen": 751435776 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039005015045135403, + "loss": 1.3871, + "theoretical_loss": 3.780944351970107, + "tokens_seen": 751501312 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039004012036108327, + "loss": 1.5453, + "theoretical_loss": 3.7809115463421334, + "tokens_seen": 751566848 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039003009027081245, + "loss": 1.4489, + "theoretical_loss": 3.7808787443755527, + "tokens_seen": 751632384 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039002006018054163, + "loss": 1.3718, + "theoretical_loss": 3.780845946069638, + "tokens_seen": 751697920 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003900100300902708, + "loss": 1.4453, + "theoretical_loss": 3.7808131514236614, + "tokens_seen": 751763456 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039000000000000005, + "loss": 1.5507, + "theoretical_loss": 3.7807803604368946, + "tokens_seen": 751828992 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003899899699097292, + "loss": 1.3485, + "theoretical_loss": 3.780747573108612, + "tokens_seen": 751894528 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003899799398194584, + "loss": 1.2957, + "theoretical_loss": 3.7807147894380853, + "tokens_seen": 751960064 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038996990972918754, + "loss": 1.4385, + "theoretical_loss": 3.7806820094245888, + "tokens_seen": 752025600 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038995987963891677, + "loss": 1.4945, + "theoretical_loss": 3.780649233067395, + "tokens_seen": 752091136 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038994984954864595, + "loss": 1.2932, + "theoretical_loss": 3.7806164603657777, + "tokens_seen": 752156672 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038993981945837513, + "loss": 1.4088, + "theoretical_loss": 3.780583691319011, + "tokens_seen": 752222208 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003899297893681043, + "loss": 1.4552, + "theoretical_loss": 3.7805509259263688, + "tokens_seen": 752287744 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003899197592778335, + "loss": 1.3965, + "theoretical_loss": 3.7805181641871255, + "tokens_seen": 752353280 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003899097291875627, + "loss": 1.5497, + "theoretical_loss": 3.7804854061005555, + "tokens_seen": 752418816 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003898996990972919, + "loss": 1.4031, + "theoretical_loss": 3.780452651665933, + "tokens_seen": 752484352 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038988966900702104, + "loss": 1.43, + "theoretical_loss": 3.780419900882533, + "tokens_seen": 752549888 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003898796389167503, + "loss": 1.45, + "theoretical_loss": 3.780387153749632, + "tokens_seen": 752615424 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003898696088264794, + "loss": 1.4972, + "theoretical_loss": 3.780354410266503, + "tokens_seen": 752680960 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038985957873620864, + "loss": 1.3592, + "theoretical_loss": 3.780321670432423, + "tokens_seen": 752746496 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003898495486459378, + "loss": 1.4263, + "theoretical_loss": 3.7802889342466672, + "tokens_seen": 752812032 + }, + { + "epoch": 0.23, + "learning_rate": 0.000389839518555667, + "loss": 1.5012, + "theoretical_loss": 3.7802562017085117, + "tokens_seen": 752877568 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003898294884653962, + "loss": 1.4646, + "theoretical_loss": 3.780223472817233, + "tokens_seen": 752943104 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003898194583751254, + "loss": 1.3714, + "theoretical_loss": 3.7801907475721057, + "tokens_seen": 753008640 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038980942828485454, + "loss": 1.3283, + "theoretical_loss": 3.7801580259724084, + "tokens_seen": 753074176 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003897993981945838, + "loss": 1.4748, + "theoretical_loss": 3.780125308017417, + "tokens_seen": 753139712 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003897893681043129, + "loss": 1.4904, + "theoretical_loss": 3.780092593706408, + "tokens_seen": 753205248 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038977933801404214, + "loss": 1.3194, + "theoretical_loss": 3.7800598830386587, + "tokens_seen": 753270784 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003897693079237713, + "loss": 1.4855, + "theoretical_loss": 3.780027176013447, + "tokens_seen": 753336320 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003897592778335005, + "loss": 1.5177, + "theoretical_loss": 3.7799944726300505, + "tokens_seen": 753401856 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003897492477432297, + "loss": 1.3088, + "theoretical_loss": 3.7799617728877464, + "tokens_seen": 753467392 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038973921765295886, + "loss": 1.4286, + "theoretical_loss": 3.7799290767858125, + "tokens_seen": 753532928 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038972918756268805, + "loss": 1.3159, + "theoretical_loss": 3.7798963843235276, + "tokens_seen": 753598464 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 255485, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4519758224487305, + "objective/train/theoretical_loss": 3.77986369550017, + "objective/train/tokens_used": 753664000, + "theoretical_loss": 3.77986369550017, + "tokens_seen": 753664000 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003897191574724173, + "loss": 1.3433, + "theoretical_loss": 3.77986369550017, + "tokens_seen": 753664000 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003897091273821464, + "loss": 1.4166, + "theoretical_loss": 3.779831010315018, + "tokens_seen": 753729536 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038969909729187564, + "loss": 1.4327, + "theoretical_loss": 3.7797983287673502, + "tokens_seen": 753795072 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038968906720160477, + "loss": 1.3968, + "theoretical_loss": 3.7797656508564463, + "tokens_seen": 753860608 + }, + { + "epoch": 0.23, + "learning_rate": 0.000389679037111334, + "loss": 1.4285, + "theoretical_loss": 3.7797329765815846, + "tokens_seen": 753926144 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003896690070210632, + "loss": 1.3057, + "theoretical_loss": 3.779700305942045, + "tokens_seen": 753991680 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038965897693079237, + "loss": 1.3207, + "theoretical_loss": 3.7796676389371076, + "tokens_seen": 754057216 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038964894684052155, + "loss": 1.3815, + "theoretical_loss": 3.7796349755660517, + "tokens_seen": 754122752 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003896389167502508, + "loss": 1.3881, + "theoretical_loss": 3.7796023158281566, + "tokens_seen": 754188288 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038962888665997997, + "loss": 1.4348, + "theoretical_loss": 3.7795696597227035, + "tokens_seen": 754253824 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038961885656970915, + "loss": 1.435, + "theoretical_loss": 3.7795370072489733, + "tokens_seen": 754319360 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038960882647943833, + "loss": 1.2438, + "theoretical_loss": 3.7795043584062453, + "tokens_seen": 754384896 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003895987963891675, + "loss": 1.5209, + "theoretical_loss": 3.779471713193801, + "tokens_seen": 754450432 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038958876629889674, + "loss": 1.5048, + "theoretical_loss": 3.7794390716109216, + "tokens_seen": 754515968 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038957873620862587, + "loss": 1.4843, + "theoretical_loss": 3.779406433656888, + "tokens_seen": 754581504 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003895687061183551, + "loss": 1.393, + "theoretical_loss": 3.7793737993309824, + "tokens_seen": 754647040 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038955867602808423, + "loss": 1.4144, + "theoretical_loss": 3.7793411686324854, + "tokens_seen": 754712576 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038954864593781347, + "loss": 1.475, + "theoretical_loss": 3.7793085415606793, + "tokens_seen": 754778112 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038953861584754265, + "loss": 1.3139, + "theoretical_loss": 3.7792759181148465, + "tokens_seen": 754843648 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038952858575727183, + "loss": 1.3769, + "theoretical_loss": 3.7792432982942685, + "tokens_seen": 754909184 + }, + { + "epoch": 0.23, + "learning_rate": 0.000389518555667001, + "loss": 1.4759, + "theoretical_loss": 3.7792106820982285, + "tokens_seen": 754974720 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038950852557673025, + "loss": 1.4008, + "theoretical_loss": 3.779178069526009, + "tokens_seen": 755040256 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003894984954864594, + "loss": 1.4137, + "theoretical_loss": 3.779145460576893, + "tokens_seen": 755105792 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003894884653961886, + "loss": 1.3653, + "theoretical_loss": 3.7791128552501636, + "tokens_seen": 755171328 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038947843530591774, + "loss": 1.3571, + "theoretical_loss": 3.7790802535451036, + "tokens_seen": 755236864 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038946840521564697, + "loss": 1.3609, + "theoretical_loss": 3.779047655460997, + "tokens_seen": 755302400 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038945837512537615, + "loss": 1.4155, + "theoretical_loss": 3.779015060997127, + "tokens_seen": 755367936 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038944834503510533, + "loss": 1.5278, + "theoretical_loss": 3.778982470152778, + "tokens_seen": 755433472 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003894383149448345, + "loss": 1.3595, + "theoretical_loss": 3.7789498829272343, + "tokens_seen": 755499008 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003894282848545637, + "loss": 1.3541, + "theoretical_loss": 3.7789172993197795, + "tokens_seen": 755564544 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003894182547642929, + "loss": 1.2885, + "theoretical_loss": 3.7788847193296986, + "tokens_seen": 755630080 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003894082246740221, + "loss": 1.3412, + "theoretical_loss": 3.7788521429562767, + "tokens_seen": 755695616 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038939819458375124, + "loss": 1.4608, + "theoretical_loss": 3.778819570198798, + "tokens_seen": 755761152 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003893881644934805, + "loss": 1.4022, + "theoretical_loss": 3.778787001056547, + "tokens_seen": 755826688 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003893781344032096, + "loss": 1.445, + "theoretical_loss": 3.7787544355288105, + "tokens_seen": 755892224 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038936810431293884, + "loss": 1.3844, + "theoretical_loss": 3.7787218736148738, + "tokens_seen": 755957760 + }, + { + "epoch": 0.23, + "learning_rate": 0.000389358074222668, + "loss": 1.4381, + "theoretical_loss": 3.778689315314022, + "tokens_seen": 756023296 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003893480441323972, + "loss": 1.3647, + "theoretical_loss": 3.778656760625541, + "tokens_seen": 756088832 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003893380140421264, + "loss": 1.4698, + "theoretical_loss": 3.7786242095487177, + "tokens_seen": 756154368 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003893279839518556, + "loss": 1.4742, + "theoretical_loss": 3.778591662082838, + "tokens_seen": 756219904 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038931795386158474, + "loss": 1.5609, + "theoretical_loss": 3.7785591182271885, + "tokens_seen": 756285440 + }, + { + "epoch": 0.23, + "learning_rate": 0.000389307923771314, + "loss": 1.2722, + "theoretical_loss": 3.7785265779810553, + "tokens_seen": 756350976 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003892978936810431, + "loss": 1.5042, + "theoretical_loss": 3.778494041343727, + "tokens_seen": 756416512 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038928786359077234, + "loss": 1.5262, + "theoretical_loss": 3.778461508314489, + "tokens_seen": 756482048 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003892778335005015, + "loss": 1.4147, + "theoretical_loss": 3.7784289788926295, + "tokens_seen": 756547584 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003892678034102307, + "loss": 1.4673, + "theoretical_loss": 3.778396453077436, + "tokens_seen": 756613120 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003892577733199599, + "loss": 1.3401, + "theoretical_loss": 3.778363930868196, + "tokens_seen": 756678656 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038924774322968906, + "loss": 1.4492, + "theoretical_loss": 3.778331412264198, + "tokens_seen": 756744192 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038923771313941825, + "loss": 1.4446, + "theoretical_loss": 3.7782988972647296, + "tokens_seen": 756809728 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003892276830491475, + "loss": 1.4594, + "theoretical_loss": 3.7782663858690797, + "tokens_seen": 756875264 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 256591, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1814011335372925, + "objective/train/theoretical_loss": 3.778233878076536, + "objective/train/tokens_used": 756940800, + "theoretical_loss": 3.778233878076536, + "tokens_seen": 756940800 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003892176529588766, + "loss": 1.2445, + "theoretical_loss": 3.778233878076536, + "tokens_seen": 756940800 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038920762286860584, + "loss": 1.3146, + "theoretical_loss": 3.7782013738863887, + "tokens_seen": 757006336 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038919759277833497, + "loss": 1.467, + "theoretical_loss": 3.7781688732979255, + "tokens_seen": 757071872 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003891875626880642, + "loss": 1.4842, + "theoretical_loss": 3.778136376310436, + "tokens_seen": 757137408 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003891775325977934, + "loss": 1.4537, + "theoretical_loss": 3.77810388292321, + "tokens_seen": 757202944 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038916750250752257, + "loss": 1.3155, + "theoretical_loss": 3.7780713931355367, + "tokens_seen": 757268480 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038915747241725175, + "loss": 1.4923, + "theoretical_loss": 3.7780389069467057, + "tokens_seen": 757334016 + }, + { + "epoch": 0.23, + "learning_rate": 0.000389147442326981, + "loss": 1.3986, + "theoretical_loss": 3.778006424356007, + "tokens_seen": 757399552 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003891374122367101, + "loss": 1.4489, + "theoretical_loss": 3.7779739453627315, + "tokens_seen": 757465088 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038912738214643935, + "loss": 1.3808, + "theoretical_loss": 3.7779414699661684, + "tokens_seen": 757530624 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003891173520561685, + "loss": 1.401, + "theoretical_loss": 3.7779089981656093, + "tokens_seen": 757596160 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003891073219658977, + "loss": 1.42, + "theoretical_loss": 3.7778765299603445, + "tokens_seen": 757661696 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003890972918756269, + "loss": 1.3043, + "theoretical_loss": 3.777844065349665, + "tokens_seen": 757727232 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038908726178535607, + "loss": 1.4067, + "theoretical_loss": 3.7778116043328627, + "tokens_seen": 757792768 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038907723169508525, + "loss": 1.4457, + "theoretical_loss": 3.777779146909228, + "tokens_seen": 757858304 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038906720160481443, + "loss": 1.4169, + "theoretical_loss": 3.777746693078053, + "tokens_seen": 757923840 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003890571715145436, + "loss": 1.4599, + "theoretical_loss": 3.7777142428386297, + "tokens_seen": 757989376 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038904714142427285, + "loss": 1.436, + "theoretical_loss": 3.777681796190249, + "tokens_seen": 758054912 + }, + { + "epoch": 0.23, + "learning_rate": 0.000389037111334002, + "loss": 1.4536, + "theoretical_loss": 3.777649353132205, + "tokens_seen": 758120448 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003890270812437312, + "loss": 1.403, + "theoretical_loss": 3.7776169136637883, + "tokens_seen": 758185984 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003890170511534604, + "loss": 1.3651, + "theoretical_loss": 3.7775844777842926, + "tokens_seen": 758251520 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003890070210631896, + "loss": 1.3874, + "theoretical_loss": 3.7775520454930103, + "tokens_seen": 758317056 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038899699097291876, + "loss": 1.4527, + "theoretical_loss": 3.777519616789234, + "tokens_seen": 758382592 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038898696088264794, + "loss": 1.4003, + "theoretical_loss": 3.7774871916722574, + "tokens_seen": 758448128 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003889769307923771, + "loss": 1.4781, + "theoretical_loss": 3.7774547701413743, + "tokens_seen": 758513664 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038896690070210635, + "loss": 1.4021, + "theoretical_loss": 3.777422352195878, + "tokens_seen": 758579200 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003889568706118355, + "loss": 1.3571, + "theoretical_loss": 3.7773899378350615, + "tokens_seen": 758644736 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003889468405215647, + "loss": 1.3002, + "theoretical_loss": 3.7773575270582196, + "tokens_seen": 758710272 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038893681043129384, + "loss": 1.3279, + "theoretical_loss": 3.7773251198646465, + "tokens_seen": 758775808 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003889267803410231, + "loss": 1.3117, + "theoretical_loss": 3.777292716253636, + "tokens_seen": 758841344 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038891675025075226, + "loss": 1.4349, + "theoretical_loss": 3.777260316224484, + "tokens_seen": 758906880 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038890672016048144, + "loss": 1.3245, + "theoretical_loss": 3.7772279197764838, + "tokens_seen": 758972416 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003888966900702106, + "loss": 1.401, + "theoretical_loss": 3.7771955269089315, + "tokens_seen": 759037952 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003888866599799398, + "loss": 1.4511, + "theoretical_loss": 3.7771631376211214, + "tokens_seen": 759103488 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038887662988966904, + "loss": 1.4426, + "theoretical_loss": 3.7771307519123494, + "tokens_seen": 759169024 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003888665997993982, + "loss": 1.5053, + "theoretical_loss": 3.7770983697819114, + "tokens_seen": 759234560 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003888565697091274, + "loss": 1.3874, + "theoretical_loss": 3.7770659912291027, + "tokens_seen": 759300096 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003888465396188566, + "loss": 1.3582, + "theoretical_loss": 3.7770336162532194, + "tokens_seen": 759365632 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003888365095285858, + "loss": 1.3512, + "theoretical_loss": 3.7770012448535577, + "tokens_seen": 759431168 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038882647943831494, + "loss": 1.4355, + "theoretical_loss": 3.7769688770294145, + "tokens_seen": 759496704 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003888164493480442, + "loss": 1.4688, + "theoretical_loss": 3.776936512780085, + "tokens_seen": 759562240 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003888064192577733, + "loss": 1.4035, + "theoretical_loss": 3.776904152104868, + "tokens_seen": 759627776 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038879638916750254, + "loss": 1.3859, + "theoretical_loss": 3.776871795003059, + "tokens_seen": 759693312 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003887863590772317, + "loss": 1.4496, + "theoretical_loss": 3.7768394414739554, + "tokens_seen": 759758848 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003887763289869609, + "loss": 1.3735, + "theoretical_loss": 3.7768070915168552, + "tokens_seen": 759824384 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003887662988966901, + "loss": 1.4332, + "theoretical_loss": 3.7767747451310556, + "tokens_seen": 759889920 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038875626880641927, + "loss": 1.4991, + "theoretical_loss": 3.776742402315854, + "tokens_seen": 759955456 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038874623871614845, + "loss": 1.3729, + "theoretical_loss": 3.7767100630705492, + "tokens_seen": 760020992 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003887362086258777, + "loss": 1.3813, + "theoretical_loss": 3.776677727394439, + "tokens_seen": 760086528 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003887261785356068, + "loss": 1.4109, + "theoretical_loss": 3.7766453952868217, + "tokens_seen": 760152064 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 257868, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6046000719070435, + "objective/train/theoretical_loss": 3.776613066746996, + "objective/train/tokens_used": 760217600, + "theoretical_loss": 3.776613066746996, + "tokens_seen": 760217600 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038871614844533604, + "loss": 1.5388, + "theoretical_loss": 3.776613066746996, + "tokens_seen": 760217600 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038870611835506517, + "loss": 1.3486, + "theoretical_loss": 3.7765807417742607, + "tokens_seen": 760283136 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003886960882647944, + "loss": 1.3253, + "theoretical_loss": 3.7765484203679147, + "tokens_seen": 760348672 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003886860581745236, + "loss": 1.3521, + "theoretical_loss": 3.776516102527258, + "tokens_seen": 760414208 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038867602808425277, + "loss": 1.4018, + "theoretical_loss": 3.7764837882515874, + "tokens_seen": 760479744 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038866599799398195, + "loss": 1.4683, + "theoretical_loss": 3.7764514775402054, + "tokens_seen": 760545280 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003886559679037112, + "loss": 1.44, + "theoretical_loss": 3.776419170392411, + "tokens_seen": 760610816 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003886459378134403, + "loss": 1.3788, + "theoretical_loss": 3.7763868668075027, + "tokens_seen": 760676352 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038863590772316955, + "loss": 1.3389, + "theoretical_loss": 3.776354566784783, + "tokens_seen": 760741888 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003886258776328987, + "loss": 1.3563, + "theoretical_loss": 3.77632227032355, + "tokens_seen": 760807424 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003886158475426279, + "loss": 1.4426, + "theoretical_loss": 3.7762899774231062, + "tokens_seen": 760872960 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003886058174523571, + "loss": 1.3213, + "theoretical_loss": 3.776257688082751, + "tokens_seen": 760938496 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038859578736208627, + "loss": 1.3372, + "theoretical_loss": 3.776225402301786, + "tokens_seen": 761004032 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038858575727181545, + "loss": 1.3198, + "theoretical_loss": 3.776193120079512, + "tokens_seen": 761069568 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038857572718154463, + "loss": 1.3146, + "theoretical_loss": 3.7761608414152303, + "tokens_seen": 761135104 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003885656970912738, + "loss": 1.3762, + "theoretical_loss": 3.776128566308243, + "tokens_seen": 761200640 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038855566700100305, + "loss": 1.4288, + "theoretical_loss": 3.776096294757852, + "tokens_seen": 761266176 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003885456369107322, + "loss": 1.363, + "theoretical_loss": 3.7760640267633576, + "tokens_seen": 761331712 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003885356068204614, + "loss": 1.5579, + "theoretical_loss": 3.776031762324064, + "tokens_seen": 761397248 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003885255767301906, + "loss": 1.3978, + "theoretical_loss": 3.775999501439273, + "tokens_seen": 761462784 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003885155466399198, + "loss": 1.4692, + "theoretical_loss": 3.775967244108286, + "tokens_seen": 761528320 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038850551654964896, + "loss": 1.5592, + "theoretical_loss": 3.775934990330407, + "tokens_seen": 761593856 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038849548645937814, + "loss": 1.3606, + "theoretical_loss": 3.7759027401049385, + "tokens_seen": 761659392 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003884854563691073, + "loss": 1.3537, + "theoretical_loss": 3.7758704934311833, + "tokens_seen": 761724928 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038847542627883655, + "loss": 1.4154, + "theoretical_loss": 3.7758382503084453, + "tokens_seen": 761790464 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003884653961885657, + "loss": 1.4222, + "theoretical_loss": 3.7758060107360274, + "tokens_seen": 761856000 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003884553660982949, + "loss": 1.5372, + "theoretical_loss": 3.775773774713234, + "tokens_seen": 761921536 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038844533600802404, + "loss": 1.4841, + "theoretical_loss": 3.775741542239369, + "tokens_seen": 761987072 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003884353059177533, + "loss": 1.392, + "theoretical_loss": 3.775709313313736, + "tokens_seen": 762052608 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038842527582748246, + "loss": 1.4632, + "theoretical_loss": 3.775677087935639, + "tokens_seen": 762118144 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038841524573721164, + "loss": 1.4095, + "theoretical_loss": 3.7756448661043835, + "tokens_seen": 762183680 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003884052156469408, + "loss": 1.514, + "theoretical_loss": 3.7756126478192735, + "tokens_seen": 762249216 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038839518555667, + "loss": 1.3984, + "theoretical_loss": 3.775580433079614, + "tokens_seen": 762314752 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003883851554663992, + "loss": 1.3744, + "theoretical_loss": 3.7755482218847103, + "tokens_seen": 762380288 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003883751253761284, + "loss": 1.4461, + "theoretical_loss": 3.775516014233868, + "tokens_seen": 762445824 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038836509528585755, + "loss": 1.4143, + "theoretical_loss": 3.7754838101263912, + "tokens_seen": 762511360 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003883550651955868, + "loss": 1.3578, + "theoretical_loss": 3.7754516095615873, + "tokens_seen": 762576896 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038834503510531596, + "loss": 1.3598, + "theoretical_loss": 3.775419412538761, + "tokens_seen": 762642432 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038833500501504514, + "loss": 1.46, + "theoretical_loss": 3.7753872190572193, + "tokens_seen": 762707968 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003883249749247743, + "loss": 1.4271, + "theoretical_loss": 3.775355029116268, + "tokens_seen": 762773504 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003883149448345035, + "loss": 1.419, + "theoretical_loss": 3.7753228427152123, + "tokens_seen": 762839040 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003883049147442327, + "loss": 1.5376, + "theoretical_loss": 3.7752906598533613, + "tokens_seen": 762904576 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003882948846539619, + "loss": 1.3986, + "theoretical_loss": 3.77525848053002, + "tokens_seen": 762970112 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038828485456369105, + "loss": 1.3132, + "theoretical_loss": 3.7752263047444963, + "tokens_seen": 763035648 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003882748244734203, + "loss": 1.379, + "theoretical_loss": 3.7751941324960976, + "tokens_seen": 763101184 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003882647943831494, + "loss": 1.3955, + "theoretical_loss": 3.7751619637841305, + "tokens_seen": 763166720 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038825476429287865, + "loss": 1.4531, + "theoretical_loss": 3.775129798607903, + "tokens_seen": 763232256 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038824473420260783, + "loss": 1.3586, + "theoretical_loss": 3.7750976369667226, + "tokens_seen": 763297792 + }, + { + "epoch": 0.23, + "learning_rate": 0.000388234704112337, + "loss": 1.4352, + "theoretical_loss": 3.7750654788598985, + "tokens_seen": 763363328 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003882246740220662, + "loss": 1.3421, + "theoretical_loss": 3.7750333242867375, + "tokens_seen": 763428864 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 258835, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.499750018119812, + "objective/train/theoretical_loss": 3.775001173246549, + "objective/train/tokens_used": 763494400, + "theoretical_loss": 3.775001173246549, + "tokens_seen": 763494400 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038821464393179537, + "loss": 1.4586, + "theoretical_loss": 3.775001173246549, + "tokens_seen": 763494400 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038820461384152455, + "loss": 1.3647, + "theoretical_loss": 3.774969025738642, + "tokens_seen": 763559936 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003881945837512538, + "loss": 1.3542, + "theoretical_loss": 3.7749368817623234, + "tokens_seen": 763625472 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003881845536609829, + "loss": 1.4595, + "theoretical_loss": 3.774904741316904, + "tokens_seen": 763691008 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038817452357071215, + "loss": 1.5278, + "theoretical_loss": 3.7748726044016916, + "tokens_seen": 763756544 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038816449348044133, + "loss": 1.4132, + "theoretical_loss": 3.774840471015997, + "tokens_seen": 763822080 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003881544633901705, + "loss": 1.4028, + "theoretical_loss": 3.774808341159129, + "tokens_seen": 763887616 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003881444332998997, + "loss": 1.3736, + "theoretical_loss": 3.7747762148303976, + "tokens_seen": 763953152 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003881344032096289, + "loss": 1.3088, + "theoretical_loss": 3.7747440920291124, + "tokens_seen": 764018688 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003881243731193581, + "loss": 1.4767, + "theoretical_loss": 3.774711972754584, + "tokens_seen": 764084224 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003881143430290873, + "loss": 1.3304, + "theoretical_loss": 3.7746798570061224, + "tokens_seen": 764149760 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038810431293881647, + "loss": 1.3982, + "theoretical_loss": 3.774647744783038, + "tokens_seen": 764215296 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038809428284854565, + "loss": 1.4787, + "theoretical_loss": 3.7746156360846426, + "tokens_seen": 764280832 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038808425275827483, + "loss": 1.495, + "theoretical_loss": 3.7745835309102462, + "tokens_seen": 764346368 + }, + { + "epoch": 0.23, + "learning_rate": 0.000388074222668004, + "loss": 1.4324, + "theoretical_loss": 3.7745514292591595, + "tokens_seen": 764411904 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038806419257773325, + "loss": 1.4325, + "theoretical_loss": 3.774519331130695, + "tokens_seen": 764477440 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003880541624874624, + "loss": 1.3322, + "theoretical_loss": 3.7744872365241635, + "tokens_seen": 764542976 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003880441323971916, + "loss": 1.3084, + "theoretical_loss": 3.774455145438877, + "tokens_seen": 764608512 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003880341023069208, + "loss": 1.4611, + "theoretical_loss": 3.7744230578741473, + "tokens_seen": 764674048 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038802407221665, + "loss": 1.5428, + "theoretical_loss": 3.7743909738292865, + "tokens_seen": 764739584 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038801404212637916, + "loss": 1.5392, + "theoretical_loss": 3.7743588933036065, + "tokens_seen": 764805120 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038800401203610834, + "loss": 1.3727, + "theoretical_loss": 3.774326816296421, + "tokens_seen": 764870656 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003879939819458375, + "loss": 1.4223, + "theoretical_loss": 3.774294742807041, + "tokens_seen": 764936192 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038798395185556675, + "loss": 1.3795, + "theoretical_loss": 3.774262672834781, + "tokens_seen": 765001728 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003879739217652959, + "loss": 1.4134, + "theoretical_loss": 3.7742306063789526, + "tokens_seen": 765067264 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003879638916750251, + "loss": 1.4229, + "theoretical_loss": 3.77419854343887, + "tokens_seen": 765132800 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038795386158475424, + "loss": 1.3777, + "theoretical_loss": 3.774166484013846, + "tokens_seen": 765198336 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003879438314944835, + "loss": 1.4319, + "theoretical_loss": 3.774134428103195, + "tokens_seen": 765263872 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038793380140421266, + "loss": 1.491, + "theoretical_loss": 3.7741023757062306, + "tokens_seen": 765329408 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038792377131394184, + "loss": 1.4196, + "theoretical_loss": 3.7740703268222666, + "tokens_seen": 765394944 + }, + { + "epoch": 0.23, + "learning_rate": 0.000387913741223671, + "loss": 1.4381, + "theoretical_loss": 3.7740382814506175, + "tokens_seen": 765460480 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003879037111334002, + "loss": 1.3951, + "theoretical_loss": 3.774006239590597, + "tokens_seen": 765526016 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003878936810431294, + "loss": 1.4037, + "theoretical_loss": 3.773974201241521, + "tokens_seen": 765591552 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003878836509528586, + "loss": 1.4469, + "theoretical_loss": 3.773942166402703, + "tokens_seen": 765657088 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038787362086258775, + "loss": 1.3974, + "theoretical_loss": 3.7739101350734585, + "tokens_seen": 765722624 + }, + { + "epoch": 0.23, + "learning_rate": 0.000387863590772317, + "loss": 1.3331, + "theoretical_loss": 3.7738781072531027, + "tokens_seen": 765788160 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038785356068204616, + "loss": 1.4892, + "theoretical_loss": 3.7738460829409513, + "tokens_seen": 765853696 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038784353059177534, + "loss": 1.3784, + "theoretical_loss": 3.773814062136319, + "tokens_seen": 765919232 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003878335005015045, + "loss": 1.458, + "theoretical_loss": 3.773782044838523, + "tokens_seen": 765984768 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003878234704112337, + "loss": 1.44, + "theoretical_loss": 3.7737500310468777, + "tokens_seen": 766050304 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003878134403209629, + "loss": 1.3507, + "theoretical_loss": 3.7737180207607004, + "tokens_seen": 766115840 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003878034102306921, + "loss": 1.4419, + "theoretical_loss": 3.7736860139793063, + "tokens_seen": 766181376 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038779338014042125, + "loss": 1.4268, + "theoretical_loss": 3.773654010702013, + "tokens_seen": 766246912 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003877833500501505, + "loss": 1.5135, + "theoretical_loss": 3.773622010928137, + "tokens_seen": 766312448 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003877733199598796, + "loss": 1.3585, + "theoretical_loss": 3.7735900146569947, + "tokens_seen": 766377984 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038776328986960885, + "loss": 1.4427, + "theoretical_loss": 3.7735580218879035, + "tokens_seen": 766443520 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038775325977933803, + "loss": 1.3425, + "theoretical_loss": 3.7735260326201807, + "tokens_seen": 766509056 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003877432296890672, + "loss": 1.4455, + "theoretical_loss": 3.773494046853144, + "tokens_seen": 766574592 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003877331995987964, + "loss": 1.3868, + "theoretical_loss": 3.773462064586111, + "tokens_seen": 766640128 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038772316950852557, + "loss": 1.3716, + "theoretical_loss": 3.773430085818399, + "tokens_seen": 766705664 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 259964, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3046302795410156, + "objective/train/theoretical_loss": 3.7733981105493273, + "objective/train/tokens_used": 766771200, + "theoretical_loss": 3.7733981105493273, + "tokens_seen": 766771200 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038771313941825475, + "loss": 1.3449, + "theoretical_loss": 3.7733981105493273, + "tokens_seen": 766771200 + }, + { + "epoch": 0.23, + "learning_rate": 0.000387703109327984, + "loss": 1.3791, + "theoretical_loss": 3.7733661387782123, + "tokens_seen": 766836736 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003876930792377131, + "loss": 1.4213, + "theoretical_loss": 3.7733341705043744, + "tokens_seen": 766902272 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038768304914744235, + "loss": 1.4681, + "theoretical_loss": 3.7733022057271306, + "tokens_seen": 766967808 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038767301905717153, + "loss": 1.4106, + "theoretical_loss": 3.7732702444458006, + "tokens_seen": 767033344 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003876629889669007, + "loss": 1.4589, + "theoretical_loss": 3.773238286659704, + "tokens_seen": 767098880 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003876529588766299, + "loss": 1.3677, + "theoretical_loss": 3.773206332368159, + "tokens_seen": 767164416 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003876429287863591, + "loss": 1.3567, + "theoretical_loss": 3.7731743815704846, + "tokens_seen": 767229952 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038763289869608826, + "loss": 1.4227, + "theoretical_loss": 3.7731424342660014, + "tokens_seen": 767295488 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003876228686058175, + "loss": 1.2631, + "theoretical_loss": 3.7731104904540294, + "tokens_seen": 767361024 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003876128385155466, + "loss": 1.404, + "theoretical_loss": 3.7730785501338873, + "tokens_seen": 767426560 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038760280842527585, + "loss": 1.4321, + "theoretical_loss": 3.773046613304896, + "tokens_seen": 767492096 + }, + { + "epoch": 0.23, + "learning_rate": 0.000387592778335005, + "loss": 1.2971, + "theoretical_loss": 3.7730146799663764, + "tokens_seen": 767557632 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003875827482447342, + "loss": 1.4303, + "theoretical_loss": 3.772982750117648, + "tokens_seen": 767623168 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003875727181544634, + "loss": 1.4366, + "theoretical_loss": 3.772950823758032, + "tokens_seen": 767688704 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003875626880641926, + "loss": 1.3554, + "theoretical_loss": 3.7729189008868493, + "tokens_seen": 767754240 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038755265797392176, + "loss": 1.4654, + "theoretical_loss": 3.7728869815034214, + "tokens_seen": 767819776 + }, + { + "epoch": 0.23, + "learning_rate": 0.000387542627883651, + "loss": 1.3598, + "theoretical_loss": 3.7728550656070685, + "tokens_seen": 767885312 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003875325977933801, + "loss": 1.3236, + "theoretical_loss": 3.7728231531971135, + "tokens_seen": 767950848 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038752256770310936, + "loss": 1.4828, + "theoretical_loss": 3.772791244272877, + "tokens_seen": 768016384 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003875125376128385, + "loss": 1.4716, + "theoretical_loss": 3.7727593388336813, + "tokens_seen": 768081920 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003875025075225677, + "loss": 1.3473, + "theoretical_loss": 3.7727274368788484, + "tokens_seen": 768147456 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003874924774322969, + "loss": 1.4107, + "theoretical_loss": 3.7726955384077008, + "tokens_seen": 768212992 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003874824473420261, + "loss": 1.167, + "theoretical_loss": 3.7726636434195604, + "tokens_seen": 768278528 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038747241725175526, + "loss": 1.3918, + "theoretical_loss": 3.7726317519137504, + "tokens_seen": 768344064 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038746238716148444, + "loss": 1.2098, + "theoretical_loss": 3.7725998638895932, + "tokens_seen": 768409600 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003874523570712136, + "loss": 1.452, + "theoretical_loss": 3.772567979346412, + "tokens_seen": 768475136 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038744232698094286, + "loss": 1.3821, + "theoretical_loss": 3.77253609828353, + "tokens_seen": 768540672 + }, + { + "epoch": 0.23, + "learning_rate": 0.000387432296890672, + "loss": 1.3593, + "theoretical_loss": 3.77250422070027, + "tokens_seen": 768606208 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003874222668004012, + "loss": 1.3911, + "theoretical_loss": 3.772472346595957, + "tokens_seen": 768671744 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038741223671013035, + "loss": 1.3219, + "theoretical_loss": 3.7724404759699137, + "tokens_seen": 768737280 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003874022066198596, + "loss": 1.4779, + "theoretical_loss": 3.772408608821464, + "tokens_seen": 768802816 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038739217652958876, + "loss": 1.424, + "theoretical_loss": 3.7723767451499324, + "tokens_seen": 768868352 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038738214643931795, + "loss": 1.3941, + "theoretical_loss": 3.772344884954643, + "tokens_seen": 768933888 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003873721163490472, + "loss": 1.4726, + "theoretical_loss": 3.77231302823492, + "tokens_seen": 768999424 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038736208625877636, + "loss": 1.4972, + "theoretical_loss": 3.772281174990089, + "tokens_seen": 769064960 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038735205616850554, + "loss": 1.4168, + "theoretical_loss": 3.772249325219475, + "tokens_seen": 769130496 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003873420260782347, + "loss": 1.4028, + "theoretical_loss": 3.7722174789224017, + "tokens_seen": 769196032 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003873319959879639, + "loss": 1.5111, + "theoretical_loss": 3.7721856360981953, + "tokens_seen": 769261568 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003873219658976931, + "loss": 1.3802, + "theoretical_loss": 3.7721537967461813, + "tokens_seen": 769327104 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003873119358074223, + "loss": 1.5035, + "theoretical_loss": 3.7721219608656855, + "tokens_seen": 769392640 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038730190571715145, + "loss": 1.4153, + "theoretical_loss": 3.7720901284560333, + "tokens_seen": 769458176 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003872918756268807, + "loss": 1.4778, + "theoretical_loss": 3.7720582995165506, + "tokens_seen": 769523712 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003872818455366098, + "loss": 1.4044, + "theoretical_loss": 3.772026474046564, + "tokens_seen": 769589248 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038727181544633905, + "loss": 1.3779, + "theoretical_loss": 3.7719946520454, + "tokens_seen": 769654784 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038726178535606823, + "loss": 1.4611, + "theoretical_loss": 3.771962833512385, + "tokens_seen": 769720320 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003872517552657974, + "loss": 1.3099, + "theoretical_loss": 3.7719310184468458, + "tokens_seen": 769785856 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003872417251755266, + "loss": 1.3525, + "theoretical_loss": 3.771899206848109, + "tokens_seen": 769851392 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038723169508525577, + "loss": 1.5788, + "theoretical_loss": 3.7718673987155027, + "tokens_seen": 769916928 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038722166499498495, + "loss": 1.3538, + "theoretical_loss": 3.771835594048353, + "tokens_seen": 769982464 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 260924, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.321885347366333, + "objective/train/theoretical_loss": 3.7718037928459887, + "objective/train/tokens_used": 770048000, + "theoretical_loss": 3.7718037928459887, + "tokens_seen": 770048000 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003872116349047142, + "loss": 1.3412, + "theoretical_loss": 3.7718037928459887, + "tokens_seen": 770048000 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003872016048144433, + "loss": 1.4401, + "theoretical_loss": 3.771771995107737, + "tokens_seen": 770113536 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038719157472417255, + "loss": 1.427, + "theoretical_loss": 3.771740200832925, + "tokens_seen": 770179072 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038718154463390173, + "loss": 1.4333, + "theoretical_loss": 3.771708410020882, + "tokens_seen": 770244608 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003871715145436309, + "loss": 1.4366, + "theoretical_loss": 3.7716766226709355, + "tokens_seen": 770310144 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003871614844533601, + "loss": 1.4281, + "theoretical_loss": 3.7716448387824144, + "tokens_seen": 770375680 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003871514543630893, + "loss": 1.4446, + "theoretical_loss": 3.7716130583546477, + "tokens_seen": 770441216 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038714142427281846, + "loss": 1.4651, + "theoretical_loss": 3.771581281386963, + "tokens_seen": 770506752 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003871313941825477, + "loss": 1.2635, + "theoretical_loss": 3.7715495078786905, + "tokens_seen": 770572288 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003871213640922768, + "loss": 1.3768, + "theoretical_loss": 3.771517737829159, + "tokens_seen": 770637824 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038711133400200605, + "loss": 1.3432, + "theoretical_loss": 3.7714859712376985, + "tokens_seen": 770703360 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003871013039117352, + "loss": 1.5253, + "theoretical_loss": 3.771454208103637, + "tokens_seen": 770768896 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003870912738214644, + "loss": 1.4277, + "theoretical_loss": 3.771422448426306, + "tokens_seen": 770834432 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003870812437311936, + "loss": 1.3783, + "theoretical_loss": 3.7713906922050344, + "tokens_seen": 770899968 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003870712136409228, + "loss": 1.3173, + "theoretical_loss": 3.771358939439153, + "tokens_seen": 770965504 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038706118355065196, + "loss": 1.5061, + "theoretical_loss": 3.771327190127992, + "tokens_seen": 771031040 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003870511534603812, + "loss": 1.4914, + "theoretical_loss": 3.7712954442708813, + "tokens_seen": 771096576 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003870411233701103, + "loss": 1.4118, + "theoretical_loss": 3.7712637018671527, + "tokens_seen": 771162112 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038703109327983956, + "loss": 1.3556, + "theoretical_loss": 3.7712319629161364, + "tokens_seen": 771227648 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003870210631895687, + "loss": 1.2878, + "theoretical_loss": 3.771200227417163, + "tokens_seen": 771293184 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003870110330992979, + "loss": 1.4129, + "theoretical_loss": 3.771168495369565, + "tokens_seen": 771358720 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003870010030090271, + "loss": 1.363, + "theoretical_loss": 3.771136766772673, + "tokens_seen": 771424256 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003869909729187563, + "loss": 1.4753, + "theoretical_loss": 3.7711050416258187, + "tokens_seen": 771489792 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038698094282848546, + "loss": 1.3755, + "theoretical_loss": 3.771073319928335, + "tokens_seen": 771555328 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038697091273821464, + "loss": 1.3809, + "theoretical_loss": 3.7710416016795523, + "tokens_seen": 771620864 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003869608826479438, + "loss": 1.3982, + "theoretical_loss": 3.7710098868788036, + "tokens_seen": 771686400 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038695085255767306, + "loss": 1.239, + "theoretical_loss": 3.7709781755254212, + "tokens_seen": 771751936 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003869408224674022, + "loss": 1.345, + "theoretical_loss": 3.770946467618738, + "tokens_seen": 771817472 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003869307923771314, + "loss": 1.4253, + "theoretical_loss": 3.7709147631580864, + "tokens_seen": 771883008 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038692076228686055, + "loss": 1.3481, + "theoretical_loss": 3.7708830621427993, + "tokens_seen": 771948544 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003869107321965898, + "loss": 1.372, + "theoretical_loss": 3.77085136457221, + "tokens_seen": 772014080 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038690070210631896, + "loss": 1.332, + "theoretical_loss": 3.7708196704456522, + "tokens_seen": 772079616 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038689067201604815, + "loss": 1.5426, + "theoretical_loss": 3.770787979762458, + "tokens_seen": 772145152 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038688064192577733, + "loss": 1.35, + "theoretical_loss": 3.770756292521963, + "tokens_seen": 772210688 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038687061183550656, + "loss": 1.4579, + "theoretical_loss": 3.7707246087235, + "tokens_seen": 772276224 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003868605817452357, + "loss": 1.3291, + "theoretical_loss": 3.770692928366403, + "tokens_seen": 772341760 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003868505516549649, + "loss": 1.3916, + "theoretical_loss": 3.7706612514500066, + "tokens_seen": 772407296 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038684052156469405, + "loss": 1.4316, + "theoretical_loss": 3.770629577973645, + "tokens_seen": 772472832 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003868304914744233, + "loss": 1.4123, + "theoretical_loss": 3.7705979079366534, + "tokens_seen": 772538368 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038682046138415247, + "loss": 1.3796, + "theoretical_loss": 3.7705662413383654, + "tokens_seen": 772603904 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038681043129388165, + "loss": 1.3882, + "theoretical_loss": 3.7705345781781174, + "tokens_seen": 772669440 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038680040120361083, + "loss": 1.394, + "theoretical_loss": 3.770502918455243, + "tokens_seen": 772734976 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038679037111334, + "loss": 1.4174, + "theoretical_loss": 3.770471262169079, + "tokens_seen": 772800512 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003867803410230692, + "loss": 1.426, + "theoretical_loss": 3.7704396093189607, + "tokens_seen": 772866048 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038677031093279843, + "loss": 1.4181, + "theoretical_loss": 3.7704079599042233, + "tokens_seen": 772931584 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038676028084252755, + "loss": 1.4334, + "theoretical_loss": 3.770376313924203, + "tokens_seen": 772997120 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003867502507522568, + "loss": 1.5139, + "theoretical_loss": 3.7703446713782354, + "tokens_seen": 773062656 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003867402206619859, + "loss": 1.3249, + "theoretical_loss": 3.7703130322656575, + "tokens_seen": 773128192 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038673019057171515, + "loss": 1.4906, + "theoretical_loss": 3.7702813965858053, + "tokens_seen": 773193728 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038672016048144433, + "loss": 1.3622, + "theoretical_loss": 3.7702497643380157, + "tokens_seen": 773259264 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 262076, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.401519775390625, + "objective/train/theoretical_loss": 3.7702181355216258, + "objective/train/tokens_used": 773324800, + "theoretical_loss": 3.7702181355216258, + "tokens_seen": 773324800 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003867101303911735, + "loss": 1.3916, + "theoretical_loss": 3.7702181355216258, + "tokens_seen": 773324800 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003867001003009027, + "loss": 1.3433, + "theoretical_loss": 3.770186510135972, + "tokens_seen": 773390336 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038669007021063193, + "loss": 1.3731, + "theoretical_loss": 3.7701548881803917, + "tokens_seen": 773455872 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038668004012036106, + "loss": 1.2787, + "theoretical_loss": 3.7701232696542224, + "tokens_seen": 773521408 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003866700100300903, + "loss": 1.4634, + "theoretical_loss": 3.7700916545568015, + "tokens_seen": 773586944 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003866599799398194, + "loss": 1.292, + "theoretical_loss": 3.7700600428874678, + "tokens_seen": 773652480 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038664994984954866, + "loss": 1.5271, + "theoretical_loss": 3.7700284346455577, + "tokens_seen": 773718016 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038663991975927784, + "loss": 1.4059, + "theoretical_loss": 3.7699968298304096, + "tokens_seen": 773783552 + }, + { + "epoch": 0.23, + "learning_rate": 0.000386629889669007, + "loss": 1.3254, + "theoretical_loss": 3.7699652284413627, + "tokens_seen": 773849088 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038661985957873625, + "loss": 1.3977, + "theoretical_loss": 3.769933630477755, + "tokens_seen": 773914624 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003866098294884654, + "loss": 1.4289, + "theoretical_loss": 3.769902035938925, + "tokens_seen": 773980160 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003865997993981946, + "loss": 1.4771, + "theoretical_loss": 3.7698704448242113, + "tokens_seen": 774045696 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003865897693079238, + "loss": 1.4116, + "theoretical_loss": 3.769838857132954, + "tokens_seen": 774111232 + }, + { + "epoch": 0.23, + "learning_rate": 0.000386579739217653, + "loss": 1.3862, + "theoretical_loss": 3.769807272864491, + "tokens_seen": 774176768 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038656970912738216, + "loss": 1.3666, + "theoretical_loss": 3.7697756920181633, + "tokens_seen": 774242304 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003865596790371114, + "loss": 1.4161, + "theoretical_loss": 3.769744114593309, + "tokens_seen": 774307840 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003865496489468405, + "loss": 1.3562, + "theoretical_loss": 3.7697125405892686, + "tokens_seen": 774373376 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038653961885656976, + "loss": 1.4718, + "theoretical_loss": 3.769680970005382, + "tokens_seen": 774438912 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003865295887662989, + "loss": 1.4609, + "theoretical_loss": 3.769649402840989, + "tokens_seen": 774504448 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003865195586760281, + "loss": 1.3485, + "theoretical_loss": 3.7696178390954302, + "tokens_seen": 774569984 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003865095285857573, + "loss": 1.4556, + "theoretical_loss": 3.7695862787680463, + "tokens_seen": 774635520 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003864994984954865, + "loss": 1.4279, + "theoretical_loss": 3.7695547218581775, + "tokens_seen": 774701056 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038648946840521566, + "loss": 1.3287, + "theoretical_loss": 3.769523168365165, + "tokens_seen": 774766592 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038647943831494484, + "loss": 1.3758, + "theoretical_loss": 3.7694916182883498, + "tokens_seen": 774832128 + }, + { + "epoch": 0.23, + "learning_rate": 0.000386469408224674, + "loss": 1.5801, + "theoretical_loss": 3.7694600716270736, + "tokens_seen": 774897664 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038645937813440326, + "loss": 1.3643, + "theoretical_loss": 3.7694285283806774, + "tokens_seen": 774963200 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003864493480441324, + "loss": 1.3802, + "theoretical_loss": 3.769396988548502, + "tokens_seen": 775028736 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003864393179538616, + "loss": 1.3881, + "theoretical_loss": 3.7693654521298905, + "tokens_seen": 775094272 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038642928786359075, + "loss": 1.444, + "theoretical_loss": 3.7693339191241844, + "tokens_seen": 775159808 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038641925777332, + "loss": 1.3893, + "theoretical_loss": 3.769302389530726, + "tokens_seen": 775225344 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038640922768304917, + "loss": 1.4665, + "theoretical_loss": 3.769270863348857, + "tokens_seen": 775290880 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038639919759277835, + "loss": 1.4767, + "theoretical_loss": 3.7692393405779208, + "tokens_seen": 775356416 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038638916750250753, + "loss": 1.4425, + "theoretical_loss": 3.7692078212172593, + "tokens_seen": 775421952 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038637913741223676, + "loss": 1.3194, + "theoretical_loss": 3.7691763052662157, + "tokens_seen": 775487488 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003863691073219659, + "loss": 1.3743, + "theoretical_loss": 3.7691447927241333, + "tokens_seen": 775553024 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003863590772316951, + "loss": 1.3864, + "theoretical_loss": 3.769113283590355, + "tokens_seen": 775618560 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038634904714142425, + "loss": 1.4873, + "theoretical_loss": 3.7690817778642245, + "tokens_seen": 775684096 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003863390170511535, + "loss": 1.2577, + "theoretical_loss": 3.7690502755450854, + "tokens_seen": 775749632 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038632898696088267, + "loss": 1.4541, + "theoretical_loss": 3.769018776632281, + "tokens_seen": 775815168 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038631895687061185, + "loss": 1.4893, + "theoretical_loss": 3.768987281125156, + "tokens_seen": 775880704 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038630892678034103, + "loss": 1.3149, + "theoretical_loss": 3.7689557890230545, + "tokens_seen": 775946240 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003862988966900702, + "loss": 1.4439, + "theoretical_loss": 3.7689243003253203, + "tokens_seen": 776011776 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003862888665997994, + "loss": 1.3934, + "theoretical_loss": 3.768892815031298, + "tokens_seen": 776077312 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038627883650952863, + "loss": 1.5469, + "theoretical_loss": 3.768861333140333, + "tokens_seen": 776142848 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038626880641925775, + "loss": 1.5057, + "theoretical_loss": 3.768829854651769, + "tokens_seen": 776208384 + }, + { + "epoch": 0.24, + "learning_rate": 0.000386258776328987, + "loss": 1.4312, + "theoretical_loss": 3.7687983795649522, + "tokens_seen": 776273920 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003862487462387161, + "loss": 1.2843, + "theoretical_loss": 3.768766907879227, + "tokens_seen": 776339456 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038623871614844535, + "loss": 1.2933, + "theoretical_loss": 3.7687354395939394, + "tokens_seen": 776404992 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038622868605817453, + "loss": 1.2719, + "theoretical_loss": 3.7687039747084348, + "tokens_seen": 776470528 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003862186559679037, + "loss": 1.3516, + "theoretical_loss": 3.768672513222059, + "tokens_seen": 776536064 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 262991, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4116148948669434, + "objective/train/theoretical_loss": 3.7686410551341583, + "objective/train/tokens_used": 776601600, + "theoretical_loss": 3.7686410551341583, + "tokens_seen": 776601600 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003862086258776329, + "loss": 1.4345, + "theoretical_loss": 3.7686410551341583, + "tokens_seen": 776601600 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038619859578736213, + "loss": 1.4193, + "theoretical_loss": 3.768609600444078, + "tokens_seen": 776667136 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038618856569709126, + "loss": 1.3683, + "theoretical_loss": 3.768578149151165, + "tokens_seen": 776732672 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003861785356068205, + "loss": 1.306, + "theoretical_loss": 3.768546701254766, + "tokens_seen": 776798208 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003861685055165496, + "loss": 1.3828, + "theoretical_loss": 3.7685152567542275, + "tokens_seen": 776863744 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038615847542627886, + "loss": 1.3464, + "theoretical_loss": 3.7684838156488962, + "tokens_seen": 776929280 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038614844533600804, + "loss": 1.4066, + "theoretical_loss": 3.7684523779381194, + "tokens_seen": 776994816 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003861384152457372, + "loss": 1.4712, + "theoretical_loss": 3.768420943621244, + "tokens_seen": 777060352 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003861283851554664, + "loss": 1.4802, + "theoretical_loss": 3.7683895126976177, + "tokens_seen": 777125888 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003861183550651956, + "loss": 1.4055, + "theoretical_loss": 3.7683580851665885, + "tokens_seen": 777191424 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038610832497492476, + "loss": 1.3799, + "theoretical_loss": 3.768326661027503, + "tokens_seen": 777256960 + }, + { + "epoch": 0.24, + "learning_rate": 0.000386098294884654, + "loss": 1.3643, + "theoretical_loss": 3.7682952402797105, + "tokens_seen": 777322496 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003860882647943831, + "loss": 1.3474, + "theoretical_loss": 3.7682638229225582, + "tokens_seen": 777388032 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038607823470411236, + "loss": 1.3553, + "theoretical_loss": 3.7682324089553947, + "tokens_seen": 777453568 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003860682046138415, + "loss": 1.3782, + "theoretical_loss": 3.768200998377569, + "tokens_seen": 777519104 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003860581745235707, + "loss": 1.2534, + "theoretical_loss": 3.7681695911884288, + "tokens_seen": 777584640 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003860481444332999, + "loss": 1.3873, + "theoretical_loss": 3.768138187387324, + "tokens_seen": 777650176 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003860381143430291, + "loss": 1.3699, + "theoretical_loss": 3.7681067869736022, + "tokens_seen": 777715712 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038602808425275826, + "loss": 1.4105, + "theoretical_loss": 3.768075389946614, + "tokens_seen": 777781248 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003860180541624875, + "loss": 1.3227, + "theoretical_loss": 3.7680439963057086, + "tokens_seen": 777846784 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003860080240722166, + "loss": 1.473, + "theoretical_loss": 3.7680126060502355, + "tokens_seen": 777912320 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038599799398194586, + "loss": 1.4835, + "theoretical_loss": 3.7679812191795437, + "tokens_seen": 777977856 + }, + { + "epoch": 0.24, + "learning_rate": 0.000385987963891675, + "loss": 1.4724, + "theoretical_loss": 3.767949835692984, + "tokens_seen": 778043392 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003859779338014042, + "loss": 1.453, + "theoretical_loss": 3.767918455589906, + "tokens_seen": 778108928 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003859679037111334, + "loss": 1.3824, + "theoretical_loss": 3.76788707886966, + "tokens_seen": 778174464 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003859578736208626, + "loss": 1.5317, + "theoretical_loss": 3.7678557055315975, + "tokens_seen": 778240000 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038594784353059177, + "loss": 1.3988, + "theoretical_loss": 3.767824335575068, + "tokens_seen": 778305536 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038593781344032095, + "loss": 1.4173, + "theoretical_loss": 3.7677929689994225, + "tokens_seen": 778371072 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038592778335005013, + "loss": 1.3633, + "theoretical_loss": 3.7677616058040124, + "tokens_seen": 778436608 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038591775325977937, + "loss": 1.3619, + "theoretical_loss": 3.7677302459881883, + "tokens_seen": 778502144 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003859077231695085, + "loss": 1.4317, + "theoretical_loss": 3.767698889551302, + "tokens_seen": 778567680 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038589769307923773, + "loss": 1.3785, + "theoretical_loss": 3.7676675364927057, + "tokens_seen": 778633216 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038588766298896685, + "loss": 1.3744, + "theoretical_loss": 3.76763618681175, + "tokens_seen": 778698752 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003858776328986961, + "loss": 1.4096, + "theoretical_loss": 3.7676048405077873, + "tokens_seen": 778764288 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003858676028084253, + "loss": 1.3638, + "theoretical_loss": 3.7675734975801696, + "tokens_seen": 778829824 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038585757271815445, + "loss": 1.3666, + "theoretical_loss": 3.7675421580282498, + "tokens_seen": 778895360 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003858475426278837, + "loss": 1.4056, + "theoretical_loss": 3.7675108218513786, + "tokens_seen": 778960896 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038583751253761287, + "loss": 1.4266, + "theoretical_loss": 3.7674794890489105, + "tokens_seen": 779026432 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038582748244734205, + "loss": 1.2859, + "theoretical_loss": 3.767448159620198, + "tokens_seen": 779091968 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038581745235707123, + "loss": 1.417, + "theoretical_loss": 3.7674168335645932, + "tokens_seen": 779157504 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003858074222668004, + "loss": 1.3488, + "theoretical_loss": 3.7673855108814496, + "tokens_seen": 779223040 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003857973921765296, + "loss": 1.4635, + "theoretical_loss": 3.7673541915701207, + "tokens_seen": 779288576 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038578736208625883, + "loss": 1.4122, + "theoretical_loss": 3.76732287562996, + "tokens_seen": 779354112 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038577733199598796, + "loss": 1.4285, + "theoretical_loss": 3.767291563060321, + "tokens_seen": 779419648 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003857673019057172, + "loss": 1.3395, + "theoretical_loss": 3.7672602538605586, + "tokens_seen": 779485184 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003857572718154463, + "loss": 1.3708, + "theoretical_loss": 3.7672289480300254, + "tokens_seen": 779550720 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038574724172517555, + "loss": 1.4165, + "theoretical_loss": 3.7671976455680767, + "tokens_seen": 779616256 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038573721163490473, + "loss": 1.3661, + "theoretical_loss": 3.767166346474066, + "tokens_seen": 779681792 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003857271815446339, + "loss": 1.368, + "theoretical_loss": 3.7671350507473482, + "tokens_seen": 779747328 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003857171514543631, + "loss": 1.2567, + "theoretical_loss": 3.7671037583872784, + "tokens_seen": 779812864 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 264230, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2322571277618408, + "objective/train/theoretical_loss": 3.767072469393211, + "objective/train/tokens_used": 779878400, + "theoretical_loss": 3.767072469393211, + "tokens_seen": 779878400 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038570712136409233, + "loss": 1.3749, + "theoretical_loss": 3.767072469393211, + "tokens_seen": 779878400 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038569709127382146, + "loss": 1.4449, + "theoretical_loss": 3.767041183764502, + "tokens_seen": 779943936 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003856870611835507, + "loss": 1.4096, + "theoretical_loss": 3.767009901500506, + "tokens_seen": 780009472 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003856770310932798, + "loss": 1.2725, + "theoretical_loss": 3.766978622600579, + "tokens_seen": 780075008 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038566700100300906, + "loss": 1.382, + "theoretical_loss": 3.7669473470640757, + "tokens_seen": 780140544 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038565697091273824, + "loss": 1.4606, + "theoretical_loss": 3.7669160748903527, + "tokens_seen": 780206080 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003856469408224674, + "loss": 1.3889, + "theoretical_loss": 3.766884806078766, + "tokens_seen": 780271616 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003856369107321966, + "loss": 1.3494, + "theoretical_loss": 3.766853540628672, + "tokens_seen": 780337152 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003856268806419258, + "loss": 1.3104, + "theoretical_loss": 3.7668222785394256, + "tokens_seen": 780402688 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038561685055165496, + "loss": 1.4384, + "theoretical_loss": 3.766791019810385, + "tokens_seen": 780468224 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003856068204613842, + "loss": 1.3378, + "theoretical_loss": 3.7667597644409065, + "tokens_seen": 780533760 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003855967903711133, + "loss": 1.3206, + "theoretical_loss": 3.7667285124303467, + "tokens_seen": 780599296 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038558676028084256, + "loss": 1.3894, + "theoretical_loss": 3.7666972637780627, + "tokens_seen": 780664832 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003855767301905717, + "loss": 1.4711, + "theoretical_loss": 3.766666018483412, + "tokens_seen": 780730368 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003855667001003009, + "loss": 1.4296, + "theoretical_loss": 3.766634776545752, + "tokens_seen": 780795904 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003855566700100301, + "loss": 1.4164, + "theoretical_loss": 3.76660353796444, + "tokens_seen": 780861440 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003855466399197593, + "loss": 1.413, + "theoretical_loss": 3.7665723027388336, + "tokens_seen": 780926976 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038553660982948846, + "loss": 1.4475, + "theoretical_loss": 3.7665410708682914, + "tokens_seen": 780992512 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003855265797392177, + "loss": 1.4208, + "theoretical_loss": 3.7665098423521712, + "tokens_seen": 781058048 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003855165496489468, + "loss": 1.3901, + "theoretical_loss": 3.766478617189832, + "tokens_seen": 781123584 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038550651955867606, + "loss": 1.3397, + "theoretical_loss": 3.766447395380631, + "tokens_seen": 781189120 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003854964894684052, + "loss": 1.3513, + "theoretical_loss": 3.7664161769239275, + "tokens_seen": 781254656 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003854864593781344, + "loss": 1.3504, + "theoretical_loss": 3.76638496181908, + "tokens_seen": 781320192 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003854764292878636, + "loss": 1.3836, + "theoretical_loss": 3.7663537500654485, + "tokens_seen": 781385728 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003854663991975928, + "loss": 1.3584, + "theoretical_loss": 3.766322541662391, + "tokens_seen": 781451264 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038545636910732197, + "loss": 1.3381, + "theoretical_loss": 3.766291336609268, + "tokens_seen": 781516800 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038544633901705115, + "loss": 1.4081, + "theoretical_loss": 3.766260134905438, + "tokens_seen": 781582336 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038543630892678033, + "loss": 1.3673, + "theoretical_loss": 3.7662289365502617, + "tokens_seen": 781647872 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038542627883650957, + "loss": 1.4918, + "theoretical_loss": 3.766197741543098, + "tokens_seen": 781713408 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003854162487462387, + "loss": 1.3647, + "theoretical_loss": 3.766166549883308, + "tokens_seen": 781778944 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038540621865596793, + "loss": 1.3611, + "theoretical_loss": 3.7661353615702504, + "tokens_seen": 781844480 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038539618856569705, + "loss": 1.3669, + "theoretical_loss": 3.7661041766032874, + "tokens_seen": 781910016 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003853861584754263, + "loss": 1.3697, + "theoretical_loss": 3.7660729949817786, + "tokens_seen": 781975552 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038537612838515547, + "loss": 1.3334, + "theoretical_loss": 3.7660418167050853, + "tokens_seen": 782041088 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038536609829488465, + "loss": 1.3629, + "theoretical_loss": 3.7660106417725676, + "tokens_seen": 782106624 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038535606820461383, + "loss": 1.4491, + "theoretical_loss": 3.765979470183588, + "tokens_seen": 782172160 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038534603811434307, + "loss": 1.3721, + "theoretical_loss": 3.765948301937506, + "tokens_seen": 782237696 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003853360080240722, + "loss": 1.44, + "theoretical_loss": 3.7659171370336844, + "tokens_seen": 782303232 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038532597793380143, + "loss": 1.3974, + "theoretical_loss": 3.765885975471485, + "tokens_seen": 782368768 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038531594784353056, + "loss": 1.3945, + "theoretical_loss": 3.7658548172502684, + "tokens_seen": 782434304 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003853059177532598, + "loss": 1.4148, + "theoretical_loss": 3.7658236623693977, + "tokens_seen": 782499840 + }, + { + "epoch": 0.24, + "learning_rate": 0.000385295887662989, + "loss": 1.4334, + "theoretical_loss": 3.765792510828234, + "tokens_seen": 782565376 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038528585757271816, + "loss": 1.3955, + "theoretical_loss": 3.7657613626261415, + "tokens_seen": 782630912 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038527582748244734, + "loss": 1.4259, + "theoretical_loss": 3.765730217762481, + "tokens_seen": 782696448 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003852657973921765, + "loss": 1.3331, + "theoretical_loss": 3.765699076236616, + "tokens_seen": 782761984 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003852557673019057, + "loss": 1.2755, + "theoretical_loss": 3.7656679380479092, + "tokens_seen": 782827520 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038524573721163493, + "loss": 1.4378, + "theoretical_loss": 3.7656368031957235, + "tokens_seen": 782893056 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038523570712136406, + "loss": 1.3741, + "theoretical_loss": 3.765605671679422, + "tokens_seen": 782958592 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003852256770310933, + "loss": 1.4417, + "theoretical_loss": 3.765574543498369, + "tokens_seen": 783024128 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003852156469408225, + "loss": 1.3391, + "theoretical_loss": 3.765543418651927, + "tokens_seen": 783089664 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 265126, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3448538780212402, + "objective/train/theoretical_loss": 3.765512297139461, + "objective/train/tokens_used": 783155200, + "theoretical_loss": 3.765512297139461, + "tokens_seen": 783155200 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038520561685055166, + "loss": 1.3644, + "theoretical_loss": 3.765512297139461, + "tokens_seen": 783155200 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038519558676028084, + "loss": 1.4324, + "theoretical_loss": 3.765481178960333, + "tokens_seen": 783220736 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038518555667001, + "loss": 1.4458, + "theoretical_loss": 3.765450064113909, + "tokens_seen": 783286272 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003851755265797392, + "loss": 1.3813, + "theoretical_loss": 3.7654189525995516, + "tokens_seen": 783351808 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038516549648946844, + "loss": 1.4728, + "theoretical_loss": 3.7653878444166273, + "tokens_seen": 783417344 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038515546639919756, + "loss": 1.3281, + "theoretical_loss": 3.765356739564499, + "tokens_seen": 783482880 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003851454363089268, + "loss": 1.2971, + "theoretical_loss": 3.7653256380425324, + "tokens_seen": 783548416 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003851354062186559, + "loss": 1.3588, + "theoretical_loss": 3.7652945398500917, + "tokens_seen": 783613952 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038512537612838516, + "loss": 1.3675, + "theoretical_loss": 3.7652634449865428, + "tokens_seen": 783679488 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003851153460381144, + "loss": 1.4008, + "theoretical_loss": 3.7652323534512506, + "tokens_seen": 783745024 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003851053159478435, + "loss": 1.4525, + "theoretical_loss": 3.765201265243581, + "tokens_seen": 783810560 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038509528585757276, + "loss": 1.4143, + "theoretical_loss": 3.765170180362899, + "tokens_seen": 783876096 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003850852557673019, + "loss": 1.507, + "theoretical_loss": 3.7651390988085716, + "tokens_seen": 783941632 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003850752256770311, + "loss": 1.4473, + "theoretical_loss": 3.765108020579963, + "tokens_seen": 784007168 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003850651955867603, + "loss": 1.4801, + "theoretical_loss": 3.7650769456764417, + "tokens_seen": 784072704 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003850551654964895, + "loss": 1.3564, + "theoretical_loss": 3.765045874097372, + "tokens_seen": 784138240 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038504513540621866, + "loss": 1.3716, + "theoretical_loss": 3.7650148058421213, + "tokens_seen": 784203776 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003850351053159479, + "loss": 1.3841, + "theoretical_loss": 3.7649837409100564, + "tokens_seen": 784269312 + }, + { + "epoch": 0.24, + "learning_rate": 0.000385025075225677, + "loss": 1.4352, + "theoretical_loss": 3.7649526793005443, + "tokens_seen": 784334848 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038501504513540626, + "loss": 1.3847, + "theoretical_loss": 3.7649216210129515, + "tokens_seen": 784400384 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003850050150451354, + "loss": 1.3029, + "theoretical_loss": 3.764890566046646, + "tokens_seen": 784465920 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003849949849548646, + "loss": 1.3967, + "theoretical_loss": 3.764859514400994, + "tokens_seen": 784531456 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003849849548645938, + "loss": 1.3791, + "theoretical_loss": 3.7648284660753646, + "tokens_seen": 784596992 + }, + { + "epoch": 0.24, + "learning_rate": 0.000384974924774323, + "loss": 1.4593, + "theoretical_loss": 3.7647974210691246, + "tokens_seen": 784662528 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038496489468405217, + "loss": 1.3658, + "theoretical_loss": 3.7647663793816424, + "tokens_seen": 784728064 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038495486459378135, + "loss": 1.4707, + "theoretical_loss": 3.764735341012285, + "tokens_seen": 784793600 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038494483450351053, + "loss": 1.3688, + "theoretical_loss": 3.7647043059604224, + "tokens_seen": 784859136 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038493480441323977, + "loss": 1.3945, + "theoretical_loss": 3.7646732742254216, + "tokens_seen": 784924672 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003849247743229689, + "loss": 1.2911, + "theoretical_loss": 3.764642245806652, + "tokens_seen": 784990208 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038491474423269813, + "loss": 1.3624, + "theoretical_loss": 3.7646112207034825, + "tokens_seen": 785055744 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038490471414242725, + "loss": 1.3169, + "theoretical_loss": 3.764580198915281, + "tokens_seen": 785121280 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003848946840521565, + "loss": 1.3972, + "theoretical_loss": 3.764549180441418, + "tokens_seen": 785186816 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038488465396188567, + "loss": 1.3092, + "theoretical_loss": 3.7645181652812623, + "tokens_seen": 785252352 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038487462387161485, + "loss": 1.4023, + "theoretical_loss": 3.7644871534341826, + "tokens_seen": 785317888 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038486459378134403, + "loss": 1.3786, + "theoretical_loss": 3.7644561448995497, + "tokens_seen": 785383424 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038485456369107327, + "loss": 1.4146, + "theoretical_loss": 3.764425139676733, + "tokens_seen": 785448960 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003848445336008024, + "loss": 1.282, + "theoretical_loss": 3.7643941377651022, + "tokens_seen": 785514496 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038483450351053163, + "loss": 1.3758, + "theoretical_loss": 3.7643631391640278, + "tokens_seen": 785580032 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038482447342026076, + "loss": 1.5554, + "theoretical_loss": 3.7643321438728803, + "tokens_seen": 785645568 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038481444332999, + "loss": 1.3411, + "theoretical_loss": 3.7643011518910297, + "tokens_seen": 785711104 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003848044132397192, + "loss": 1.4073, + "theoretical_loss": 3.764270163217847, + "tokens_seen": 785776640 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038479438314944836, + "loss": 1.4064, + "theoretical_loss": 3.764239177852703, + "tokens_seen": 785842176 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038478435305917754, + "loss": 1.4086, + "theoretical_loss": 3.764208195794969, + "tokens_seen": 785907712 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003847743229689067, + "loss": 1.357, + "theoretical_loss": 3.7641772170440158, + "tokens_seen": 785973248 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003847642928786359, + "loss": 1.3277, + "theoretical_loss": 3.7641462415992155, + "tokens_seen": 786038784 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038475426278836513, + "loss": 1.3685, + "theoretical_loss": 3.7641152694599382, + "tokens_seen": 786104320 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038474423269809426, + "loss": 1.3569, + "theoretical_loss": 3.764084300625557, + "tokens_seen": 786169856 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003847342026078235, + "loss": 1.366, + "theoretical_loss": 3.7640533350954435, + "tokens_seen": 786235392 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003847241725175527, + "loss": 1.4366, + "theoretical_loss": 3.7640223728689692, + "tokens_seen": 786300928 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038471414242728186, + "loss": 1.4065, + "theoretical_loss": 3.763991413945507, + "tokens_seen": 786366464 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 266440, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4344490766525269, + "objective/train/theoretical_loss": 3.763960458324429, + "objective/train/tokens_used": 786432000, + "theoretical_loss": 3.763960458324429, + "tokens_seen": 786432000 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038470411233701104, + "loss": 1.4155, + "theoretical_loss": 3.763960458324429, + "tokens_seen": 786432000 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003846940822467402, + "loss": 1.4489, + "theoretical_loss": 3.763929506005108, + "tokens_seen": 786497536 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003846840521564694, + "loss": 1.5513, + "theoretical_loss": 3.7638985569869163, + "tokens_seen": 786563072 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038467402206619864, + "loss": 1.3261, + "theoretical_loss": 3.7638676112692275, + "tokens_seen": 786628608 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038466399197592776, + "loss": 1.4464, + "theoretical_loss": 3.7638366688514138, + "tokens_seen": 786694144 + }, + { + "epoch": 0.24, + "learning_rate": 0.000384653961885657, + "loss": 1.4254, + "theoretical_loss": 3.7638057297328493, + "tokens_seen": 786759680 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003846439317953861, + "loss": 1.4014, + "theoretical_loss": 3.7637747939129067, + "tokens_seen": 786825216 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038463390170511536, + "loss": 1.4377, + "theoretical_loss": 3.7637438613909606, + "tokens_seen": 786890752 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038462387161484454, + "loss": 1.4519, + "theoretical_loss": 3.7637129321663836, + "tokens_seen": 786956288 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003846138415245737, + "loss": 1.3608, + "theoretical_loss": 3.7636820062385503, + "tokens_seen": 787021824 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003846038114343029, + "loss": 1.4694, + "theoretical_loss": 3.7636510836068355, + "tokens_seen": 787087360 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003845937813440321, + "loss": 1.3186, + "theoretical_loss": 3.7636201642706117, + "tokens_seen": 787152896 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038458375125376127, + "loss": 1.3967, + "theoretical_loss": 3.7635892482292554, + "tokens_seen": 787218432 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003845737211634905, + "loss": 1.3127, + "theoretical_loss": 3.7635583354821396, + "tokens_seen": 787283968 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038456369107321963, + "loss": 1.4398, + "theoretical_loss": 3.7635274260286398, + "tokens_seen": 787349504 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038455366098294886, + "loss": 1.4733, + "theoretical_loss": 3.763496519868131, + "tokens_seen": 787415040 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038454363089267805, + "loss": 1.4247, + "theoretical_loss": 3.7634656169999885, + "tokens_seen": 787480576 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038453360080240723, + "loss": 1.4566, + "theoretical_loss": 3.7634347174235874, + "tokens_seen": 787546112 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003845235707121364, + "loss": 1.4386, + "theoretical_loss": 3.7634038211383025, + "tokens_seen": 787611648 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003845135406218656, + "loss": 1.4582, + "theoretical_loss": 3.763372928143511, + "tokens_seen": 787677184 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038450351053159477, + "loss": 1.3988, + "theoretical_loss": 3.763342038438587, + "tokens_seen": 787742720 + }, + { + "epoch": 0.24, + "learning_rate": 0.000384493480441324, + "loss": 1.3935, + "theoretical_loss": 3.7633111520229083, + "tokens_seen": 787808256 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038448345035105313, + "loss": 1.4011, + "theoretical_loss": 3.7632802688958495, + "tokens_seen": 787873792 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038447342026078237, + "loss": 1.3653, + "theoretical_loss": 3.7632493890567877, + "tokens_seen": 787939328 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003844633901705115, + "loss": 1.4479, + "theoretical_loss": 3.7632185125051, + "tokens_seen": 788004864 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038445336008024073, + "loss": 1.3608, + "theoretical_loss": 3.763187639240161, + "tokens_seen": 788070400 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003844433299899699, + "loss": 1.4255, + "theoretical_loss": 3.7631567692613497, + "tokens_seen": 788135936 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003844332998996991, + "loss": 1.3483, + "theoretical_loss": 3.763125902568042, + "tokens_seen": 788201472 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003844232698094283, + "loss": 1.3836, + "theoretical_loss": 3.763095039159616, + "tokens_seen": 788267008 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038441323971915745, + "loss": 1.3874, + "theoretical_loss": 3.7630641790354478, + "tokens_seen": 788332544 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038440320962888664, + "loss": 1.4159, + "theoretical_loss": 3.763033322194916, + "tokens_seen": 788398080 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038439317953861587, + "loss": 1.3185, + "theoretical_loss": 3.763002468637398, + "tokens_seen": 788463616 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038438314944834505, + "loss": 1.35, + "theoretical_loss": 3.762971618362271, + "tokens_seen": 788529152 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038437311935807423, + "loss": 1.3793, + "theoretical_loss": 3.762940771368914, + "tokens_seen": 788594688 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038436308926780347, + "loss": 1.3543, + "theoretical_loss": 3.762909927656705, + "tokens_seen": 788660224 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003843530591775326, + "loss": 1.4432, + "theoretical_loss": 3.7628790872250217, + "tokens_seen": 788725760 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038434302908726183, + "loss": 1.3973, + "theoretical_loss": 3.762848250073244, + "tokens_seen": 788791296 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038433299899699096, + "loss": 1.4232, + "theoretical_loss": 3.7628174162007486, + "tokens_seen": 788856832 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003843229689067202, + "loss": 1.4575, + "theoretical_loss": 3.762786585606916, + "tokens_seen": 788922368 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003843129388164494, + "loss": 1.3832, + "theoretical_loss": 3.7627557582911253, + "tokens_seen": 788987904 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038430290872617856, + "loss": 1.4057, + "theoretical_loss": 3.7627249342527547, + "tokens_seen": 789053440 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038429287863590774, + "loss": 1.3308, + "theoretical_loss": 3.762694113491184, + "tokens_seen": 789118976 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003842828485456369, + "loss": 1.3385, + "theoretical_loss": 3.762663296005793, + "tokens_seen": 789184512 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003842728184553661, + "loss": 1.4845, + "theoretical_loss": 3.7626324817959613, + "tokens_seen": 789250048 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038426278836509533, + "loss": 1.3744, + "theoretical_loss": 3.762601670861069, + "tokens_seen": 789315584 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038425275827482446, + "loss": 1.4858, + "theoretical_loss": 3.7625708632004953, + "tokens_seen": 789381120 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003842427281845537, + "loss": 1.3864, + "theoretical_loss": 3.762540058813622, + "tokens_seen": 789446656 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003842326980942829, + "loss": 1.313, + "theoretical_loss": 3.762509257699828, + "tokens_seen": 789512192 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038422266800401206, + "loss": 1.4242, + "theoretical_loss": 3.7624784598584946, + "tokens_seen": 789577728 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038421263791374124, + "loss": 1.4373, + "theoretical_loss": 3.7624476652890024, + "tokens_seen": 789643264 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 267756, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4175002574920654, + "objective/train/theoretical_loss": 3.762416873990732, + "objective/train/tokens_used": 789708800, + "theoretical_loss": 3.762416873990732, + "tokens_seen": 789708800 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003842026078234704, + "loss": 1.4581, + "theoretical_loss": 3.762416873990732, + "tokens_seen": 789708800 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003841925777331996, + "loss": 1.4456, + "theoretical_loss": 3.7623860859630653, + "tokens_seen": 789774336 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038418254764292884, + "loss": 1.3428, + "theoretical_loss": 3.762355301205383, + "tokens_seen": 789839872 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038417251755265796, + "loss": 1.3844, + "theoretical_loss": 3.7623245197170663, + "tokens_seen": 789905408 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003841624874623872, + "loss": 1.2894, + "theoretical_loss": 3.7622937414974977, + "tokens_seen": 789970944 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003841524573721163, + "loss": 1.3511, + "theoretical_loss": 3.762262966546058, + "tokens_seen": 790036480 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038414242728184556, + "loss": 1.437, + "theoretical_loss": 3.762232194862129, + "tokens_seen": 790102016 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038413239719157474, + "loss": 1.4198, + "theoretical_loss": 3.7622014264450936, + "tokens_seen": 790167552 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003841223671013039, + "loss": 1.3361, + "theoretical_loss": 3.7621706612943333, + "tokens_seen": 790233088 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003841123370110331, + "loss": 1.3954, + "theoretical_loss": 3.7621398994092314, + "tokens_seen": 790298624 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003841023069207623, + "loss": 1.3583, + "theoretical_loss": 3.76210914078917, + "tokens_seen": 790364160 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038409227683049147, + "loss": 1.404, + "theoretical_loss": 3.762078385433531, + "tokens_seen": 790429696 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003840822467402207, + "loss": 1.4138, + "theoretical_loss": 3.7620476333416994, + "tokens_seen": 790495232 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038407221664994983, + "loss": 1.4214, + "theoretical_loss": 3.7620168845130566, + "tokens_seen": 790560768 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038406218655967907, + "loss": 1.3964, + "theoretical_loss": 3.761986138946986, + "tokens_seen": 790626304 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038405215646940825, + "loss": 1.3697, + "theoretical_loss": 3.761955396642872, + "tokens_seen": 790691840 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038404212637913743, + "loss": 1.3989, + "theoretical_loss": 3.761924657600097, + "tokens_seen": 790757376 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003840320962888666, + "loss": 1.3992, + "theoretical_loss": 3.7618939218180456, + "tokens_seen": 790822912 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003840220661985958, + "loss": 1.3641, + "theoretical_loss": 3.7618631892961014, + "tokens_seen": 790888448 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038401203610832497, + "loss": 1.456, + "theoretical_loss": 3.7618324600336486, + "tokens_seen": 790953984 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003840020060180542, + "loss": 1.4862, + "theoretical_loss": 3.7618017340300716, + "tokens_seen": 791019520 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038399197592778333, + "loss": 1.4017, + "theoretical_loss": 3.761771011284755, + "tokens_seen": 791085056 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038398194583751257, + "loss": 1.3877, + "theoretical_loss": 3.761740291797082, + "tokens_seen": 791150592 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003839719157472417, + "loss": 1.415, + "theoretical_loss": 3.7617095755664396, + "tokens_seen": 791216128 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038396188565697093, + "loss": 1.3751, + "theoretical_loss": 3.761678862592211, + "tokens_seen": 791281664 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003839518555667001, + "loss": 1.2903, + "theoretical_loss": 3.761648152873782, + "tokens_seen": 791347200 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003839418254764293, + "loss": 1.4147, + "theoretical_loss": 3.761617446410538, + "tokens_seen": 791412736 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003839317953861585, + "loss": 1.4164, + "theoretical_loss": 3.761586743201864, + "tokens_seen": 791478272 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038392176529588765, + "loss": 1.4316, + "theoretical_loss": 3.761556043247146, + "tokens_seen": 791543808 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038391173520561684, + "loss": 1.5085, + "theoretical_loss": 3.7615253465457696, + "tokens_seen": 791609344 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038390170511534607, + "loss": 1.2648, + "theoretical_loss": 3.7614946530971207, + "tokens_seen": 791674880 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003838916750250752, + "loss": 1.3331, + "theoretical_loss": 3.7614639629005855, + "tokens_seen": 791740416 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038388164493480443, + "loss": 1.4572, + "theoretical_loss": 3.76143327595555, + "tokens_seen": 791805952 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003838716148445336, + "loss": 1.4483, + "theoretical_loss": 3.761402592261401, + "tokens_seen": 791871488 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003838615847542628, + "loss": 1.4067, + "theoretical_loss": 3.761371911817525, + "tokens_seen": 791937024 + }, + { + "epoch": 0.24, + "learning_rate": 0.000383851554663992, + "loss": 1.3976, + "theoretical_loss": 3.761341234623309, + "tokens_seen": 792002560 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038384152457372116, + "loss": 1.3803, + "theoretical_loss": 3.7613105606781394, + "tokens_seen": 792068096 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038383149448345034, + "loss": 1.3655, + "theoretical_loss": 3.7612798899814037, + "tokens_seen": 792133632 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003838214643931796, + "loss": 1.3776, + "theoretical_loss": 3.7612492225324887, + "tokens_seen": 792199168 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003838114343029087, + "loss": 1.4006, + "theoretical_loss": 3.761218558330783, + "tokens_seen": 792264704 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038380140421263794, + "loss": 1.3208, + "theoretical_loss": 3.761187897375673, + "tokens_seen": 792330240 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038379137412236706, + "loss": 1.5009, + "theoretical_loss": 3.761157239666547, + "tokens_seen": 792395776 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003837813440320963, + "loss": 1.3956, + "theoretical_loss": 3.761126585202792, + "tokens_seen": 792461312 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003837713139418255, + "loss": 1.347, + "theoretical_loss": 3.761095933983798, + "tokens_seen": 792526848 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038376128385155466, + "loss": 1.3289, + "theoretical_loss": 3.7610652860089515, + "tokens_seen": 792592384 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038375125376128384, + "loss": 1.3679, + "theoretical_loss": 3.761034641277642, + "tokens_seen": 792657920 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003837412236710131, + "loss": 1.3612, + "theoretical_loss": 3.7610039997892577, + "tokens_seen": 792723456 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003837311935807422, + "loss": 1.3797, + "theoretical_loss": 3.7609733615431873, + "tokens_seen": 792788992 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038372116349047144, + "loss": 1.3184, + "theoretical_loss": 3.76094272653882, + "tokens_seen": 792854528 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038371113340020057, + "loss": 1.3642, + "theoretical_loss": 3.760912094775544, + "tokens_seen": 792920064 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 268636, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.438559651374817, + "objective/train/theoretical_loss": 3.7608814662527505, + "objective/train/tokens_used": 792985600, + "theoretical_loss": 3.7608814662527505, + "tokens_seen": 792985600 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003837011033099298, + "loss": 1.5353, + "theoretical_loss": 3.7608814662527505, + "tokens_seen": 792985600 + }, + { + "epoch": 0.24, + "learning_rate": 0.000383691073219659, + "loss": 1.3923, + "theoretical_loss": 3.7608508409698267, + "tokens_seen": 793051136 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038368104312938816, + "loss": 1.4271, + "theoretical_loss": 3.7608202189261637, + "tokens_seen": 793116672 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038367101303911735, + "loss": 1.4113, + "theoretical_loss": 3.760789600121151, + "tokens_seen": 793182208 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003836609829488465, + "loss": 1.5514, + "theoretical_loss": 3.760758984554178, + "tokens_seen": 793247744 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003836509528585757, + "loss": 1.3406, + "theoretical_loss": 3.7607283722246345, + "tokens_seen": 793313280 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038364092276830494, + "loss": 1.3716, + "theoretical_loss": 3.760697763131912, + "tokens_seen": 793378816 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003836308926780341, + "loss": 1.4495, + "theoretical_loss": 3.7606671572754005, + "tokens_seen": 793444352 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003836208625877633, + "loss": 1.4384, + "theoretical_loss": 3.7606365546544898, + "tokens_seen": 793509888 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003836108324974925, + "loss": 1.3761, + "theoretical_loss": 3.7606059552685713, + "tokens_seen": 793575424 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038360080240722167, + "loss": 1.3959, + "theoretical_loss": 3.760575359117036, + "tokens_seen": 793640960 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003835907723169509, + "loss": 1.5219, + "theoretical_loss": 3.760544766199275, + "tokens_seen": 793706496 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038358074222668003, + "loss": 1.2779, + "theoretical_loss": 3.760514176514679, + "tokens_seen": 793772032 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038357071213640927, + "loss": 1.3128, + "theoretical_loss": 3.7604835900626394, + "tokens_seen": 793837568 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038356068204613845, + "loss": 1.417, + "theoretical_loss": 3.760453006842549, + "tokens_seen": 793903104 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038355065195586763, + "loss": 1.3531, + "theoretical_loss": 3.7604224268537982, + "tokens_seen": 793968640 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003835406218655968, + "loss": 1.4578, + "theoretical_loss": 3.7603918500957794, + "tokens_seen": 794034176 + }, + { + "epoch": 0.24, + "learning_rate": 0.000383530591775326, + "loss": 1.4031, + "theoretical_loss": 3.7603612765678847, + "tokens_seen": 794099712 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038352056168505517, + "loss": 1.4506, + "theoretical_loss": 3.7603307062695066, + "tokens_seen": 794165248 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003835105315947844, + "loss": 1.4032, + "theoretical_loss": 3.7603001392000364, + "tokens_seen": 794230784 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038350050150451353, + "loss": 1.4892, + "theoretical_loss": 3.760269575358868, + "tokens_seen": 794296320 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038349047141424277, + "loss": 1.2981, + "theoretical_loss": 3.7602390147453937, + "tokens_seen": 794361856 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003834804413239719, + "loss": 1.42, + "theoretical_loss": 3.760208457359006, + "tokens_seen": 794427392 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038347041123370113, + "loss": 1.3722, + "theoretical_loss": 3.7601779031990983, + "tokens_seen": 794492928 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003834603811434303, + "loss": 1.3087, + "theoretical_loss": 3.7601473522650637, + "tokens_seen": 794558464 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003834503510531595, + "loss": 1.291, + "theoretical_loss": 3.7601168045562954, + "tokens_seen": 794624000 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003834403209628887, + "loss": 1.3915, + "theoretical_loss": 3.760086260072188, + "tokens_seen": 794689536 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038343029087261786, + "loss": 1.2879, + "theoretical_loss": 3.7600557188121337, + "tokens_seen": 794755072 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038342026078234704, + "loss": 1.3856, + "theoretical_loss": 3.7600251807755276, + "tokens_seen": 794820608 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038341023069207627, + "loss": 1.3512, + "theoretical_loss": 3.759994645961762, + "tokens_seen": 794886144 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003834002006018054, + "loss": 1.3404, + "theoretical_loss": 3.759964114370234, + "tokens_seen": 794951680 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038339017051153463, + "loss": 1.2996, + "theoretical_loss": 3.759933586000335, + "tokens_seen": 795017216 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003833801404212638, + "loss": 1.3849, + "theoretical_loss": 3.759903060851461, + "tokens_seen": 795082752 + }, + { + "epoch": 0.24, + "learning_rate": 0.000383370110330993, + "loss": 1.4054, + "theoretical_loss": 3.759872538923007, + "tokens_seen": 795148288 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003833600802407222, + "loss": 1.4285, + "theoretical_loss": 3.759842020214367, + "tokens_seen": 795213824 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038335005015045136, + "loss": 1.3951, + "theoretical_loss": 3.7598115047249365, + "tokens_seen": 795279360 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038334002006018054, + "loss": 1.3307, + "theoretical_loss": 3.759780992454111, + "tokens_seen": 795344896 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003833299899699098, + "loss": 1.3331, + "theoretical_loss": 3.7597504834012847, + "tokens_seen": 795410432 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003833199598796389, + "loss": 1.3897, + "theoretical_loss": 3.7597199775658545, + "tokens_seen": 795475968 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038330992978936814, + "loss": 1.4442, + "theoretical_loss": 3.7596894749472147, + "tokens_seen": 795541504 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038329989969909726, + "loss": 1.3634, + "theoretical_loss": 3.7596589755447622, + "tokens_seen": 795607040 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003832898696088265, + "loss": 1.3909, + "theoretical_loss": 3.759628479357893, + "tokens_seen": 795672576 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003832798395185557, + "loss": 1.3957, + "theoretical_loss": 3.7595979863860025, + "tokens_seen": 795738112 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038326980942828486, + "loss": 1.2753, + "theoretical_loss": 3.759567496628488, + "tokens_seen": 795803648 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038325977933801404, + "loss": 1.4124, + "theoretical_loss": 3.759537010084745, + "tokens_seen": 795869184 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003832497492477433, + "loss": 1.3929, + "theoretical_loss": 3.759506526754171, + "tokens_seen": 795934720 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003832397191574724, + "loss": 1.4118, + "theoretical_loss": 3.759476046636162, + "tokens_seen": 796000256 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038322968906720164, + "loss": 1.4005, + "theoretical_loss": 3.759445569730115, + "tokens_seen": 796065792 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038321965897693077, + "loss": 1.3994, + "theoretical_loss": 3.7594150960354282, + "tokens_seen": 796131328 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038320962888666, + "loss": 1.5606, + "theoretical_loss": 3.7593846255514984, + "tokens_seen": 796196864 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 269782, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3991293907165527, + "objective/train/theoretical_loss": 3.7593541582777226, + "objective/train/tokens_used": 796262400, + "theoretical_loss": 3.7593541582777226, + "tokens_seen": 796262400 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003831995987963892, + "loss": 1.3657, + "theoretical_loss": 3.7593541582777226, + "tokens_seen": 796262400 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038318956870611836, + "loss": 1.3352, + "theoretical_loss": 3.7593236942134993, + "tokens_seen": 796327936 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038317953861584755, + "loss": 1.345, + "theoretical_loss": 3.7592932333582247, + "tokens_seen": 796393472 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003831695085255767, + "loss": 1.4472, + "theoretical_loss": 3.759262775711299, + "tokens_seen": 796459008 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003831594784353059, + "loss": 1.3875, + "theoretical_loss": 3.759232321272118, + "tokens_seen": 796524544 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038314944834503514, + "loss": 1.4437, + "theoretical_loss": 3.7592018700400818, + "tokens_seen": 796590080 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038313941825476427, + "loss": 1.4157, + "theoretical_loss": 3.759171422014588, + "tokens_seen": 796655616 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003831293881644935, + "loss": 1.3658, + "theoretical_loss": 3.7591409771950355, + "tokens_seen": 796721152 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038311935807422263, + "loss": 1.3136, + "theoretical_loss": 3.759110535580823, + "tokens_seen": 796786688 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038310932798395187, + "loss": 1.2985, + "theoretical_loss": 3.759080097171349, + "tokens_seen": 796852224 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038309929789368105, + "loss": 1.3943, + "theoretical_loss": 3.7590496619660136, + "tokens_seen": 796917760 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038308926780341023, + "loss": 1.4396, + "theoretical_loss": 3.759019229964215, + "tokens_seen": 796983296 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003830792377131394, + "loss": 1.2957, + "theoretical_loss": 3.758988801165353, + "tokens_seen": 797048832 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038306920762286865, + "loss": 1.285, + "theoretical_loss": 3.7589583755688274, + "tokens_seen": 797114368 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003830591775325978, + "loss": 1.4313, + "theoretical_loss": 3.7589279531740374, + "tokens_seen": 797179904 + }, + { + "epoch": 0.24, + "learning_rate": 0.000383049147442327, + "loss": 1.4444, + "theoretical_loss": 3.7588975339803836, + "tokens_seen": 797245440 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038303911735205614, + "loss": 1.409, + "theoretical_loss": 3.7588671179872657, + "tokens_seen": 797310976 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038302908726178537, + "loss": 1.3593, + "theoretical_loss": 3.7588367051940836, + "tokens_seen": 797376512 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038301905717151455, + "loss": 1.3616, + "theoretical_loss": 3.7588062956002384, + "tokens_seen": 797442048 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038300902708124373, + "loss": 1.4214, + "theoretical_loss": 3.75877588920513, + "tokens_seen": 797507584 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003829989969909729, + "loss": 1.4732, + "theoretical_loss": 3.7587454860081597, + "tokens_seen": 797573120 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003829889669007021, + "loss": 1.3446, + "theoretical_loss": 3.7587150860087277, + "tokens_seen": 797638656 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003829789368104313, + "loss": 1.4237, + "theoretical_loss": 3.7586846892062358, + "tokens_seen": 797704192 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003829689067201605, + "loss": 1.3777, + "theoretical_loss": 3.758654295600085, + "tokens_seen": 797769728 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038295887662988964, + "loss": 1.4095, + "theoretical_loss": 3.758623905189676, + "tokens_seen": 797835264 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003829488465396189, + "loss": 1.3212, + "theoretical_loss": 3.758593517974411, + "tokens_seen": 797900800 + }, + { + "epoch": 0.24, + "learning_rate": 0.000382938816449348, + "loss": 1.4054, + "theoretical_loss": 3.7585631339536913, + "tokens_seen": 797966336 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038292878635907724, + "loss": 1.4107, + "theoretical_loss": 3.7585327531269197, + "tokens_seen": 798031872 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003829187562688064, + "loss": 1.3559, + "theoretical_loss": 3.7585023754934963, + "tokens_seen": 798097408 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003829087261785356, + "loss": 1.4303, + "theoretical_loss": 3.7584720010528248, + "tokens_seen": 798162944 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003828986960882648, + "loss": 1.3154, + "theoretical_loss": 3.7584416298043073, + "tokens_seen": 798228480 + }, + { + "epoch": 0.24, + "learning_rate": 0.000382888665997994, + "loss": 1.437, + "theoretical_loss": 3.758411261747346, + "tokens_seen": 798294016 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003828786359077232, + "loss": 1.285, + "theoretical_loss": 3.7583808968813437, + "tokens_seen": 798359552 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003828686058174524, + "loss": 1.3417, + "theoretical_loss": 3.758350535205703, + "tokens_seen": 798425088 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038285857572718156, + "loss": 1.3528, + "theoretical_loss": 3.7583201767198267, + "tokens_seen": 798490624 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038284854563691074, + "loss": 1.3069, + "theoretical_loss": 3.7582898214231184, + "tokens_seen": 798556160 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038283851554664, + "loss": 1.3915, + "theoretical_loss": 3.7582594693149813, + "tokens_seen": 798621696 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003828284854563691, + "loss": 1.3072, + "theoretical_loss": 3.7582291203948195, + "tokens_seen": 798687232 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038281845536609834, + "loss": 1.3509, + "theoretical_loss": 3.758198774662035, + "tokens_seen": 798752768 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038280842527582746, + "loss": 1.4127, + "theoretical_loss": 3.7581684321160322, + "tokens_seen": 798818304 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003827983951855567, + "loss": 1.3787, + "theoretical_loss": 3.758138092756216, + "tokens_seen": 798883840 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003827883650952859, + "loss": 1.4508, + "theoretical_loss": 3.75810775658199, + "tokens_seen": 798949376 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038277833500501506, + "loss": 1.4534, + "theoretical_loss": 3.7580774235927574, + "tokens_seen": 799014912 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038276830491474424, + "loss": 1.4431, + "theoretical_loss": 3.7580470937879236, + "tokens_seen": 799080448 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003827582748244735, + "loss": 1.3072, + "theoretical_loss": 3.758016767166893, + "tokens_seen": 799145984 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003827482447342026, + "loss": 1.3391, + "theoretical_loss": 3.757986443729071, + "tokens_seen": 799211520 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038273821464393184, + "loss": 1.3518, + "theoretical_loss": 3.7579561234738614, + "tokens_seen": 799277056 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038272818455366097, + "loss": 1.3305, + "theoretical_loss": 3.75792580640067, + "tokens_seen": 799342592 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003827181544633902, + "loss": 1.2625, + "theoretical_loss": 3.7578954925089008, + "tokens_seen": 799408128 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003827081243731194, + "loss": 1.4137, + "theoretical_loss": 3.757865181797961, + "tokens_seen": 799473664 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 270647, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4399938583374023, + "objective/train/theoretical_loss": 3.757834874267255, + "objective/train/tokens_used": 799539200, + "theoretical_loss": 3.757834874267255, + "tokens_seen": 799539200 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038269809428284856, + "loss": 1.4036, + "theoretical_loss": 3.757834874267255, + "tokens_seen": 799539200 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038268806419257775, + "loss": 1.484, + "theoretical_loss": 3.757804569916188, + "tokens_seen": 799604736 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003826780341023069, + "loss": 1.491, + "theoretical_loss": 3.7577742687441678, + "tokens_seen": 799670272 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003826680040120361, + "loss": 1.3694, + "theoretical_loss": 3.757743970750598, + "tokens_seen": 799735808 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038265797392176534, + "loss": 1.2685, + "theoretical_loss": 3.7577136759348866, + "tokens_seen": 799801344 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038264794383149447, + "loss": 1.3125, + "theoretical_loss": 3.757683384296439, + "tokens_seen": 799866880 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003826379137412237, + "loss": 1.3306, + "theoretical_loss": 3.7576530958346623, + "tokens_seen": 799932416 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038262788365095283, + "loss": 1.3885, + "theoretical_loss": 3.7576228105489626, + "tokens_seen": 799997952 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038261785356068207, + "loss": 1.2871, + "theoretical_loss": 3.757592528438747, + "tokens_seen": 800063488 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038260782347041125, + "loss": 1.3216, + "theoretical_loss": 3.7575622495034224, + "tokens_seen": 800129024 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038259779338014043, + "loss": 1.3128, + "theoretical_loss": 3.7575319737423953, + "tokens_seen": 800194560 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003825877632898696, + "loss": 1.3145, + "theoretical_loss": 3.757501701155074, + "tokens_seen": 800260096 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038257773319959885, + "loss": 1.3254, + "theoretical_loss": 3.757471431740866, + "tokens_seen": 800325632 + }, + { + "epoch": 0.24, + "learning_rate": 0.000382567703109328, + "loss": 1.4336, + "theoretical_loss": 3.757441165499178, + "tokens_seen": 800391168 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003825576730190572, + "loss": 1.4067, + "theoretical_loss": 3.7574109024294184, + "tokens_seen": 800456704 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038254764292878634, + "loss": 1.2791, + "theoretical_loss": 3.757380642530995, + "tokens_seen": 800522240 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038253761283851557, + "loss": 1.406, + "theoretical_loss": 3.7573503858033157, + "tokens_seen": 800587776 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038252758274824475, + "loss": 1.3217, + "theoretical_loss": 3.7573201322457885, + "tokens_seen": 800653312 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038251755265797393, + "loss": 1.38, + "theoretical_loss": 3.757289881857823, + "tokens_seen": 800718848 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003825075225677031, + "loss": 1.4669, + "theoretical_loss": 3.7572596346388263, + "tokens_seen": 800784384 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003824974924774323, + "loss": 1.3623, + "theoretical_loss": 3.7572293905882077, + "tokens_seen": 800849920 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003824874623871615, + "loss": 1.3165, + "theoretical_loss": 3.7571991497053765, + "tokens_seen": 800915456 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003824774322968907, + "loss": 1.3681, + "theoretical_loss": 3.757168911989741, + "tokens_seen": 800980992 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038246740220661984, + "loss": 1.4447, + "theoretical_loss": 3.7571386774407114, + "tokens_seen": 801046528 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003824573721163491, + "loss": 1.2445, + "theoretical_loss": 3.757108446057696, + "tokens_seen": 801112064 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003824473420260782, + "loss": 1.3046, + "theoretical_loss": 3.7570782178401045, + "tokens_seen": 801177600 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038243731193580744, + "loss": 1.3492, + "theoretical_loss": 3.757047992787347, + "tokens_seen": 801243136 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003824272818455366, + "loss": 1.3807, + "theoretical_loss": 3.757017770898833, + "tokens_seen": 801308672 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003824172517552658, + "loss": 1.447, + "theoretical_loss": 3.756987552173973, + "tokens_seen": 801374208 + }, + { + "epoch": 0.24, + "learning_rate": 0.000382407221664995, + "loss": 1.3631, + "theoretical_loss": 3.7569573366121762, + "tokens_seen": 801439744 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003823971915747242, + "loss": 1.2988, + "theoretical_loss": 3.7569271242128535, + "tokens_seen": 801505280 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038238716148445334, + "loss": 1.3977, + "theoretical_loss": 3.7568969149754157, + "tokens_seen": 801570816 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003823771313941826, + "loss": 1.3825, + "theoretical_loss": 3.756866708899273, + "tokens_seen": 801636352 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003823671013039117, + "loss": 1.2762, + "theoretical_loss": 3.756836505983836, + "tokens_seen": 801701888 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038235707121364094, + "loss": 1.3848, + "theoretical_loss": 3.756806306228516, + "tokens_seen": 801767424 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003823470411233701, + "loss": 1.3313, + "theoretical_loss": 3.7567761096327237, + "tokens_seen": 801832960 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003823370110330993, + "loss": 1.2999, + "theoretical_loss": 3.7567459161958707, + "tokens_seen": 801898496 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003823269809428285, + "loss": 1.3199, + "theoretical_loss": 3.7567157259173682, + "tokens_seen": 801964032 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038231695085255766, + "loss": 1.3243, + "theoretical_loss": 3.756685538796628, + "tokens_seen": 802029568 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038230692076228685, + "loss": 1.3842, + "theoretical_loss": 3.756655354833062, + "tokens_seen": 802095104 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003822968906720161, + "loss": 1.4075, + "theoretical_loss": 3.756625174026081, + "tokens_seen": 802160640 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003822868605817452, + "loss": 1.3077, + "theoretical_loss": 3.756594996375098, + "tokens_seen": 802226176 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038227683049147444, + "loss": 1.4627, + "theoretical_loss": 3.7565648218795253, + "tokens_seen": 802291712 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038226680040120357, + "loss": 1.3496, + "theoretical_loss": 3.756534650538775, + "tokens_seen": 802357248 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003822567703109328, + "loss": 1.3088, + "theoretical_loss": 3.7565044823522595, + "tokens_seen": 802422784 + }, + { + "epoch": 0.24, + "learning_rate": 0.000382246740220662, + "loss": 1.2201, + "theoretical_loss": 3.7564743173193915, + "tokens_seen": 802488320 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038223671013039117, + "loss": 1.3453, + "theoretical_loss": 3.756444155439584, + "tokens_seen": 802553856 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038222668004012035, + "loss": 1.4104, + "theoretical_loss": 3.7564139967122494, + "tokens_seen": 802619392 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003822166499498496, + "loss": 1.326, + "theoretical_loss": 3.7563838411368016, + "tokens_seen": 802684928 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003822066198595787, + "loss": 1.3391, + "theoretical_loss": 3.756353688712654, + "tokens_seen": 802750464 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 271826, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2298088073730469, + "objective/train/theoretical_loss": 3.7563235394392187, + "objective/train/tokens_used": 802816000, + "theoretical_loss": 3.7563235394392187, + "tokens_seen": 802816000 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038219658976930795, + "loss": 1.2671, + "theoretical_loss": 3.7563235394392187, + "tokens_seen": 802816000 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003821865596790371, + "loss": 1.4074, + "theoretical_loss": 3.756293393315911, + "tokens_seen": 802881536 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003821765295887663, + "loss": 1.4339, + "theoretical_loss": 3.7562632503421436, + "tokens_seen": 802947072 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003821664994984955, + "loss": 1.3823, + "theoretical_loss": 3.7562331105173308, + "tokens_seen": 803012608 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038215646940822467, + "loss": 1.3655, + "theoretical_loss": 3.756202973840886, + "tokens_seen": 803078144 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038214643931795385, + "loss": 1.3339, + "theoretical_loss": 3.7561728403122254, + "tokens_seen": 803143680 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038213640922768303, + "loss": 1.4635, + "theoretical_loss": 3.7561427099307614, + "tokens_seen": 803209216 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038212637913741227, + "loss": 1.3966, + "theoretical_loss": 3.756112582695909, + "tokens_seen": 803274752 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038211634904714145, + "loss": 1.3137, + "theoretical_loss": 3.7560824586070836, + "tokens_seen": 803340288 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038210631895687063, + "loss": 1.381, + "theoretical_loss": 3.756052337663699, + "tokens_seen": 803405824 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003820962888665998, + "loss": 1.4082, + "theoretical_loss": 3.756022219865171, + "tokens_seen": 803471360 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038208625877632905, + "loss": 1.4405, + "theoretical_loss": 3.7559921052109155, + "tokens_seen": 803536896 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003820762286860582, + "loss": 1.3404, + "theoretical_loss": 3.7559619937003457, + "tokens_seen": 803602432 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003820661985957874, + "loss": 1.3239, + "theoretical_loss": 3.755931885332879, + "tokens_seen": 803667968 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038205616850551654, + "loss": 1.4311, + "theoretical_loss": 3.75590178010793, + "tokens_seen": 803733504 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038204613841524577, + "loss": 1.4329, + "theoretical_loss": 3.7558716780249153, + "tokens_seen": 803799040 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038203610832497495, + "loss": 1.3588, + "theoretical_loss": 3.75584157908325, + "tokens_seen": 803864576 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038202607823470413, + "loss": 1.4393, + "theoretical_loss": 3.755811483282351, + "tokens_seen": 803930112 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003820160481444333, + "loss": 1.2391, + "theoretical_loss": 3.7557813906216344, + "tokens_seen": 803995648 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003820060180541625, + "loss": 1.4347, + "theoretical_loss": 3.755751301100516, + "tokens_seen": 804061184 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003819959879638917, + "loss": 1.4483, + "theoretical_loss": 3.755721214718413, + "tokens_seen": 804126720 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003819859578736209, + "loss": 1.3322, + "theoretical_loss": 3.755691131474742, + "tokens_seen": 804192256 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038197592778335004, + "loss": 1.456, + "theoretical_loss": 3.7556610513689197, + "tokens_seen": 804257792 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003819658976930793, + "loss": 1.4155, + "theoretical_loss": 3.755630974400364, + "tokens_seen": 804323328 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003819558676028084, + "loss": 1.4429, + "theoretical_loss": 3.7556009005684907, + "tokens_seen": 804388864 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038194583751253764, + "loss": 1.4093, + "theoretical_loss": 3.7555708298727186, + "tokens_seen": 804454400 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003819358074222668, + "loss": 1.256, + "theoretical_loss": 3.7555407623124637, + "tokens_seen": 804519936 + }, + { + "epoch": 0.24, + "learning_rate": 0.000381925777331996, + "loss": 1.3353, + "theoretical_loss": 3.755510697887145, + "tokens_seen": 804585472 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003819157472417252, + "loss": 1.3792, + "theoretical_loss": 3.75548063659618, + "tokens_seen": 804651008 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003819057171514544, + "loss": 1.4711, + "theoretical_loss": 3.755450578438986, + "tokens_seen": 804716544 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038189568706118354, + "loss": 1.3698, + "theoretical_loss": 3.7554205234149824, + "tokens_seen": 804782080 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003818856569709128, + "loss": 1.4436, + "theoretical_loss": 3.755390471523586, + "tokens_seen": 804847616 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003818756268806419, + "loss": 1.2479, + "theoretical_loss": 3.755360422764217, + "tokens_seen": 804913152 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038186559679037114, + "loss": 1.417, + "theoretical_loss": 3.755330377136292, + "tokens_seen": 804978688 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003818555667001003, + "loss": 1.4199, + "theoretical_loss": 3.7553003346392315, + "tokens_seen": 805044224 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003818455366098295, + "loss": 1.3726, + "theoretical_loss": 3.7552702952724535, + "tokens_seen": 805109760 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003818355065195587, + "loss": 1.3288, + "theoretical_loss": 3.755240259035377, + "tokens_seen": 805175296 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038182547642928786, + "loss": 1.2688, + "theoretical_loss": 3.755210225927422, + "tokens_seen": 805240832 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038181544633901705, + "loss": 1.2689, + "theoretical_loss": 3.7551801959480073, + "tokens_seen": 805306368 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003818054162487463, + "loss": 1.3754, + "theoretical_loss": 3.7551501690965523, + "tokens_seen": 805371904 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003817953861584754, + "loss": 1.3841, + "theoretical_loss": 3.755120145372477, + "tokens_seen": 805437440 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038178535606820464, + "loss": 1.3748, + "theoretical_loss": 3.7550901247752013, + "tokens_seen": 805502976 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038177532597793377, + "loss": 1.2466, + "theoretical_loss": 3.755060107304145, + "tokens_seen": 805568512 + }, + { + "epoch": 0.24, + "learning_rate": 0.000381765295887663, + "loss": 1.358, + "theoretical_loss": 3.7550300929587284, + "tokens_seen": 805634048 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003817552657973922, + "loss": 1.3338, + "theoretical_loss": 3.7550000817383715, + "tokens_seen": 805699584 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038174523570712137, + "loss": 1.2535, + "theoretical_loss": 3.754970073642496, + "tokens_seen": 805765120 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038173520561685055, + "loss": 1.4096, + "theoretical_loss": 3.7549400686705203, + "tokens_seen": 805830656 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003817251755265798, + "loss": 1.375, + "theoretical_loss": 3.754910066821867, + "tokens_seen": 805896192 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003817151454363089, + "loss": 1.4329, + "theoretical_loss": 3.7548800680959564, + "tokens_seen": 805961728 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038170511534603815, + "loss": 1.4633, + "theoretical_loss": 3.75485007249221, + "tokens_seen": 806027264 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 272702, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.486374020576477, + "objective/train/theoretical_loss": 3.754820080010048, + "objective/train/tokens_used": 806092800, + "theoretical_loss": 3.754820080010048, + "tokens_seen": 806092800 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003816950852557673, + "loss": 1.4214, + "theoretical_loss": 3.754820080010048, + "tokens_seen": 806092800 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003816850551654965, + "loss": 1.3554, + "theoretical_loss": 3.7547900906488927, + "tokens_seen": 806158336 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003816750250752257, + "loss": 1.3145, + "theoretical_loss": 3.7547601044081658, + "tokens_seen": 806223872 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038166499498495487, + "loss": 1.1976, + "theoretical_loss": 3.7547301212872886, + "tokens_seen": 806289408 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038165496489468405, + "loss": 1.3716, + "theoretical_loss": 3.7547001412856824, + "tokens_seen": 806354944 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038164493480441323, + "loss": 1.3935, + "theoretical_loss": 3.7546701644027705, + "tokens_seen": 806420480 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003816349047141424, + "loss": 1.3223, + "theoretical_loss": 3.754640190637974, + "tokens_seen": 806486016 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038162487462387165, + "loss": 1.4058, + "theoretical_loss": 3.754610219990716, + "tokens_seen": 806551552 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003816148445336008, + "loss": 1.2976, + "theoretical_loss": 3.7545802524604177, + "tokens_seen": 806617088 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038160481444333, + "loss": 1.404, + "theoretical_loss": 3.7545502880465036, + "tokens_seen": 806682624 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038159478435305914, + "loss": 1.406, + "theoretical_loss": 3.754520326748395, + "tokens_seen": 806748160 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003815847542627884, + "loss": 1.3819, + "theoretical_loss": 3.7544903685655155, + "tokens_seen": 806813696 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038157472417251756, + "loss": 1.2834, + "theoretical_loss": 3.754460413497288, + "tokens_seen": 806879232 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038156469408224674, + "loss": 1.3251, + "theoretical_loss": 3.7544304615431354, + "tokens_seen": 806944768 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003815546639919759, + "loss": 1.3537, + "theoretical_loss": 3.7544005127024818, + "tokens_seen": 807010304 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038154463390170515, + "loss": 1.348, + "theoretical_loss": 3.7543705669747496, + "tokens_seen": 807075840 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003815346038114343, + "loss": 1.4396, + "theoretical_loss": 3.754340624359364, + "tokens_seen": 807141376 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003815245737211635, + "loss": 1.444, + "theoretical_loss": 3.7543106848557484, + "tokens_seen": 807206912 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038151454363089264, + "loss": 1.4749, + "theoretical_loss": 3.754280748463326, + "tokens_seen": 807272448 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003815045135406219, + "loss": 1.3016, + "theoretical_loss": 3.7542508151815213, + "tokens_seen": 807337984 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038149448345035106, + "loss": 1.2479, + "theoretical_loss": 3.75422088500976, + "tokens_seen": 807403520 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038148445336008024, + "loss": 1.3808, + "theoretical_loss": 3.7541909579474644, + "tokens_seen": 807469056 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003814744232698094, + "loss": 1.3142, + "theoretical_loss": 3.7541610339940603, + "tokens_seen": 807534592 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003814643931795386, + "loss": 1.3063, + "theoretical_loss": 3.754131113148972, + "tokens_seen": 807600128 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003814543630892678, + "loss": 1.3045, + "theoretical_loss": 3.7541011954116255, + "tokens_seen": 807665664 + }, + { + "epoch": 0.24, + "learning_rate": 0.000381444332998997, + "loss": 1.27, + "theoretical_loss": 3.754071280781445, + "tokens_seen": 807731200 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038143430290872614, + "loss": 1.3778, + "theoretical_loss": 3.7540413692578554, + "tokens_seen": 807796736 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003814242728184554, + "loss": 1.3837, + "theoretical_loss": 3.7540114608402826, + "tokens_seen": 807862272 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038141424272818456, + "loss": 1.4217, + "theoretical_loss": 3.753981555528152, + "tokens_seen": 807927808 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038140421263791374, + "loss": 1.4109, + "theoretical_loss": 3.7539516533208896, + "tokens_seen": 807993344 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003813941825476429, + "loss": 1.4193, + "theoretical_loss": 3.753921754217921, + "tokens_seen": 808058880 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003813841524573721, + "loss": 1.4355, + "theoretical_loss": 3.7538918582186724, + "tokens_seen": 808124416 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038137412236710134, + "loss": 1.3191, + "theoretical_loss": 3.75386196532257, + "tokens_seen": 808189952 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003813640922768305, + "loss": 1.3772, + "theoretical_loss": 3.753832075529039, + "tokens_seen": 808255488 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003813540621865597, + "loss": 1.2545, + "theoretical_loss": 3.7538021888375077, + "tokens_seen": 808321024 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003813440320962889, + "loss": 1.3212, + "theoretical_loss": 3.7537723052474012, + "tokens_seen": 808386560 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038133400200601806, + "loss": 1.2267, + "theoretical_loss": 3.753742424758147, + "tokens_seen": 808452096 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038132397191574725, + "loss": 1.372, + "theoretical_loss": 3.7537125473691715, + "tokens_seen": 808517632 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003813139418254765, + "loss": 1.3612, + "theoretical_loss": 3.7536826730799024, + "tokens_seen": 808583168 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003813039117352056, + "loss": 1.4281, + "theoretical_loss": 3.753652801889767, + "tokens_seen": 808648704 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038129388164493484, + "loss": 1.4132, + "theoretical_loss": 3.7536229337981912, + "tokens_seen": 808714240 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038128385155466397, + "loss": 1.3848, + "theoretical_loss": 3.753593068804604, + "tokens_seen": 808779776 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003812738214643932, + "loss": 1.3338, + "theoretical_loss": 3.7535632069084333, + "tokens_seen": 808845312 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003812637913741224, + "loss": 1.2504, + "theoretical_loss": 3.753533348109106, + "tokens_seen": 808910848 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038125376128385157, + "loss": 1.382, + "theoretical_loss": 3.7535034924060504, + "tokens_seen": 808976384 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038124373119358075, + "loss": 1.3413, + "theoretical_loss": 3.7534736397986945, + "tokens_seen": 809041920 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038123370110331, + "loss": 1.3585, + "theoretical_loss": 3.753443790286467, + "tokens_seen": 809107456 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003812236710130391, + "loss": 1.3441, + "theoretical_loss": 3.7534139438687957, + "tokens_seen": 809172992 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038121364092276835, + "loss": 1.2377, + "theoretical_loss": 3.7533841005451096, + "tokens_seen": 809238528 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003812036108324975, + "loss": 1.3394, + "theoretical_loss": 3.7533542603148375, + "tokens_seen": 809304064 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 273917, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.302206039428711, + "objective/train/theoretical_loss": 3.7533244231774083, + "objective/train/tokens_used": 809369600, + "theoretical_loss": 3.7533244231774083, + "tokens_seen": 809369600 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003811935807422267, + "loss": 1.3797, + "theoretical_loss": 3.7533244231774083, + "tokens_seen": 809369600 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003811835506519559, + "loss": 1.2286, + "theoretical_loss": 3.753294589132251, + "tokens_seen": 809435136 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038117352056168507, + "loss": 1.3138, + "theoretical_loss": 3.753264758178794, + "tokens_seen": 809500672 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038116349047141425, + "loss": 1.4168, + "theoretical_loss": 3.7532349303164683, + "tokens_seen": 809566208 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038115346038114343, + "loss": 1.3114, + "theoretical_loss": 3.7532051055447018, + "tokens_seen": 809631744 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003811434302908726, + "loss": 1.2512, + "theoretical_loss": 3.753175283862925, + "tokens_seen": 809697280 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038113340020060185, + "loss": 1.3507, + "theoretical_loss": 3.753145465270568, + "tokens_seen": 809762816 + }, + { + "epoch": 0.25, + "learning_rate": 0.000381123370110331, + "loss": 1.4238, + "theoretical_loss": 3.753115649767059, + "tokens_seen": 809828352 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003811133400200602, + "loss": 1.481, + "theoretical_loss": 3.75308583735183, + "tokens_seen": 809893888 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038110330992978934, + "loss": 1.4428, + "theoretical_loss": 3.753056028024311, + "tokens_seen": 809959424 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003810932798395186, + "loss": 1.3954, + "theoretical_loss": 3.753026221783932, + "tokens_seen": 810024960 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038108324974924776, + "loss": 1.3799, + "theoretical_loss": 3.752996418630123, + "tokens_seen": 810090496 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038107321965897694, + "loss": 1.3011, + "theoretical_loss": 3.7529666185623154, + "tokens_seen": 810156032 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003810631895687061, + "loss": 1.3418, + "theoretical_loss": 3.75293682157994, + "tokens_seen": 810221568 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038105315947843535, + "loss": 1.4447, + "theoretical_loss": 3.7529070276824283, + "tokens_seen": 810287104 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003810431293881645, + "loss": 1.3561, + "theoretical_loss": 3.7528772368692103, + "tokens_seen": 810352640 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003810330992978937, + "loss": 1.3738, + "theoretical_loss": 3.7528474491397183, + "tokens_seen": 810418176 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038102306920762284, + "loss": 1.4345, + "theoretical_loss": 3.7528176644933833, + "tokens_seen": 810483712 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003810130391173521, + "loss": 1.2456, + "theoretical_loss": 3.7527878829296366, + "tokens_seen": 810549248 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038100300902708126, + "loss": 1.3874, + "theoretical_loss": 3.752758104447911, + "tokens_seen": 810614784 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038099297893681044, + "loss": 1.2978, + "theoretical_loss": 3.7527283290476374, + "tokens_seen": 810680320 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003809829488465396, + "loss": 1.3117, + "theoretical_loss": 3.752698556728248, + "tokens_seen": 810745856 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003809729187562688, + "loss": 1.2631, + "theoretical_loss": 3.7526687874891755, + "tokens_seen": 810811392 + }, + { + "epoch": 0.25, + "learning_rate": 0.000380962888665998, + "loss": 1.4318, + "theoretical_loss": 3.752639021329852, + "tokens_seen": 810876928 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003809528585757272, + "loss": 1.3354, + "theoretical_loss": 3.75260925824971, + "tokens_seen": 810942464 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038094282848545635, + "loss": 1.4063, + "theoretical_loss": 3.752579498248183, + "tokens_seen": 811008000 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003809327983951856, + "loss": 1.3832, + "theoretical_loss": 3.752549741324702, + "tokens_seen": 811073536 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038092276830491476, + "loss": 1.4467, + "theoretical_loss": 3.752519987478702, + "tokens_seen": 811139072 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038091273821464394, + "loss": 1.3604, + "theoretical_loss": 3.7524902367096145, + "tokens_seen": 811204608 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003809027081243731, + "loss": 1.33, + "theoretical_loss": 3.7524604890168733, + "tokens_seen": 811270144 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003808926780341023, + "loss": 1.3555, + "theoretical_loss": 3.752430744399912, + "tokens_seen": 811335680 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003808826479438315, + "loss": 1.3036, + "theoretical_loss": 3.7524010028581634, + "tokens_seen": 811401216 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003808726178535607, + "loss": 1.3094, + "theoretical_loss": 3.752371264391063, + "tokens_seen": 811466752 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038086258776328985, + "loss": 1.3667, + "theoretical_loss": 3.752341528998043, + "tokens_seen": 811532288 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003808525576730191, + "loss": 1.3089, + "theoretical_loss": 3.7523117966785384, + "tokens_seen": 811597824 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003808425275827482, + "loss": 1.4471, + "theoretical_loss": 3.752282067431983, + "tokens_seen": 811663360 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038083249749247745, + "loss": 1.389, + "theoretical_loss": 3.7522523412578104, + "tokens_seen": 811728896 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003808224674022066, + "loss": 1.4455, + "theoretical_loss": 3.7522226181554563, + "tokens_seen": 811794432 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003808124373119358, + "loss": 1.3514, + "theoretical_loss": 3.7521928981243544, + "tokens_seen": 811859968 + }, + { + "epoch": 0.25, + "learning_rate": 0.000380802407221665, + "loss": 1.3593, + "theoretical_loss": 3.7521631811639398, + "tokens_seen": 811925504 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038079237713139417, + "loss": 1.3543, + "theoretical_loss": 3.752133467273648, + "tokens_seen": 811991040 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038078234704112335, + "loss": 1.3058, + "theoretical_loss": 3.7521037564529127, + "tokens_seen": 812056576 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003807723169508526, + "loss": 1.4211, + "theoretical_loss": 3.7520740487011706, + "tokens_seen": 812122112 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003807622868605817, + "loss": 1.3354, + "theoretical_loss": 3.752044344017856, + "tokens_seen": 812187648 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038075225677031095, + "loss": 1.3865, + "theoretical_loss": 3.752014642402405, + "tokens_seen": 812253184 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038074222668004013, + "loss": 1.2514, + "theoretical_loss": 3.751984943854253, + "tokens_seen": 812318720 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003807321965897693, + "loss": 1.3682, + "theoretical_loss": 3.751955248372836, + "tokens_seen": 812384256 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003807221664994985, + "loss": 1.4409, + "theoretical_loss": 3.75192555595759, + "tokens_seen": 812449792 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003807121364092277, + "loss": 1.3059, + "theoretical_loss": 3.751895866607951, + "tokens_seen": 812515328 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038070210631895685, + "loss": 1.4137, + "theoretical_loss": 3.7518661803233546, + "tokens_seen": 812580864 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 275091, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3865528106689453, + "objective/train/theoretical_loss": 3.7518364971032385, + "objective/train/tokens_used": 812646400, + "theoretical_loss": 3.7518364971032385, + "tokens_seen": 812646400 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003806920762286861, + "loss": 1.4747, + "theoretical_loss": 3.7518364971032385, + "tokens_seen": 812646400 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003806820461384152, + "loss": 1.2552, + "theoretical_loss": 3.751806816947038, + "tokens_seen": 812711936 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038067201604814445, + "loss": 1.4733, + "theoretical_loss": 3.751777139854191, + "tokens_seen": 812777472 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003806619859578736, + "loss": 1.4559, + "theoretical_loss": 3.7517474658241334, + "tokens_seen": 812843008 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003806519558676028, + "loss": 1.42, + "theoretical_loss": 3.7517177948563027, + "tokens_seen": 812908544 + }, + { + "epoch": 0.25, + "learning_rate": 0.000380641925777332, + "loss": 1.3355, + "theoretical_loss": 3.7516881269501354, + "tokens_seen": 812974080 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003806318956870612, + "loss": 1.3949, + "theoretical_loss": 3.75165846210507, + "tokens_seen": 813039616 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003806218655967904, + "loss": 1.4954, + "theoretical_loss": 3.7516288003205434, + "tokens_seen": 813105152 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038061183550651954, + "loss": 1.3274, + "theoretical_loss": 3.7515991415959933, + "tokens_seen": 813170688 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003806018054162488, + "loss": 1.431, + "theoretical_loss": 3.7515694859308564, + "tokens_seen": 813236224 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038059177532597796, + "loss": 1.3452, + "theoretical_loss": 3.7515398333245717, + "tokens_seen": 813301760 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038058174523570714, + "loss": 1.382, + "theoretical_loss": 3.751510183776577, + "tokens_seen": 813367296 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003805717151454363, + "loss": 1.3631, + "theoretical_loss": 3.7514805372863105, + "tokens_seen": 813432832 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038056168505516555, + "loss": 1.3725, + "theoretical_loss": 3.7514508938532107, + "tokens_seen": 813498368 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003805516549648947, + "loss": 1.3515, + "theoretical_loss": 3.7514212534767157, + "tokens_seen": 813563904 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003805416248746239, + "loss": 1.4056, + "theoretical_loss": 3.7513916161562646, + "tokens_seen": 813629440 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038053159478435304, + "loss": 1.401, + "theoretical_loss": 3.7513619818912964, + "tokens_seen": 813694976 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003805215646940823, + "loss": 1.3773, + "theoretical_loss": 3.751332350681249, + "tokens_seen": 813760512 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038051153460381146, + "loss": 1.3731, + "theoretical_loss": 3.751302722525562, + "tokens_seen": 813826048 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038050150451354064, + "loss": 1.4223, + "theoretical_loss": 3.7512730974236748, + "tokens_seen": 813891584 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003804914744232698, + "loss": 1.3697, + "theoretical_loss": 3.751243475375027, + "tokens_seen": 813957120 + }, + { + "epoch": 0.25, + "learning_rate": 0.000380481444332999, + "loss": 1.3456, + "theoretical_loss": 3.7512138563790574, + "tokens_seen": 814022656 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003804714142427282, + "loss": 1.3042, + "theoretical_loss": 3.7511842404352063, + "tokens_seen": 814088192 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003804613841524574, + "loss": 1.3737, + "theoretical_loss": 3.7511546275429133, + "tokens_seen": 814153728 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038045135406218655, + "loss": 1.3907, + "theoretical_loss": 3.7511250177016184, + "tokens_seen": 814219264 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003804413239719158, + "loss": 1.3826, + "theoretical_loss": 3.7510954109107617, + "tokens_seen": 814284800 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038043129388164496, + "loss": 1.3359, + "theoretical_loss": 3.751065807169784, + "tokens_seen": 814350336 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038042126379137414, + "loss": 1.4075, + "theoretical_loss": 3.7510362064781244, + "tokens_seen": 814415872 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003804112337011033, + "loss": 1.3676, + "theoretical_loss": 3.7510066088352247, + "tokens_seen": 814481408 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003804012036108325, + "loss": 1.4007, + "theoretical_loss": 3.750977014240525, + "tokens_seen": 814546944 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003803911735205617, + "loss": 1.3148, + "theoretical_loss": 3.750947422693467, + "tokens_seen": 814612480 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003803811434302909, + "loss": 1.2975, + "theoretical_loss": 3.7509178341934906, + "tokens_seen": 814678016 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038037111334002005, + "loss": 1.2358, + "theoretical_loss": 3.7508882487400372, + "tokens_seen": 814743552 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003803610832497493, + "loss": 1.3856, + "theoretical_loss": 3.7508586663325483, + "tokens_seen": 814809088 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003803510531594784, + "loss": 1.3118, + "theoretical_loss": 3.750829086970466, + "tokens_seen": 814874624 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038034102306920765, + "loss": 1.4366, + "theoretical_loss": 3.750799510653231, + "tokens_seen": 814940160 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038033099297893683, + "loss": 1.2905, + "theoretical_loss": 3.7507699373802854, + "tokens_seen": 815005696 + }, + { + "epoch": 0.25, + "learning_rate": 0.000380320962888666, + "loss": 1.4446, + "theoretical_loss": 3.750740367151071, + "tokens_seen": 815071232 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003803109327983952, + "loss": 1.4262, + "theoretical_loss": 3.750710799965029, + "tokens_seen": 815136768 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038030090270812437, + "loss": 1.3686, + "theoretical_loss": 3.7506812358216037, + "tokens_seen": 815202304 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038029087261785355, + "loss": 1.364, + "theoretical_loss": 3.7506516747202356, + "tokens_seen": 815267840 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003802808425275828, + "loss": 1.3487, + "theoretical_loss": 3.7506221166603675, + "tokens_seen": 815333376 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003802708124373119, + "loss": 1.3965, + "theoretical_loss": 3.750592561641443, + "tokens_seen": 815398912 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038026078234704115, + "loss": 1.3208, + "theoretical_loss": 3.750563009662904, + "tokens_seen": 815464448 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038025075225677033, + "loss": 1.3896, + "theoretical_loss": 3.750533460724193, + "tokens_seen": 815529984 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003802407221664995, + "loss": 1.3915, + "theoretical_loss": 3.750503914824754, + "tokens_seen": 815595520 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003802306920762287, + "loss": 1.4046, + "theoretical_loss": 3.75047437196403, + "tokens_seen": 815661056 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003802206619859579, + "loss": 1.2971, + "theoretical_loss": 3.750444832141464, + "tokens_seen": 815726592 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038021063189568705, + "loss": 1.4224, + "theoretical_loss": 3.7504152953564995, + "tokens_seen": 815792128 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003802006018054163, + "loss": 1.3273, + "theoretical_loss": 3.750385761608581, + "tokens_seen": 815857664 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 276065, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4268724918365479, + "objective/train/theoretical_loss": 3.7503562308971516, + "objective/train/tokens_used": 815923200, + "theoretical_loss": 3.7503562308971516, + "tokens_seen": 815923200 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003801905717151454, + "loss": 1.3892, + "theoretical_loss": 3.7503562308971516, + "tokens_seen": 815923200 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038018054162487465, + "loss": 1.4688, + "theoretical_loss": 3.7503267032216554, + "tokens_seen": 815988736 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003801705115346038, + "loss": 1.3814, + "theoretical_loss": 3.750297178581536, + "tokens_seen": 816054272 + }, + { + "epoch": 0.25, + "learning_rate": 0.000380160481444333, + "loss": 1.4108, + "theoretical_loss": 3.7502676569762383, + "tokens_seen": 816119808 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003801504513540622, + "loss": 1.3683, + "theoretical_loss": 3.7502381384052064, + "tokens_seen": 816185344 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003801404212637914, + "loss": 1.3371, + "theoretical_loss": 3.750208622867885, + "tokens_seen": 816250880 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038013039117352056, + "loss": 1.441, + "theoretical_loss": 3.7501791103637188, + "tokens_seen": 816316416 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038012036108324974, + "loss": 1.3899, + "theoretical_loss": 3.7501496008921524, + "tokens_seen": 816381952 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003801103309929789, + "loss": 1.3461, + "theoretical_loss": 3.750120094452631, + "tokens_seen": 816447488 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038010030090270816, + "loss": 1.4579, + "theoretical_loss": 3.750090591044599, + "tokens_seen": 816513024 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003800902708124373, + "loss": 1.4388, + "theoretical_loss": 3.750061090667503, + "tokens_seen": 816578560 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003800802407221665, + "loss": 1.4373, + "theoretical_loss": 3.750031593320787, + "tokens_seen": 816644096 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003800702106318957, + "loss": 1.3351, + "theoretical_loss": 3.750002099003898, + "tokens_seen": 816709632 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003800601805416249, + "loss": 1.4184, + "theoretical_loss": 3.7499726077162805, + "tokens_seen": 816775168 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038005015045135406, + "loss": 1.4141, + "theoretical_loss": 3.749943119457381, + "tokens_seen": 816840704 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038004012036108324, + "loss": 1.4104, + "theoretical_loss": 3.749913634226645, + "tokens_seen": 816906240 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003800300902708124, + "loss": 1.4292, + "theoretical_loss": 3.7498841520235193, + "tokens_seen": 816971776 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038002006018054166, + "loss": 1.3648, + "theoretical_loss": 3.7498546728474498, + "tokens_seen": 817037312 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003800100300902708, + "loss": 1.4308, + "theoretical_loss": 3.7498251966978824, + "tokens_seen": 817102848 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038, + "loss": 1.4175, + "theoretical_loss": 3.7497957235742647, + "tokens_seen": 817168384 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037998996990972915, + "loss": 1.3047, + "theoretical_loss": 3.749766253476043, + "tokens_seen": 817233920 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003799799398194584, + "loss": 1.3071, + "theoretical_loss": 3.7497367864026634, + "tokens_seen": 817299456 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037996990972918756, + "loss": 1.4167, + "theoretical_loss": 3.749707322353575, + "tokens_seen": 817364992 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037995987963891675, + "loss": 1.3579, + "theoretical_loss": 3.7496778613282222, + "tokens_seen": 817430528 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003799498495486459, + "loss": 1.3507, + "theoretical_loss": 3.749648403326054, + "tokens_seen": 817496064 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037993981945837516, + "loss": 1.4676, + "theoretical_loss": 3.7496189483465177, + "tokens_seen": 817561600 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003799297893681043, + "loss": 1.2732, + "theoretical_loss": 3.749589496389061, + "tokens_seen": 817627136 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003799197592778335, + "loss": 1.3472, + "theoretical_loss": 3.749560047453131, + "tokens_seen": 817692672 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037990972918756265, + "loss": 1.4911, + "theoretical_loss": 3.7495306015381766, + "tokens_seen": 817758208 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003798996990972919, + "loss": 1.4094, + "theoretical_loss": 3.7495011586436444, + "tokens_seen": 817823744 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037988966900702107, + "loss": 1.3193, + "theoretical_loss": 3.7494717187689837, + "tokens_seen": 817889280 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037987963891675025, + "loss": 1.4365, + "theoretical_loss": 3.7494422819136424, + "tokens_seen": 817954816 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003798696088264795, + "loss": 1.4143, + "theoretical_loss": 3.7494128480770694, + "tokens_seen": 818020352 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003798595787362086, + "loss": 1.3685, + "theoretical_loss": 3.749383417258713, + "tokens_seen": 818085888 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037984954864593785, + "loss": 1.4502, + "theoretical_loss": 3.7493539894580215, + "tokens_seen": 818151424 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037983951855566703, + "loss": 1.357, + "theoretical_loss": 3.749324564674444, + "tokens_seen": 818216960 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003798294884653962, + "loss": 1.282, + "theoretical_loss": 3.74929514290743, + "tokens_seen": 818282496 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003798194583751254, + "loss": 1.2657, + "theoretical_loss": 3.749265724156429, + "tokens_seen": 818348032 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037980942828485457, + "loss": 1.4077, + "theoretical_loss": 3.7492363084208895, + "tokens_seen": 818413568 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037979939819458375, + "loss": 1.3963, + "theoretical_loss": 3.7492068957002607, + "tokens_seen": 818479104 + }, + { + "epoch": 0.25, + "learning_rate": 0.000379789368104313, + "loss": 1.4127, + "theoretical_loss": 3.749177485993993, + "tokens_seen": 818544640 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003797793380140421, + "loss": 1.3584, + "theoretical_loss": 3.749148079301537, + "tokens_seen": 818610176 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037976930792377135, + "loss": 1.4001, + "theoretical_loss": 3.7491186756223405, + "tokens_seen": 818675712 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037975927783350053, + "loss": 1.4324, + "theoretical_loss": 3.749089274955855, + "tokens_seen": 818741248 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003797492477432297, + "loss": 1.4068, + "theoretical_loss": 3.74905987730153, + "tokens_seen": 818806784 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003797392176529589, + "loss": 1.383, + "theoretical_loss": 3.7490304826588163, + "tokens_seen": 818872320 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003797291875626881, + "loss": 1.4167, + "theoretical_loss": 3.7490010910271647, + "tokens_seen": 818937856 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037971915747241725, + "loss": 1.4405, + "theoretical_loss": 3.748971702406025, + "tokens_seen": 819003392 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003797091273821465, + "loss": 1.3431, + "theoretical_loss": 3.7489423167948486, + "tokens_seen": 819068928 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003796990972918756, + "loss": 1.3898, + "theoretical_loss": 3.7489129341930862, + "tokens_seen": 819134464 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 277366, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3635298013687134, + "objective/train/theoretical_loss": 3.7488835546001886, + "objective/train/tokens_used": 819200000, + "theoretical_loss": 3.7488835546001886, + "tokens_seen": 819200000 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037968906720160485, + "loss": 1.4112, + "theoretical_loss": 3.7488835546001886, + "tokens_seen": 819200000 + }, + { + "epoch": 0.25, + "learning_rate": 0.000379679037111334, + "loss": 1.3727, + "theoretical_loss": 3.7488541780156077, + "tokens_seen": 819265536 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003796690070210632, + "loss": 1.4286, + "theoretical_loss": 3.7488248044387946, + "tokens_seen": 819331072 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003796589769307924, + "loss": 1.4442, + "theoretical_loss": 3.7487954338692, + "tokens_seen": 819396608 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003796489468405216, + "loss": 1.2139, + "theoretical_loss": 3.7487660663062767, + "tokens_seen": 819462144 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037963891675025076, + "loss": 1.4538, + "theoretical_loss": 3.7487367017494755, + "tokens_seen": 819527680 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037962888665997994, + "loss": 1.4403, + "theoretical_loss": 3.7487073401982487, + "tokens_seen": 819593216 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003796188565697091, + "loss": 1.3789, + "theoretical_loss": 3.7486779816520492, + "tokens_seen": 819658752 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037960882647943836, + "loss": 1.4173, + "theoretical_loss": 3.7486486261103282, + "tokens_seen": 819724288 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003795987963891675, + "loss": 1.3574, + "theoretical_loss": 3.748619273572538, + "tokens_seen": 819789824 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003795887662988967, + "loss": 1.4203, + "theoretical_loss": 3.748589924038132, + "tokens_seen": 819855360 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003795787362086259, + "loss": 1.3874, + "theoretical_loss": 3.748560577506562, + "tokens_seen": 819920896 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003795687061183551, + "loss": 1.2706, + "theoretical_loss": 3.748531233977281, + "tokens_seen": 819986432 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037955867602808426, + "loss": 1.439, + "theoretical_loss": 3.7485018934497423, + "tokens_seen": 820051968 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037954864593781344, + "loss": 1.3518, + "theoretical_loss": 3.7484725559233985, + "tokens_seen": 820117504 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003795386158475426, + "loss": 1.4234, + "theoretical_loss": 3.7484432213977033, + "tokens_seen": 820183040 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037952858575727186, + "loss": 1.3764, + "theoretical_loss": 3.7484138898721096, + "tokens_seen": 820248576 + }, + { + "epoch": 0.25, + "learning_rate": 0.000379518555667001, + "loss": 1.3409, + "theoretical_loss": 3.748384561346071, + "tokens_seen": 820314112 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003795085255767302, + "loss": 1.2951, + "theoretical_loss": 3.7483552358190417, + "tokens_seen": 820379648 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037949849548645935, + "loss": 1.4304, + "theoretical_loss": 3.7483259132904747, + "tokens_seen": 820445184 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003794884653961886, + "loss": 1.3615, + "theoretical_loss": 3.7482965937598243, + "tokens_seen": 820510720 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037947843530591776, + "loss": 1.363, + "theoretical_loss": 3.748267277226545, + "tokens_seen": 820576256 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037946840521564695, + "loss": 1.4582, + "theoretical_loss": 3.74823796369009, + "tokens_seen": 820641792 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003794583751253761, + "loss": 1.3916, + "theoretical_loss": 3.7482086531499146, + "tokens_seen": 820707328 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037944834503510536, + "loss": 1.45, + "theoretical_loss": 3.748179345605473, + "tokens_seen": 820772864 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003794383149448345, + "loss": 1.3078, + "theoretical_loss": 3.74815004105622, + "tokens_seen": 820838400 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003794282848545637, + "loss": 1.4222, + "theoretical_loss": 3.7481207395016103, + "tokens_seen": 820903936 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037941825476429285, + "loss": 1.4211, + "theoretical_loss": 3.7480914409410984, + "tokens_seen": 820969472 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003794082246740221, + "loss": 1.2764, + "theoretical_loss": 3.7480621453741394, + "tokens_seen": 821035008 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037939819458375127, + "loss": 1.4384, + "theoretical_loss": 3.74803285280019, + "tokens_seen": 821100544 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037938816449348045, + "loss": 1.4508, + "theoretical_loss": 3.7480035632187034, + "tokens_seen": 821166080 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037937813440320963, + "loss": 1.4239, + "theoretical_loss": 3.7479742766291366, + "tokens_seen": 821231616 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003793681043129388, + "loss": 1.4824, + "theoretical_loss": 3.747944993030945, + "tokens_seen": 821297152 + }, + { + "epoch": 0.25, + "learning_rate": 0.000379358074222668, + "loss": 1.3444, + "theoretical_loss": 3.7479157124235836, + "tokens_seen": 821362688 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037934804413239723, + "loss": 1.4122, + "theoretical_loss": 3.7478864348065093, + "tokens_seen": 821428224 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037933801404212635, + "loss": 1.4586, + "theoretical_loss": 3.747857160179178, + "tokens_seen": 821493760 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003793279839518556, + "loss": 1.3762, + "theoretical_loss": 3.747827888541045, + "tokens_seen": 821559296 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003793179538615847, + "loss": 1.3776, + "theoretical_loss": 3.7477986198915683, + "tokens_seen": 821624832 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037930792377131395, + "loss": 1.3853, + "theoretical_loss": 3.7477693542302033, + "tokens_seen": 821690368 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037929789368104313, + "loss": 1.3842, + "theoretical_loss": 3.7477400915564063, + "tokens_seen": 821755904 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003792878635907723, + "loss": 1.5044, + "theoretical_loss": 3.747710831869635, + "tokens_seen": 821821440 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003792778335005015, + "loss": 1.4916, + "theoretical_loss": 3.747681575169346, + "tokens_seen": 821886976 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037926780341023073, + "loss": 1.3599, + "theoretical_loss": 3.747652321454997, + "tokens_seen": 821952512 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037925777331995986, + "loss": 1.3666, + "theoretical_loss": 3.7476230707260436, + "tokens_seen": 822018048 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003792477432296891, + "loss": 1.3632, + "theoretical_loss": 3.7475938229819445, + "tokens_seen": 822083584 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003792377131394182, + "loss": 1.2431, + "theoretical_loss": 3.747564578222157, + "tokens_seen": 822149120 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037922768304914746, + "loss": 1.387, + "theoretical_loss": 3.747535336446138, + "tokens_seen": 822214656 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037921765295887664, + "loss": 1.3295, + "theoretical_loss": 3.7475060976533463, + "tokens_seen": 822280192 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003792076228686058, + "loss": 1.4008, + "theoretical_loss": 3.7474768618432392, + "tokens_seen": 822345728 + }, + { + "epoch": 0.25, + "learning_rate": 0.000379197592778335, + "loss": 1.4553, + "theoretical_loss": 3.747447629015275, + "tokens_seen": 822411264 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 278303, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3363862037658691, + "objective/train/theoretical_loss": 3.747418399168912, + "objective/train/tokens_used": 822476800, + "theoretical_loss": 3.747418399168912, + "tokens_seen": 822476800 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003791875626880642, + "loss": 1.4493, + "theoretical_loss": 3.747418399168912, + "tokens_seen": 822476800 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037917753259779336, + "loss": 1.3531, + "theoretical_loss": 3.747389172303608, + "tokens_seen": 822542336 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003791675025075226, + "loss": 1.326, + "theoretical_loss": 3.7473599484188225, + "tokens_seen": 822607872 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003791574724172517, + "loss": 1.3631, + "theoretical_loss": 3.7473307275140133, + "tokens_seen": 822673408 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037914744232698096, + "loss": 1.3286, + "theoretical_loss": 3.747301509588639, + "tokens_seen": 822738944 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003791374122367101, + "loss": 1.4162, + "theoretical_loss": 3.7472722946421593, + "tokens_seen": 822804480 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003791273821464393, + "loss": 1.3873, + "theoretical_loss": 3.747243082674033, + "tokens_seen": 822870016 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037911735205616856, + "loss": 1.4199, + "theoretical_loss": 3.7472138736837186, + "tokens_seen": 822935552 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003791073219658977, + "loss": 1.3534, + "theoretical_loss": 3.747184667670677, + "tokens_seen": 823001088 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003790972918756269, + "loss": 1.3305, + "theoretical_loss": 3.747155464634366, + "tokens_seen": 823066624 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003790872617853561, + "loss": 1.4447, + "theoretical_loss": 3.747126264574246, + "tokens_seen": 823132160 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003790772316950853, + "loss": 1.3661, + "theoretical_loss": 3.7470970674897766, + "tokens_seen": 823197696 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037906720160481446, + "loss": 1.3916, + "theoretical_loss": 3.7470678733804177, + "tokens_seen": 823263232 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037905717151454364, + "loss": 1.3556, + "theoretical_loss": 3.7470386822456296, + "tokens_seen": 823328768 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003790471414242728, + "loss": 1.3548, + "theoretical_loss": 3.7470094940848724, + "tokens_seen": 823394304 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037903711133400206, + "loss": 1.3521, + "theoretical_loss": 3.7469803088976064, + "tokens_seen": 823459840 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003790270812437312, + "loss": 1.4173, + "theoretical_loss": 3.7469511266832916, + "tokens_seen": 823525376 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003790170511534604, + "loss": 1.3653, + "theoretical_loss": 3.7469219474413893, + "tokens_seen": 823590912 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037900702106318955, + "loss": 1.35, + "theoretical_loss": 3.74689277117136, + "tokens_seen": 823656448 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003789969909729188, + "loss": 1.3675, + "theoretical_loss": 3.7468635978726645, + "tokens_seen": 823721984 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037898696088264796, + "loss": 1.4263, + "theoretical_loss": 3.746834427544764, + "tokens_seen": 823787520 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037897693079237715, + "loss": 1.4326, + "theoretical_loss": 3.7468052601871196, + "tokens_seen": 823853056 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003789669007021063, + "loss": 1.3771, + "theoretical_loss": 3.7467760957991922, + "tokens_seen": 823918592 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037895687061183556, + "loss": 1.4476, + "theoretical_loss": 3.746746934380444, + "tokens_seen": 823984128 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003789468405215647, + "loss": 1.3916, + "theoretical_loss": 3.7467177759303363, + "tokens_seen": 824049664 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003789368104312939, + "loss": 1.3875, + "theoretical_loss": 3.7466886204483307, + "tokens_seen": 824115200 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037892678034102305, + "loss": 1.2932, + "theoretical_loss": 3.746659467933889, + "tokens_seen": 824180736 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003789167502507523, + "loss": 1.4705, + "theoretical_loss": 3.7466303183864733, + "tokens_seen": 824246272 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037890672016048147, + "loss": 1.3674, + "theoretical_loss": 3.746601171805546, + "tokens_seen": 824311808 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037889669007021065, + "loss": 1.4299, + "theoretical_loss": 3.7465720281905694, + "tokens_seen": 824377344 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037888665997993983, + "loss": 1.3307, + "theoretical_loss": 3.746542887541005, + "tokens_seen": 824442880 + }, + { + "epoch": 0.25, + "learning_rate": 0.000378876629889669, + "loss": 1.3304, + "theoretical_loss": 3.746513749856317, + "tokens_seen": 824508416 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003788665997993982, + "loss": 1.3234, + "theoretical_loss": 3.7464846151359668, + "tokens_seen": 824573952 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037885656970912743, + "loss": 1.5006, + "theoretical_loss": 3.7464554833794175, + "tokens_seen": 824639488 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037884653961885655, + "loss": 1.3465, + "theoretical_loss": 3.7464263545861325, + "tokens_seen": 824705024 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003788365095285858, + "loss": 1.2899, + "theoretical_loss": 3.7463972287555745, + "tokens_seen": 824770560 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003788264794383149, + "loss": 1.3555, + "theoretical_loss": 3.7463681058872074, + "tokens_seen": 824836096 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037881644934804415, + "loss": 1.383, + "theoretical_loss": 3.746338985980494, + "tokens_seen": 824901632 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037880641925777333, + "loss": 1.3121, + "theoretical_loss": 3.746309869034898, + "tokens_seen": 824967168 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003787963891675025, + "loss": 1.4183, + "theoretical_loss": 3.746280755049883, + "tokens_seen": 825032704 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003787863590772317, + "loss": 1.4006, + "theoretical_loss": 3.7462516440249134, + "tokens_seen": 825098240 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037877632898696093, + "loss": 1.3895, + "theoretical_loss": 3.7462225359594523, + "tokens_seen": 825163776 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037876629889669006, + "loss": 1.3944, + "theoretical_loss": 3.7461934308529647, + "tokens_seen": 825229312 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003787562688064193, + "loss": 1.2974, + "theoretical_loss": 3.7461643287049142, + "tokens_seen": 825294848 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003787462387161484, + "loss": 1.4797, + "theoretical_loss": 3.7461352295147656, + "tokens_seen": 825360384 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037873620862587766, + "loss": 1.3116, + "theoretical_loss": 3.746106133281983, + "tokens_seen": 825425920 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037872617853560684, + "loss": 1.4383, + "theoretical_loss": 3.746077040006032, + "tokens_seen": 825491456 + }, + { + "epoch": 0.25, + "learning_rate": 0.000378716148445336, + "loss": 1.3696, + "theoretical_loss": 3.7460479496863757, + "tokens_seen": 825556992 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003787061183550652, + "loss": 1.3932, + "theoretical_loss": 3.746018862322481, + "tokens_seen": 825622528 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003786960882647944, + "loss": 1.3702, + "theoretical_loss": 3.7459897779138114, + "tokens_seen": 825688064 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 279544, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4786391258239746, + "objective/train/theoretical_loss": 3.745960696459833, + "objective/train/tokens_used": 825753600, + "theoretical_loss": 3.745960696459833, + "tokens_seen": 825753600 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037868605817452356, + "loss": 1.4496, + "theoretical_loss": 3.745960696459833, + "tokens_seen": 825753600 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003786760280842528, + "loss": 1.4251, + "theoretical_loss": 3.7459316179600117, + "tokens_seen": 825819136 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003786659979939819, + "loss": 1.312, + "theoretical_loss": 3.7459025424138117, + "tokens_seen": 825884672 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037865596790371116, + "loss": 1.376, + "theoretical_loss": 3.7458734698206992, + "tokens_seen": 825950208 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003786459378134403, + "loss": 1.337, + "theoretical_loss": 3.74584440018014, + "tokens_seen": 826015744 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003786359077231695, + "loss": 1.404, + "theoretical_loss": 3.7458153334916, + "tokens_seen": 826081280 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003786258776328987, + "loss": 1.4417, + "theoretical_loss": 3.7457862697545456, + "tokens_seen": 826146816 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003786158475426279, + "loss": 1.3205, + "theoretical_loss": 3.7457572089684428, + "tokens_seen": 826212352 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037860581745235706, + "loss": 1.3684, + "theoretical_loss": 3.7457281511327576, + "tokens_seen": 826277888 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003785957873620863, + "loss": 1.3208, + "theoretical_loss": 3.7456990962469567, + "tokens_seen": 826343424 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003785857572718154, + "loss": 1.4375, + "theoretical_loss": 3.745670044310507, + "tokens_seen": 826408960 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037857572718154466, + "loss": 1.3211, + "theoretical_loss": 3.7456409953228746, + "tokens_seen": 826474496 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003785656970912738, + "loss": 1.3323, + "theoretical_loss": 3.745611949283527, + "tokens_seen": 826540032 + }, + { + "epoch": 0.25, + "learning_rate": 0.000378555667001003, + "loss": 1.5277, + "theoretical_loss": 3.745582906191931, + "tokens_seen": 826605568 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003785456369107322, + "loss": 1.3055, + "theoretical_loss": 3.7455538660475542, + "tokens_seen": 826671104 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003785356068204614, + "loss": 1.4522, + "theoretical_loss": 3.7455248288498635, + "tokens_seen": 826736640 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037852557673019057, + "loss": 1.3473, + "theoretical_loss": 3.745495794598326, + "tokens_seen": 826802176 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037851554663991975, + "loss": 1.3328, + "theoretical_loss": 3.74546676329241, + "tokens_seen": 826867712 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037850551654964893, + "loss": 1.3389, + "theoretical_loss": 3.7454377349315826, + "tokens_seen": 826933248 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037849548645937816, + "loss": 1.3345, + "theoretical_loss": 3.7454087095153126, + "tokens_seen": 826998784 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003784854563691073, + "loss": 1.3783, + "theoretical_loss": 3.7453796870430667, + "tokens_seen": 827064320 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003784754262788365, + "loss": 1.2566, + "theoretical_loss": 3.7453506675143142, + "tokens_seen": 827129856 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037846539618856565, + "loss": 1.3404, + "theoretical_loss": 3.7453216509285223, + "tokens_seen": 827195392 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003784553660982949, + "loss": 1.4644, + "theoretical_loss": 3.7452926372851607, + "tokens_seen": 827260928 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037844533600802407, + "loss": 1.4153, + "theoretical_loss": 3.7452636265836965, + "tokens_seen": 827326464 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037843530591775325, + "loss": 1.2976, + "theoretical_loss": 3.7452346188236, + "tokens_seen": 827392000 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037842527582748243, + "loss": 1.3887, + "theoretical_loss": 3.745205614004339, + "tokens_seen": 827457536 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037841524573721167, + "loss": 1.31, + "theoretical_loss": 3.745176612125382, + "tokens_seen": 827523072 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003784052156469408, + "loss": 1.4057, + "theoretical_loss": 3.7451476131861994, + "tokens_seen": 827588608 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037839518555667003, + "loss": 1.3729, + "theoretical_loss": 3.74511861718626, + "tokens_seen": 827654144 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037838515546639916, + "loss": 1.5434, + "theoretical_loss": 3.7450896241250327, + "tokens_seen": 827719680 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003783751253761284, + "loss": 1.369, + "theoretical_loss": 3.7450606340019874, + "tokens_seen": 827785216 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037836509528585763, + "loss": 1.3663, + "theoretical_loss": 3.7450316468165936, + "tokens_seen": 827850752 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037835506519558675, + "loss": 1.4167, + "theoretical_loss": 3.7450026625683215, + "tokens_seen": 827916288 + }, + { + "epoch": 0.25, + "learning_rate": 0.000378345035105316, + "loss": 1.4257, + "theoretical_loss": 3.7449736812566403, + "tokens_seen": 827981824 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003783350050150451, + "loss": 1.3882, + "theoretical_loss": 3.744944702881021, + "tokens_seen": 828047360 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037832497492477435, + "loss": 1.3561, + "theoretical_loss": 3.744915727440933, + "tokens_seen": 828112896 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037831494483450353, + "loss": 1.2374, + "theoretical_loss": 3.7448867549358473, + "tokens_seen": 828178432 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003783049147442327, + "loss": 1.3988, + "theoretical_loss": 3.744857785365234, + "tokens_seen": 828243968 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003782948846539619, + "loss": 1.4126, + "theoretical_loss": 3.744828818728563, + "tokens_seen": 828309504 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037828485456369113, + "loss": 1.4163, + "theoretical_loss": 3.744799855025307, + "tokens_seen": 828375040 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037827482447342026, + "loss": 1.3167, + "theoretical_loss": 3.744770894254935, + "tokens_seen": 828440576 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003782647943831495, + "loss": 1.4272, + "theoretical_loss": 3.744741936416919, + "tokens_seen": 828506112 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003782547642928786, + "loss": 1.4342, + "theoretical_loss": 3.74471298151073, + "tokens_seen": 828571648 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037824473420260786, + "loss": 1.2974, + "theoretical_loss": 3.744684029535839, + "tokens_seen": 828637184 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037823470411233704, + "loss": 1.3289, + "theoretical_loss": 3.744655080491718, + "tokens_seen": 828702720 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003782246740220662, + "loss": 1.3551, + "theoretical_loss": 3.744626134377838, + "tokens_seen": 828768256 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003782146439317954, + "loss": 1.3334, + "theoretical_loss": 3.7445971911936713, + "tokens_seen": 828833792 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003782046138415246, + "loss": 1.4333, + "theoretical_loss": 3.7445682509386895, + "tokens_seen": 828899328 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037819458375125376, + "loss": 1.3208, + "theoretical_loss": 3.7445393136123646, + "tokens_seen": 828964864 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 280685, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4547964334487915, + "objective/train/theoretical_loss": 3.7445103792141685, + "objective/train/tokens_used": 829030400, + "theoretical_loss": 3.7445103792141685, + "tokens_seen": 829030400 + }, + { + "epoch": 0.25, + "learning_rate": 0.000378184553660983, + "loss": 1.4222, + "theoretical_loss": 3.7445103792141685, + "tokens_seen": 829030400 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003781745235707121, + "loss": 1.3387, + "theoretical_loss": 3.7444814477435737, + "tokens_seen": 829095936 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037816449348044136, + "loss": 1.3001, + "theoretical_loss": 3.7444525192000526, + "tokens_seen": 829161472 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003781544633901705, + "loss": 1.3839, + "theoretical_loss": 3.744423593583078, + "tokens_seen": 829227008 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003781444332998997, + "loss": 1.2735, + "theoretical_loss": 3.744394670892122, + "tokens_seen": 829292544 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003781344032096289, + "loss": 1.3506, + "theoretical_loss": 3.744365751126658, + "tokens_seen": 829358080 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003781243731193581, + "loss": 1.4506, + "theoretical_loss": 3.744336834286158, + "tokens_seen": 829423616 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037811434302908726, + "loss": 1.372, + "theoretical_loss": 3.7443079203700966, + "tokens_seen": 829489152 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003781043129388165, + "loss": 1.4511, + "theoretical_loss": 3.7442790093779452, + "tokens_seen": 829554688 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003780942828485456, + "loss": 1.4085, + "theoretical_loss": 3.744250101309179, + "tokens_seen": 829620224 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037808425275827486, + "loss": 1.4687, + "theoretical_loss": 3.74422119616327, + "tokens_seen": 829685760 + }, + { + "epoch": 0.25, + "learning_rate": 0.000378074222668004, + "loss": 1.2944, + "theoretical_loss": 3.744192293939693, + "tokens_seen": 829751296 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003780641925777332, + "loss": 1.3512, + "theoretical_loss": 3.744163394637921, + "tokens_seen": 829816832 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003780541624874624, + "loss": 1.3428, + "theoretical_loss": 3.744134498257428, + "tokens_seen": 829882368 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003780441323971916, + "loss": 1.5518, + "theoretical_loss": 3.7441056047976886, + "tokens_seen": 829947904 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037803410230692077, + "loss": 1.3489, + "theoretical_loss": 3.744076714258177, + "tokens_seen": 830013440 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037802407221664995, + "loss": 1.4312, + "theoretical_loss": 3.744047826638366, + "tokens_seen": 830078976 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037801404212637913, + "loss": 1.3114, + "theoretical_loss": 3.7440189419377314, + "tokens_seen": 830144512 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037800401203610836, + "loss": 1.289, + "theoretical_loss": 3.743990060155748, + "tokens_seen": 830210048 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003779939819458375, + "loss": 1.3827, + "theoretical_loss": 3.7439611812918896, + "tokens_seen": 830275584 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037798395185556673, + "loss": 1.2411, + "theoretical_loss": 3.743932305345632, + "tokens_seen": 830341120 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037797392176529585, + "loss": 1.2892, + "theoretical_loss": 3.7439034323164493, + "tokens_seen": 830406656 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003779638916750251, + "loss": 1.3785, + "theoretical_loss": 3.7438745622038176, + "tokens_seen": 830472192 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037795386158475427, + "loss": 1.2673, + "theoretical_loss": 3.743845695007211, + "tokens_seen": 830537728 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037794383149448345, + "loss": 1.4367, + "theoretical_loss": 3.7438168307261055, + "tokens_seen": 830603264 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037793380140421263, + "loss": 1.3509, + "theoretical_loss": 3.7437879693599774, + "tokens_seen": 830668800 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037792377131394187, + "loss": 1.407, + "theoretical_loss": 3.743759110908301, + "tokens_seen": 830734336 + }, + { + "epoch": 0.25, + "learning_rate": 0.000377913741223671, + "loss": 1.4297, + "theoretical_loss": 3.743730255370553, + "tokens_seen": 830799872 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037790371113340023, + "loss": 1.383, + "theoretical_loss": 3.743701402746209, + "tokens_seen": 830865408 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037789368104312936, + "loss": 1.3552, + "theoretical_loss": 3.743672553034745, + "tokens_seen": 830930944 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003778836509528586, + "loss": 1.2655, + "theoretical_loss": 3.743643706235638, + "tokens_seen": 830996480 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003778736208625878, + "loss": 1.309, + "theoretical_loss": 3.7436148623483634, + "tokens_seen": 831062016 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037786359077231695, + "loss": 1.3519, + "theoretical_loss": 3.7435860213723977, + "tokens_seen": 831127552 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037785356068204614, + "loss": 1.399, + "theoretical_loss": 3.7435571833072183, + "tokens_seen": 831193088 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003778435305917753, + "loss": 1.3899, + "theoretical_loss": 3.743528348152301, + "tokens_seen": 831258624 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003778335005015045, + "loss": 1.4371, + "theoretical_loss": 3.743499515907124, + "tokens_seen": 831324160 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037782347041123373, + "loss": 1.3974, + "theoretical_loss": 3.7434706865711633, + "tokens_seen": 831389696 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037781344032096286, + "loss": 1.3621, + "theoretical_loss": 3.7434418601438963, + "tokens_seen": 831455232 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003778034102306921, + "loss": 1.431, + "theoretical_loss": 3.7434130366248004, + "tokens_seen": 831520768 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003777933801404213, + "loss": 1.3706, + "theoretical_loss": 3.743384216013353, + "tokens_seen": 831586304 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037778335005015046, + "loss": 1.4033, + "theoretical_loss": 3.7433553983090313, + "tokens_seen": 831651840 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037777331995987964, + "loss": 1.4968, + "theoretical_loss": 3.743326583511313, + "tokens_seen": 831717376 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003777632898696088, + "loss": 1.3021, + "theoretical_loss": 3.7432977716196767, + "tokens_seen": 831782912 + }, + { + "epoch": 0.25, + "learning_rate": 0.000377753259779338, + "loss": 1.4788, + "theoretical_loss": 3.7432689626336, + "tokens_seen": 831848448 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037774322968906724, + "loss": 1.5217, + "theoretical_loss": 3.7432401565525613, + "tokens_seen": 831913984 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037773319959879636, + "loss": 1.3123, + "theoretical_loss": 3.743211353376038, + "tokens_seen": 831979520 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003777231695085256, + "loss": 1.2802, + "theoretical_loss": 3.743182553103509, + "tokens_seen": 832045056 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003777131394182547, + "loss": 1.3379, + "theoretical_loss": 3.7431537557344527, + "tokens_seen": 832110592 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037770310932798396, + "loss": 1.3913, + "theoretical_loss": 3.7431249612683484, + "tokens_seen": 832176128 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037769307923771314, + "loss": 1.4486, + "theoretical_loss": 3.743096169704674, + "tokens_seen": 832241664 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 281458, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3786439895629883, + "objective/train/theoretical_loss": 3.7430673810429083, + "objective/train/tokens_used": 832307200, + "theoretical_loss": 3.7430673810429083, + "tokens_seen": 832307200 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003776830491474423, + "loss": 1.3855, + "theoretical_loss": 3.7430673810429083, + "tokens_seen": 832307200 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003776730190571715, + "loss": 1.3793, + "theoretical_loss": 3.743038595282531, + "tokens_seen": 832372736 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003776629889669007, + "loss": 1.5625, + "theoretical_loss": 3.7430098124230216, + "tokens_seen": 832438272 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037765295887662987, + "loss": 1.4366, + "theoretical_loss": 3.7429810324638586, + "tokens_seen": 832503808 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003776429287863591, + "loss": 1.4548, + "theoretical_loss": 3.7429522554045223, + "tokens_seen": 832569344 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037763289869608823, + "loss": 1.4031, + "theoretical_loss": 3.7429234812444907, + "tokens_seen": 832634880 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037762286860581746, + "loss": 1.3547, + "theoretical_loss": 3.7428947099832452, + "tokens_seen": 832700416 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003776128385155467, + "loss": 1.3955, + "theoretical_loss": 3.7428659416202654, + "tokens_seen": 832765952 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003776028084252758, + "loss": 1.4511, + "theoretical_loss": 3.7428371761550303, + "tokens_seen": 832831488 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037759277833500506, + "loss": 1.4532, + "theoretical_loss": 3.7428084135870208, + "tokens_seen": 832897024 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003775827482447342, + "loss": 1.5376, + "theoretical_loss": 3.7427796539157177, + "tokens_seen": 832962560 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003775727181544634, + "loss": 1.3583, + "theoretical_loss": 3.7427508971406, + "tokens_seen": 833028096 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003775626880641926, + "loss": 1.3734, + "theoretical_loss": 3.7427221432611493, + "tokens_seen": 833093632 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003775526579739218, + "loss": 1.3193, + "theoretical_loss": 3.7426933922768457, + "tokens_seen": 833159168 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037754262788365097, + "loss": 1.4165, + "theoretical_loss": 3.74266464418717, + "tokens_seen": 833224704 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037753259779338015, + "loss": 1.4199, + "theoretical_loss": 3.7426358989916038, + "tokens_seen": 833290240 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037752256770310933, + "loss": 1.3609, + "theoretical_loss": 3.742607156689628, + "tokens_seen": 833355776 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037751253761283857, + "loss": 1.4064, + "theoretical_loss": 3.742578417280723, + "tokens_seen": 833421312 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003775025075225677, + "loss": 1.5362, + "theoretical_loss": 3.7425496807643706, + "tokens_seen": 833486848 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037749247743229693, + "loss": 1.3424, + "theoretical_loss": 3.7425209471400525, + "tokens_seen": 833552384 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037748244734202605, + "loss": 1.3403, + "theoretical_loss": 3.7424922164072503, + "tokens_seen": 833617920 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003774724172517553, + "loss": 1.5086, + "theoretical_loss": 3.7424634885654453, + "tokens_seen": 833683456 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037746238716148447, + "loss": 1.3651, + "theoretical_loss": 3.7424347636141198, + "tokens_seen": 833748992 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037745235707121365, + "loss": 1.4252, + "theoretical_loss": 3.742406041552756, + "tokens_seen": 833814528 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037744232698094283, + "loss": 1.3582, + "theoretical_loss": 3.742377322380835, + "tokens_seen": 833880064 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037743229689067207, + "loss": 1.4508, + "theoretical_loss": 3.74234860609784, + "tokens_seen": 833945600 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003774222668004012, + "loss": 1.3863, + "theoretical_loss": 3.742319892703253, + "tokens_seen": 834011136 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037741223671013043, + "loss": 1.4499, + "theoretical_loss": 3.7422911821965568, + "tokens_seen": 834076672 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037740220661985956, + "loss": 1.4669, + "theoretical_loss": 3.7422624745772337, + "tokens_seen": 834142208 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003773921765295888, + "loss": 1.365, + "theoretical_loss": 3.742233769844767, + "tokens_seen": 834207744 + }, + { + "epoch": 0.25, + "learning_rate": 0.000377382146439318, + "loss": 1.3759, + "theoretical_loss": 3.74220506799864, + "tokens_seen": 834273280 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037737211634904715, + "loss": 1.4081, + "theoretical_loss": 3.7421763690383347, + "tokens_seen": 834338816 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037736208625877634, + "loss": 1.4797, + "theoretical_loss": 3.742147672963334, + "tokens_seen": 834404352 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003773520561685055, + "loss": 1.3624, + "theoretical_loss": 3.742118979773123, + "tokens_seen": 834469888 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003773420260782347, + "loss": 1.384, + "theoretical_loss": 3.742090289467184, + "tokens_seen": 834535424 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037733199598796393, + "loss": 1.5325, + "theoretical_loss": 3.7420616020450006, + "tokens_seen": 834600960 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037732196589769306, + "loss": 1.3348, + "theoretical_loss": 3.742032917506057, + "tokens_seen": 834666496 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003773119358074223, + "loss": 1.436, + "theoretical_loss": 3.742004235849836, + "tokens_seen": 834732032 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003773019057171515, + "loss": 1.4272, + "theoretical_loss": 3.7419755570758233, + "tokens_seen": 834797568 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037729187562688066, + "loss": 1.3025, + "theoretical_loss": 3.7419468811835017, + "tokens_seen": 834863104 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037728184553660984, + "loss": 1.3625, + "theoretical_loss": 3.741918208172356, + "tokens_seen": 834928640 + }, + { + "epoch": 0.25, + "learning_rate": 0.000377271815446339, + "loss": 1.446, + "theoretical_loss": 3.74188953804187, + "tokens_seen": 834994176 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003772617853560682, + "loss": 1.4703, + "theoretical_loss": 3.7418608707915295, + "tokens_seen": 835059712 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037725175526579744, + "loss": 1.3565, + "theoretical_loss": 3.741832206420818, + "tokens_seen": 835125248 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037724172517552656, + "loss": 1.3982, + "theoretical_loss": 3.7418035449292204, + "tokens_seen": 835190784 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003772316950852558, + "loss": 1.366, + "theoretical_loss": 3.741774886316222, + "tokens_seen": 835256320 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003772216649949849, + "loss": 1.4359, + "theoretical_loss": 3.741746230581308, + "tokens_seen": 835321856 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037721163490471416, + "loss": 1.2922, + "theoretical_loss": 3.7417175777239633, + "tokens_seen": 835387392 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037720160481444334, + "loss": 1.3505, + "theoretical_loss": 3.741688927743673, + "tokens_seen": 835452928 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003771915747241725, + "loss": 1.3685, + "theoretical_loss": 3.7416602806399233, + "tokens_seen": 835518464 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 282600, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4472484588623047, + "objective/train/theoretical_loss": 3.7416316364121993, + "objective/train/tokens_used": 835584000, + "theoretical_loss": 3.7416316364121993, + "tokens_seen": 835584000 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003771815446339017, + "loss": 1.4749, + "theoretical_loss": 3.7416316364121993, + "tokens_seen": 835584000 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003771715145436309, + "loss": 1.4764, + "theoretical_loss": 3.7416029950599867, + "tokens_seen": 835649536 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037716148445336007, + "loss": 1.2856, + "theoretical_loss": 3.7415743565827713, + "tokens_seen": 835715072 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003771514543630893, + "loss": 1.403, + "theoretical_loss": 3.741545720980039, + "tokens_seen": 835780608 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037714142427281843, + "loss": 1.2554, + "theoretical_loss": 3.7415170882512765, + "tokens_seen": 835846144 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037713139418254766, + "loss": 1.3125, + "theoretical_loss": 3.7414884583959696, + "tokens_seen": 835911680 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037712136409227685, + "loss": 1.4617, + "theoretical_loss": 3.7414598314136045, + "tokens_seen": 835977216 + }, + { + "epoch": 0.25, + "learning_rate": 0.000377111334002006, + "loss": 1.4144, + "theoretical_loss": 3.741431207303668, + "tokens_seen": 836042752 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003771013039117352, + "loss": 1.4236, + "theoretical_loss": 3.7414025860656475, + "tokens_seen": 836108288 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003770912738214644, + "loss": 1.3068, + "theoretical_loss": 3.7413739676990283, + "tokens_seen": 836173824 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037708124373119357, + "loss": 1.3519, + "theoretical_loss": 3.741345352203298, + "tokens_seen": 836239360 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003770712136409228, + "loss": 1.398, + "theoretical_loss": 3.741316739577944, + "tokens_seen": 836304896 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037706118355065193, + "loss": 1.471, + "theoretical_loss": 3.741288129822453, + "tokens_seen": 836370432 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037705115346038117, + "loss": 1.4376, + "theoretical_loss": 3.7412595229363124, + "tokens_seen": 836435968 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003770411233701103, + "loss": 1.3937, + "theoretical_loss": 3.74123091891901, + "tokens_seen": 836501504 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037703109327983953, + "loss": 1.5057, + "theoretical_loss": 3.741202317770033, + "tokens_seen": 836567040 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003770210631895687, + "loss": 1.4399, + "theoretical_loss": 3.7411737194888692, + "tokens_seen": 836632576 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003770110330992979, + "loss": 1.2943, + "theoretical_loss": 3.7411451240750067, + "tokens_seen": 836698112 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003770010030090271, + "loss": 1.3308, + "theoretical_loss": 3.7411165315279327, + "tokens_seen": 836763648 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037699097291875625, + "loss": 1.4704, + "theoretical_loss": 3.7410879418471357, + "tokens_seen": 836829184 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037698094282848544, + "loss": 1.3678, + "theoretical_loss": 3.741059355032104, + "tokens_seen": 836894720 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037697091273821467, + "loss": 1.4735, + "theoretical_loss": 3.7410307710823263, + "tokens_seen": 836960256 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003769608826479438, + "loss": 1.5086, + "theoretical_loss": 3.7410021899972907, + "tokens_seen": 837025792 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037695085255767303, + "loss": 1.3963, + "theoretical_loss": 3.740973611776486, + "tokens_seen": 837091328 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003769408224674022, + "loss": 1.3932, + "theoretical_loss": 3.740945036419401, + "tokens_seen": 837156864 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003769307923771314, + "loss": 1.3275, + "theoretical_loss": 3.740916463925524, + "tokens_seen": 837222400 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003769207622868606, + "loss": 1.3469, + "theoretical_loss": 3.740887894294344, + "tokens_seen": 837287936 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037691073219658976, + "loss": 1.3954, + "theoretical_loss": 3.740859327525352, + "tokens_seen": 837353472 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037690070210631894, + "loss": 1.3678, + "theoretical_loss": 3.740830763618035, + "tokens_seen": 837419008 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003768906720160482, + "loss": 1.3383, + "theoretical_loss": 3.7408022025718832, + "tokens_seen": 837484544 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003768806419257773, + "loss": 1.314, + "theoretical_loss": 3.740773644386387, + "tokens_seen": 837550080 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037687061183550654, + "loss": 1.4123, + "theoretical_loss": 3.7407450890610345, + "tokens_seen": 837615616 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003768605817452357, + "loss": 1.3496, + "theoretical_loss": 3.7407165365953166, + "tokens_seen": 837681152 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003768505516549649, + "loss": 1.4132, + "theoretical_loss": 3.740687986988723, + "tokens_seen": 837746688 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037684052156469413, + "loss": 1.3769, + "theoretical_loss": 3.740659440240744, + "tokens_seen": 837812224 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037683049147442326, + "loss": 1.2933, + "theoretical_loss": 3.740630896350869, + "tokens_seen": 837877760 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003768204613841525, + "loss": 1.2713, + "theoretical_loss": 3.740602355318589, + "tokens_seen": 837943296 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003768104312938817, + "loss": 1.3487, + "theoretical_loss": 3.7405738171433947, + "tokens_seen": 838008832 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037680040120361086, + "loss": 1.3934, + "theoretical_loss": 3.740545281824776, + "tokens_seen": 838074368 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037679037111334004, + "loss": 1.4323, + "theoretical_loss": 3.7405167493622242, + "tokens_seen": 838139904 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003767803410230692, + "loss": 1.3918, + "theoretical_loss": 3.740488219755229, + "tokens_seen": 838205440 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003767703109327984, + "loss": 1.4489, + "theoretical_loss": 3.740459693003283, + "tokens_seen": 838270976 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037676028084252764, + "loss": 1.3274, + "theoretical_loss": 3.7404311691058765, + "tokens_seen": 838336512 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037675025075225676, + "loss": 1.4543, + "theoretical_loss": 3.740402648062501, + "tokens_seen": 838402048 + }, + { + "epoch": 0.25, + "learning_rate": 0.000376740220661986, + "loss": 1.4407, + "theoretical_loss": 3.740374129872647, + "tokens_seen": 838467584 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003767301905717151, + "loss": 1.446, + "theoretical_loss": 3.7403456145358076, + "tokens_seen": 838533120 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037672016048144436, + "loss": 1.3563, + "theoretical_loss": 3.740317102051473, + "tokens_seen": 838598656 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037671013039117354, + "loss": 1.4621, + "theoretical_loss": 3.7402885924191356, + "tokens_seen": 838664192 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003767001003009027, + "loss": 1.3803, + "theoretical_loss": 3.740260085638287, + "tokens_seen": 838729728 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003766900702106319, + "loss": 1.2715, + "theoretical_loss": 3.7402315817084197, + "tokens_seen": 838795264 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 283474, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4712613821029663, + "objective/train/theoretical_loss": 3.7402030806290254, + "objective/train/tokens_used": 838860800, + "theoretical_loss": 3.7402030806290254, + "tokens_seen": 838860800 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003766800401203611, + "loss": 1.3596, + "theoretical_loss": 3.7402030806290254, + "tokens_seen": 838860800 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037667001003009027, + "loss": 1.4016, + "theoretical_loss": 3.7401745823995967, + "tokens_seen": 838926336 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003766599799398195, + "loss": 1.5555, + "theoretical_loss": 3.7401460870196255, + "tokens_seen": 838991872 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037664994984954863, + "loss": 1.512, + "theoretical_loss": 3.7401175944886047, + "tokens_seen": 839057408 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037663991975927786, + "loss": 1.4263, + "theoretical_loss": 3.740089104806027, + "tokens_seen": 839122944 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037662988966900705, + "loss": 1.3757, + "theoretical_loss": 3.7400606179713853, + "tokens_seen": 839188480 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003766198595787362, + "loss": 1.4487, + "theoretical_loss": 3.7400321339841724, + "tokens_seen": 839254016 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003766098294884654, + "loss": 1.3008, + "theoretical_loss": 3.740003652843882, + "tokens_seen": 839319552 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003765997993981946, + "loss": 1.3795, + "theoretical_loss": 3.739975174550006, + "tokens_seen": 839385088 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037658976930792377, + "loss": 1.3784, + "theoretical_loss": 3.7399466991020383, + "tokens_seen": 839450624 + }, + { + "epoch": 0.25, + "learning_rate": 0.000376579739217653, + "loss": 1.1968, + "theoretical_loss": 3.739918226499473, + "tokens_seen": 839516160 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037656970912738213, + "loss": 1.3925, + "theoretical_loss": 3.7398897567418024, + "tokens_seen": 839581696 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037655967903711137, + "loss": 1.3204, + "theoretical_loss": 3.7398612898285215, + "tokens_seen": 839647232 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003765496489468405, + "loss": 1.3825, + "theoretical_loss": 3.7398328257591236, + "tokens_seen": 839712768 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037653961885656973, + "loss": 1.4346, + "theoretical_loss": 3.7398043645331027, + "tokens_seen": 839778304 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003765295887662989, + "loss": 1.4478, + "theoretical_loss": 3.7397759061499523, + "tokens_seen": 839843840 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003765195586760281, + "loss": 1.3401, + "theoretical_loss": 3.7397474506091677, + "tokens_seen": 839909376 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003765095285857573, + "loss": 1.3728, + "theoretical_loss": 3.7397189979102428, + "tokens_seen": 839974912 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037649949849548645, + "loss": 1.4372, + "theoretical_loss": 3.739690548052672, + "tokens_seen": 840040448 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037648946840521564, + "loss": 1.4856, + "theoretical_loss": 3.7396621010359494, + "tokens_seen": 840105984 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037647943831494487, + "loss": 1.3918, + "theoretical_loss": 3.739633656859571, + "tokens_seen": 840171520 + }, + { + "epoch": 0.25, + "learning_rate": 0.000376469408224674, + "loss": 1.4993, + "theoretical_loss": 3.739605215523031, + "tokens_seen": 840237056 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037645937813440323, + "loss": 1.384, + "theoretical_loss": 3.7395767770258246, + "tokens_seen": 840302592 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003764493480441324, + "loss": 1.5103, + "theoretical_loss": 3.739548341367446, + "tokens_seen": 840368128 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003764393179538616, + "loss": 1.3721, + "theoretical_loss": 3.7395199085473916, + "tokens_seen": 840433664 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003764292878635908, + "loss": 1.5094, + "theoretical_loss": 3.7394914785651565, + "tokens_seen": 840499200 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037641925777331996, + "loss": 1.4497, + "theoretical_loss": 3.739463051420236, + "tokens_seen": 840564736 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037640922768304914, + "loss": 1.3626, + "theoretical_loss": 3.739434627112126, + "tokens_seen": 840630272 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003763991975927784, + "loss": 1.4354, + "theoretical_loss": 3.7394062056403223, + "tokens_seen": 840695808 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003763891675025075, + "loss": 1.458, + "theoretical_loss": 3.73937778700432, + "tokens_seen": 840761344 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037637913741223674, + "loss": 1.3647, + "theoretical_loss": 3.739349371203616, + "tokens_seen": 840826880 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037636910732196586, + "loss": 1.3514, + "theoretical_loss": 3.739320958237707, + "tokens_seen": 840892416 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003763590772316951, + "loss": 1.3151, + "theoretical_loss": 3.7392925481060884, + "tokens_seen": 840957952 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003763490471414243, + "loss": 1.3245, + "theoretical_loss": 3.739264140808256, + "tokens_seen": 841023488 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037633901705115346, + "loss": 1.3786, + "theoretical_loss": 3.7392357363437077, + "tokens_seen": 841089024 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037632898696088264, + "loss": 1.3795, + "theoretical_loss": 3.73920733471194, + "tokens_seen": 841154560 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003763189568706119, + "loss": 1.3452, + "theoretical_loss": 3.7391789359124488, + "tokens_seen": 841220096 + }, + { + "epoch": 0.25, + "learning_rate": 0.000376308926780341, + "loss": 1.3568, + "theoretical_loss": 3.739150539944732, + "tokens_seen": 841285632 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037629889669007024, + "loss": 1.3282, + "theoretical_loss": 3.7391221468082856, + "tokens_seen": 841351168 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037628886659979937, + "loss": 1.2626, + "theoretical_loss": 3.7390937565026086, + "tokens_seen": 841416704 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003762788365095286, + "loss": 1.4679, + "theoretical_loss": 3.7390653690271964, + "tokens_seen": 841482240 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003762688064192578, + "loss": 1.3515, + "theoretical_loss": 3.7390369843815474, + "tokens_seen": 841547776 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037625877632898696, + "loss": 1.4425, + "theoretical_loss": 3.7390086025651588, + "tokens_seen": 841613312 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037624874623871615, + "loss": 1.3695, + "theoretical_loss": 3.7389802235775287, + "tokens_seen": 841678848 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003762387161484453, + "loss": 1.2855, + "theoretical_loss": 3.738951847418155, + "tokens_seen": 841744384 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003762286860581745, + "loss": 1.3254, + "theoretical_loss": 3.738923474086535, + "tokens_seen": 841809920 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037621865596790374, + "loss": 1.3861, + "theoretical_loss": 3.7388951035821676, + "tokens_seen": 841875456 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037620862587763287, + "loss": 1.4263, + "theoretical_loss": 3.7388667359045504, + "tokens_seen": 841940992 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003761985957873621, + "loss": 1.3513, + "theoretical_loss": 3.7388383710531823, + "tokens_seen": 842006528 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037618856569709123, + "loss": 1.3916, + "theoretical_loss": 3.7388100090275618, + "tokens_seen": 842072064 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 284666, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3507171869277954, + "objective/train/theoretical_loss": 3.738781649827186, + "objective/train/tokens_used": 842137600, + "theoretical_loss": 3.738781649827186, + "tokens_seen": 842137600 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037617853560682047, + "loss": 1.3975, + "theoretical_loss": 3.738781649827186, + "tokens_seen": 842137600 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037616850551654965, + "loss": 1.4733, + "theoretical_loss": 3.7387532934515564, + "tokens_seen": 842203136 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037615847542627883, + "loss": 1.4024, + "theoretical_loss": 3.738724939900169, + "tokens_seen": 842268672 + }, + { + "epoch": 0.26, + "learning_rate": 0.000376148445336008, + "loss": 1.4222, + "theoretical_loss": 3.738696589172525, + "tokens_seen": 842334208 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037613841524573725, + "loss": 1.2599, + "theoretical_loss": 3.7386682412681225, + "tokens_seen": 842399744 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037612838515546637, + "loss": 1.4669, + "theoretical_loss": 3.73863989618646, + "tokens_seen": 842465280 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003761183550651956, + "loss": 1.4058, + "theoretical_loss": 3.738611553927039, + "tokens_seen": 842530816 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003761083249749248, + "loss": 1.3665, + "theoretical_loss": 3.738583214489357, + "tokens_seen": 842596352 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037609829488465397, + "loss": 1.3299, + "theoretical_loss": 3.7385548778729145, + "tokens_seen": 842661888 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003760882647943832, + "loss": 1.3615, + "theoretical_loss": 3.738526544077211, + "tokens_seen": 842727424 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037607823470411233, + "loss": 1.3903, + "theoretical_loss": 3.738498213101747, + "tokens_seen": 842792960 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037606820461384157, + "loss": 1.391, + "theoretical_loss": 3.7384698849460216, + "tokens_seen": 842858496 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003760581745235707, + "loss": 1.4049, + "theoretical_loss": 3.7384415596095355, + "tokens_seen": 842924032 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037604814443329993, + "loss": 1.393, + "theoretical_loss": 3.7384132370917884, + "tokens_seen": 842989568 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003760381143430291, + "loss": 1.4029, + "theoretical_loss": 3.738384917392282, + "tokens_seen": 843055104 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003760280842527583, + "loss": 1.5124, + "theoretical_loss": 3.7383566005105155, + "tokens_seen": 843120640 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003760180541624875, + "loss": 1.3449, + "theoretical_loss": 3.7383282864459897, + "tokens_seen": 843186176 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037600802407221665, + "loss": 1.3749, + "theoretical_loss": 3.7382999751982062, + "tokens_seen": 843251712 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037599799398194584, + "loss": 1.3084, + "theoretical_loss": 3.738271666766665, + "tokens_seen": 843317248 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037598796389167507, + "loss": 1.3378, + "theoretical_loss": 3.738243361150868, + "tokens_seen": 843382784 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003759779338014042, + "loss": 1.3626, + "theoretical_loss": 3.7382150583503155, + "tokens_seen": 843448320 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037596790371113343, + "loss": 1.3944, + "theoretical_loss": 3.7381867583645096, + "tokens_seen": 843513856 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003759578736208626, + "loss": 1.4947, + "theoretical_loss": 3.738158461192951, + "tokens_seen": 843579392 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003759478435305918, + "loss": 1.3526, + "theoretical_loss": 3.738130166835141, + "tokens_seen": 843644928 + }, + { + "epoch": 0.26, + "learning_rate": 0.000375937813440321, + "loss": 1.3933, + "theoretical_loss": 3.7381018752905826, + "tokens_seen": 843710464 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037592778335005016, + "loss": 1.3292, + "theoretical_loss": 3.7380735865587766, + "tokens_seen": 843776000 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037591775325977934, + "loss": 1.4209, + "theoretical_loss": 3.738045300639225, + "tokens_seen": 843841536 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003759077231695086, + "loss": 1.3404, + "theoretical_loss": 3.7380170175314302, + "tokens_seen": 843907072 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003758976930792377, + "loss": 1.2978, + "theoretical_loss": 3.737988737234894, + "tokens_seen": 843972608 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037588766298896694, + "loss": 1.3782, + "theoretical_loss": 3.737960459749119, + "tokens_seen": 844038144 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037587763289869606, + "loss": 1.4622, + "theoretical_loss": 3.737932185073607, + "tokens_seen": 844103680 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003758676028084253, + "loss": 1.38, + "theoretical_loss": 3.7379039132078615, + "tokens_seen": 844169216 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003758575727181545, + "loss": 1.3855, + "theoretical_loss": 3.7378756441513845, + "tokens_seen": 844234752 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037584754262788366, + "loss": 1.4914, + "theoretical_loss": 3.737847377903679, + "tokens_seen": 844300288 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037583751253761284, + "loss": 1.3441, + "theoretical_loss": 3.737819114464248, + "tokens_seen": 844365824 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003758274824473421, + "loss": 1.5023, + "theoretical_loss": 3.7377908538325944, + "tokens_seen": 844431360 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003758174523570712, + "loss": 1.4645, + "theoretical_loss": 3.7377625960082215, + "tokens_seen": 844496896 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037580742226680044, + "loss": 1.3533, + "theoretical_loss": 3.7377343409906327, + "tokens_seen": 844562432 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037579739217652957, + "loss": 1.4839, + "theoretical_loss": 3.7377060887793316, + "tokens_seen": 844627968 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003757873620862588, + "loss": 1.4153, + "theoretical_loss": 3.7376778393738217, + "tokens_seen": 844693504 + }, + { + "epoch": 0.26, + "learning_rate": 0.000375777331995988, + "loss": 1.4797, + "theoretical_loss": 3.737649592773606, + "tokens_seen": 844759040 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037576730190571716, + "loss": 1.3707, + "theoretical_loss": 3.737621348978189, + "tokens_seen": 844824576 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037575727181544635, + "loss": 1.3482, + "theoretical_loss": 3.7375931079870743, + "tokens_seen": 844890112 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003757472417251755, + "loss": 1.332, + "theoretical_loss": 3.7375648697997663, + "tokens_seen": 844955648 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003757372116349047, + "loss": 1.3683, + "theoretical_loss": 3.737536634415769, + "tokens_seen": 845021184 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037572718154463394, + "loss": 1.3003, + "theoretical_loss": 3.737508401834587, + "tokens_seen": 845086720 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037571715145436307, + "loss": 1.3779, + "theoretical_loss": 3.7374801720557245, + "tokens_seen": 845152256 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003757071213640923, + "loss": 1.421, + "theoretical_loss": 3.737451945078686, + "tokens_seen": 845217792 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037569709127382143, + "loss": 1.3136, + "theoretical_loss": 3.737423720902976, + "tokens_seen": 845283328 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037568706118355067, + "loss": 1.4105, + "theoretical_loss": 3.7373954995281005, + "tokens_seen": 845348864 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 285774, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3740968704223633, + "objective/train/theoretical_loss": 3.7373672809535634, + "objective/train/tokens_used": 845414400, + "theoretical_loss": 3.7373672809535634, + "tokens_seen": 845414400 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037567703109327985, + "loss": 1.4331, + "theoretical_loss": 3.7373672809535634, + "tokens_seen": 845414400 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037566700100300903, + "loss": 1.3983, + "theoretical_loss": 3.7373390651788694, + "tokens_seen": 845479936 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003756569709127382, + "loss": 1.5544, + "theoretical_loss": 3.7373108522035245, + "tokens_seen": 845545472 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037564694082246745, + "loss": 1.2139, + "theoretical_loss": 3.737282642027034, + "tokens_seen": 845611008 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003756369107321966, + "loss": 1.4518, + "theoretical_loss": 3.737254434648903, + "tokens_seen": 845676544 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003756268806419258, + "loss": 1.3426, + "theoretical_loss": 3.7372262300686376, + "tokens_seen": 845742080 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037561685055165494, + "loss": 1.305, + "theoretical_loss": 3.737198028285743, + "tokens_seen": 845807616 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037560682046138417, + "loss": 1.3851, + "theoretical_loss": 3.7371698292997255, + "tokens_seen": 845873152 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037559679037111335, + "loss": 1.3513, + "theoretical_loss": 3.73714163311009, + "tokens_seen": 845938688 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037558676028084253, + "loss": 1.3629, + "theoretical_loss": 3.737113439716344, + "tokens_seen": 846004224 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003755767301905717, + "loss": 1.3198, + "theoretical_loss": 3.737085249117993, + "tokens_seen": 846069760 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003755667001003009, + "loss": 1.4489, + "theoretical_loss": 3.737057061314543, + "tokens_seen": 846135296 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003755566700100301, + "loss": 1.3953, + "theoretical_loss": 3.7370288763055015, + "tokens_seen": 846200832 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003755466399197593, + "loss": 1.3556, + "theoretical_loss": 3.7370006940903737, + "tokens_seen": 846266368 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037553660982948844, + "loss": 1.4019, + "theoretical_loss": 3.7369725146686674, + "tokens_seen": 846331904 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003755265797392177, + "loss": 1.3189, + "theoretical_loss": 3.7369443380398897, + "tokens_seen": 846397440 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003755165496489468, + "loss": 1.3203, + "theoretical_loss": 3.736916164203546, + "tokens_seen": 846462976 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037550651955867604, + "loss": 1.3987, + "theoretical_loss": 3.7368879931591454, + "tokens_seen": 846528512 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003754964894684052, + "loss": 1.3986, + "theoretical_loss": 3.736859824906193, + "tokens_seen": 846594048 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003754864593781344, + "loss": 1.2903, + "theoretical_loss": 3.736831659444198, + "tokens_seen": 846659584 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003754764292878636, + "loss": 1.3504, + "theoretical_loss": 3.7368034967726667, + "tokens_seen": 846725120 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003754663991975928, + "loss": 1.3231, + "theoretical_loss": 3.736775336891107, + "tokens_seen": 846790656 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037545636910732194, + "loss": 1.3254, + "theoretical_loss": 3.736747179799027, + "tokens_seen": 846856192 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003754463390170512, + "loss": 1.4365, + "theoretical_loss": 3.736719025495934, + "tokens_seen": 846921728 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003754363089267803, + "loss": 1.3794, + "theoretical_loss": 3.7366908739813365, + "tokens_seen": 846987264 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037542627883650954, + "loss": 1.3816, + "theoretical_loss": 3.736662725254742, + "tokens_seen": 847052800 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003754162487462387, + "loss": 1.4314, + "theoretical_loss": 3.736634579315659, + "tokens_seen": 847118336 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003754062186559679, + "loss": 1.3934, + "theoretical_loss": 3.736606436163596, + "tokens_seen": 847183872 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003753961885656971, + "loss": 1.3556, + "theoretical_loss": 3.7365782957980613, + "tokens_seen": 847249408 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037538615847542626, + "loss": 1.4035, + "theoretical_loss": 3.736550158218563, + "tokens_seen": 847314944 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037537612838515544, + "loss": 1.3679, + "theoretical_loss": 3.736522023424611, + "tokens_seen": 847380480 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003753660982948847, + "loss": 1.2572, + "theoretical_loss": 3.736493891415713, + "tokens_seen": 847446016 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037535606820461386, + "loss": 1.4419, + "theoretical_loss": 3.7364657621913784, + "tokens_seen": 847511552 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037534603811434304, + "loss": 1.3918, + "theoretical_loss": 3.7364376357511166, + "tokens_seen": 847577088 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003753360080240723, + "loss": 1.3795, + "theoretical_loss": 3.7364095120944363, + "tokens_seen": 847642624 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003753259779338014, + "loss": 1.4356, + "theoretical_loss": 3.736381391220847, + "tokens_seen": 847708160 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037531594784353064, + "loss": 1.4106, + "theoretical_loss": 3.736353273129858, + "tokens_seen": 847773696 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037530591775325977, + "loss": 1.3747, + "theoretical_loss": 3.7363251578209797, + "tokens_seen": 847839232 + }, + { + "epoch": 0.26, + "learning_rate": 0.000375295887662989, + "loss": 1.3645, + "theoretical_loss": 3.7362970452937208, + "tokens_seen": 847904768 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003752858575727182, + "loss": 1.4675, + "theoretical_loss": 3.7362689355475913, + "tokens_seen": 847970304 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037527582748244736, + "loss": 1.4721, + "theoretical_loss": 3.7362408285821016, + "tokens_seen": 848035840 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037526579739217655, + "loss": 1.4267, + "theoretical_loss": 3.7362127243967613, + "tokens_seen": 848101376 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003752557673019057, + "loss": 1.3947, + "theoretical_loss": 3.7361846229910816, + "tokens_seen": 848166912 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003752457372116349, + "loss": 1.3424, + "theoretical_loss": 3.736156524364571, + "tokens_seen": 848232448 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037523570712136414, + "loss": 1.414, + "theoretical_loss": 3.736128428516742, + "tokens_seen": 848297984 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037522567703109327, + "loss": 1.3698, + "theoretical_loss": 3.736100335447104, + "tokens_seen": 848363520 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003752156469408225, + "loss": 1.4458, + "theoretical_loss": 3.736072245155168, + "tokens_seen": 848429056 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037520561685055163, + "loss": 1.3057, + "theoretical_loss": 3.7360441576404444, + "tokens_seen": 848494592 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037519558676028087, + "loss": 1.39, + "theoretical_loss": 3.7360160729024443, + "tokens_seen": 848560128 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037518555667001005, + "loss": 1.4237, + "theoretical_loss": 3.7359879909406795, + "tokens_seen": 848625664 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 286696, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4635753631591797, + "objective/train/theoretical_loss": 3.7359599117546605, + "objective/train/tokens_used": 848691200, + "theoretical_loss": 3.7359599117546605, + "tokens_seen": 848691200 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037517552657973923, + "loss": 1.4717, + "theoretical_loss": 3.7359599117546605, + "tokens_seen": 848691200 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003751654964894684, + "loss": 1.4462, + "theoretical_loss": 3.735931835343899, + "tokens_seen": 848756736 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037515546639919765, + "loss": 1.4034, + "theoretical_loss": 3.7359037617079056, + "tokens_seen": 848822272 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003751454363089268, + "loss": 1.3322, + "theoretical_loss": 3.735875690846193, + "tokens_seen": 848887808 + }, + { + "epoch": 0.26, + "learning_rate": 0.000375135406218656, + "loss": 1.3266, + "theoretical_loss": 3.735847622758272, + "tokens_seen": 848953344 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037512537612838514, + "loss": 1.3786, + "theoretical_loss": 3.7358195574436555, + "tokens_seen": 849018880 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037511534603811437, + "loss": 1.3246, + "theoretical_loss": 3.7357914949018545, + "tokens_seen": 849084416 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037510531594784355, + "loss": 1.2573, + "theoretical_loss": 3.735763435132381, + "tokens_seen": 849149952 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037509528585757273, + "loss": 1.3239, + "theoretical_loss": 3.735735378134748, + "tokens_seen": 849215488 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003750852557673019, + "loss": 1.4323, + "theoretical_loss": 3.735707323908467, + "tokens_seen": 849281024 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003750752256770311, + "loss": 1.4113, + "theoretical_loss": 3.7356792724530505, + "tokens_seen": 849346560 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003750651955867603, + "loss": 1.2134, + "theoretical_loss": 3.735651223768012, + "tokens_seen": 849412096 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003750551654964895, + "loss": 1.2743, + "theoretical_loss": 3.735623177852863, + "tokens_seen": 849477632 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037504513540621864, + "loss": 1.4546, + "theoretical_loss": 3.7355951347071166, + "tokens_seen": 849543168 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003750351053159479, + "loss": 1.4201, + "theoretical_loss": 3.7355670943302863, + "tokens_seen": 849608704 + }, + { + "epoch": 0.26, + "learning_rate": 0.000375025075225677, + "loss": 1.3628, + "theoretical_loss": 3.735539056721885, + "tokens_seen": 849674240 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037501504513540624, + "loss": 1.3628, + "theoretical_loss": 3.7355110218814254, + "tokens_seen": 849739776 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003750050150451354, + "loss": 1.4475, + "theoretical_loss": 3.7354829898084208, + "tokens_seen": 849805312 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003749949849548646, + "loss": 1.3497, + "theoretical_loss": 3.735454960502385, + "tokens_seen": 849870848 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003749849548645938, + "loss": 1.4971, + "theoretical_loss": 3.735426933962831, + "tokens_seen": 849936384 + }, + { + "epoch": 0.26, + "learning_rate": 0.000374974924774323, + "loss": 1.3869, + "theoretical_loss": 3.7353989101892733, + "tokens_seen": 850001920 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037496489468405214, + "loss": 1.3631, + "theoretical_loss": 3.7353708891812256, + "tokens_seen": 850067456 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003749548645937814, + "loss": 1.4148, + "theoretical_loss": 3.735342870938201, + "tokens_seen": 850132992 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003749448345035105, + "loss": 1.3345, + "theoretical_loss": 3.7353148554597144, + "tokens_seen": 850198528 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037493480441323974, + "loss": 1.408, + "theoretical_loss": 3.7352868427452792, + "tokens_seen": 850264064 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003749247743229689, + "loss": 1.3708, + "theoretical_loss": 3.7352588327944103, + "tokens_seen": 850329600 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003749147442326981, + "loss": 1.373, + "theoretical_loss": 3.735230825606622, + "tokens_seen": 850395136 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003749047141424273, + "loss": 1.3898, + "theoretical_loss": 3.735202821181428, + "tokens_seen": 850460672 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037489468405215646, + "loss": 1.45, + "theoretical_loss": 3.7351748195183436, + "tokens_seen": 850526208 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037488465396188564, + "loss": 1.442, + "theoretical_loss": 3.735146820616884, + "tokens_seen": 850591744 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003748746238716149, + "loss": 1.4079, + "theoretical_loss": 3.735118824476564, + "tokens_seen": 850657280 + }, + { + "epoch": 0.26, + "learning_rate": 0.000374864593781344, + "loss": 1.3563, + "theoretical_loss": 3.7350908310968975, + "tokens_seen": 850722816 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037485456369107324, + "loss": 1.4647, + "theoretical_loss": 3.735062840477401, + "tokens_seen": 850788352 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037484453360080237, + "loss": 1.2714, + "theoretical_loss": 3.735034852617589, + "tokens_seen": 850853888 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003748345035105316, + "loss": 1.4108, + "theoretical_loss": 3.7350068675169776, + "tokens_seen": 850919424 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003748244734202608, + "loss": 1.3859, + "theoretical_loss": 3.734978885175081, + "tokens_seen": 850984960 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037481444332998997, + "loss": 1.3777, + "theoretical_loss": 3.734950905591416, + "tokens_seen": 851050496 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037480441323971915, + "loss": 1.3384, + "theoretical_loss": 3.7349229287654984, + "tokens_seen": 851116032 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003747943831494484, + "loss": 1.4299, + "theoretical_loss": 3.734894954696843, + "tokens_seen": 851181568 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003747843530591775, + "loss": 1.3852, + "theoretical_loss": 3.7348669833849666, + "tokens_seen": 851247104 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037477432296890675, + "loss": 1.4486, + "theoretical_loss": 3.734839014829385, + "tokens_seen": 851312640 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037476429287863587, + "loss": 1.369, + "theoretical_loss": 3.7348110490296147, + "tokens_seen": 851378176 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003747542627883651, + "loss": 1.3649, + "theoretical_loss": 3.734783085985172, + "tokens_seen": 851443712 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003747442326980943, + "loss": 1.397, + "theoretical_loss": 3.7347551256955738, + "tokens_seen": 851509248 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037473420260782347, + "loss": 1.3367, + "theoretical_loss": 3.734727168160336, + "tokens_seen": 851574784 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037472417251755265, + "loss": 1.3658, + "theoretical_loss": 3.734699213378975, + "tokens_seen": 851640320 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037471414242728183, + "loss": 1.2727, + "theoretical_loss": 3.734671261351009, + "tokens_seen": 851705856 + }, + { + "epoch": 0.26, + "learning_rate": 0.000374704112337011, + "loss": 1.3791, + "theoretical_loss": 3.7346433120759537, + "tokens_seen": 851771392 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037469408224674025, + "loss": 1.3849, + "theoretical_loss": 3.734615365553327, + "tokens_seen": 851836928 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003746840521564694, + "loss": 1.4183, + "theoretical_loss": 3.7345874217826456, + "tokens_seen": 851902464 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 287810, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3656646013259888, + "objective/train/theoretical_loss": 3.734559480763427, + "objective/train/tokens_used": 851968000, + "theoretical_loss": 3.734559480763427, + "tokens_seen": 851968000 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003746740220661986, + "loss": 1.4139, + "theoretical_loss": 3.734559480763427, + "tokens_seen": 851968000 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037466399197592774, + "loss": 1.4548, + "theoretical_loss": 3.734531542495189, + "tokens_seen": 852033536 + }, + { + "epoch": 0.26, + "learning_rate": 0.000374653961885657, + "loss": 1.2787, + "theoretical_loss": 3.7345036069774484, + "tokens_seen": 852099072 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037464393179538615, + "loss": 1.2892, + "theoretical_loss": 3.734475674209724, + "tokens_seen": 852164608 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037463390170511534, + "loss": 1.3616, + "theoretical_loss": 3.7344477441915322, + "tokens_seen": 852230144 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037462387161484457, + "loss": 1.4457, + "theoretical_loss": 3.7344198169223928, + "tokens_seen": 852295680 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037461384152457375, + "loss": 1.3122, + "theoretical_loss": 3.734391892401822, + "tokens_seen": 852361216 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037460381143430293, + "loss": 1.2347, + "theoretical_loss": 3.7343639706293392, + "tokens_seen": 852426752 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003745937813440321, + "loss": 1.4272, + "theoretical_loss": 3.734336051604462, + "tokens_seen": 852492288 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003745837512537613, + "loss": 1.3957, + "theoretical_loss": 3.7343081353267094, + "tokens_seen": 852557824 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003745737211634905, + "loss": 1.4194, + "theoretical_loss": 3.7342802217955997, + "tokens_seen": 852623360 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003745636910732197, + "loss": 1.3686, + "theoretical_loss": 3.7342523110106516, + "tokens_seen": 852688896 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037455366098294884, + "loss": 1.5171, + "theoretical_loss": 3.7342244029713845, + "tokens_seen": 852754432 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003745436308926781, + "loss": 1.377, + "theoretical_loss": 3.7341964976773157, + "tokens_seen": 852819968 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003745336008024072, + "loss": 1.44, + "theoretical_loss": 3.734168595127966, + "tokens_seen": 852885504 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037452357071213644, + "loss": 1.3337, + "theoretical_loss": 3.7341406953228535, + "tokens_seen": 852951040 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003745135406218656, + "loss": 1.3134, + "theoretical_loss": 3.7341127982614974, + "tokens_seen": 853016576 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003745035105315948, + "loss": 1.3942, + "theoretical_loss": 3.734084903943418, + "tokens_seen": 853082112 + }, + { + "epoch": 0.26, + "learning_rate": 0.000374493480441324, + "loss": 1.409, + "theoretical_loss": 3.734057012368134, + "tokens_seen": 853147648 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003744834503510532, + "loss": 1.3972, + "theoretical_loss": 3.7340291235351657, + "tokens_seen": 853213184 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037447342026078234, + "loss": 1.3043, + "theoretical_loss": 3.734001237444032, + "tokens_seen": 853278720 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003744633901705116, + "loss": 1.3404, + "theoretical_loss": 3.7339733540942537, + "tokens_seen": 853344256 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003744533600802407, + "loss": 1.4178, + "theoretical_loss": 3.7339454734853508, + "tokens_seen": 853409792 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037444332998996994, + "loss": 1.3941, + "theoretical_loss": 3.733917595616842, + "tokens_seen": 853475328 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003744332998996991, + "loss": 1.4999, + "theoretical_loss": 3.7338897204882495, + "tokens_seen": 853540864 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003744232698094283, + "loss": 1.4106, + "theoretical_loss": 3.7338618480990924, + "tokens_seen": 853606400 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003744132397191575, + "loss": 1.3963, + "theoretical_loss": 3.7338339784488914, + "tokens_seen": 853671936 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037440320962888666, + "loss": 1.4198, + "theoretical_loss": 3.7338061115371675, + "tokens_seen": 853737472 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037439317953861585, + "loss": 1.4319, + "theoretical_loss": 3.7337782473634413, + "tokens_seen": 853803008 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003743831494483451, + "loss": 1.4434, + "theoretical_loss": 3.733750385927234, + "tokens_seen": 853868544 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003743731193580742, + "loss": 1.3693, + "theoretical_loss": 3.733722527228065, + "tokens_seen": 853934080 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037436308926780344, + "loss": 1.3998, + "theoretical_loss": 3.7336946712654573, + "tokens_seen": 853999616 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037435305917753257, + "loss": 1.3739, + "theoretical_loss": 3.733666818038931, + "tokens_seen": 854065152 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003743430290872618, + "loss": 1.3373, + "theoretical_loss": 3.733638967548008, + "tokens_seen": 854130688 + }, + { + "epoch": 0.26, + "learning_rate": 0.000374332998996991, + "loss": 1.3202, + "theoretical_loss": 3.733611119792209, + "tokens_seen": 854196224 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037432296890672017, + "loss": 1.3507, + "theoretical_loss": 3.733583274771057, + "tokens_seen": 854261760 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037431293881644935, + "loss": 1.3587, + "theoretical_loss": 3.733555432484072, + "tokens_seen": 854327296 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003743029087261786, + "loss": 1.3921, + "theoretical_loss": 3.733527592930777, + "tokens_seen": 854392832 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003742928786359077, + "loss": 1.3463, + "theoretical_loss": 3.7334997561106933, + "tokens_seen": 854458368 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037428284854563695, + "loss": 1.3704, + "theoretical_loss": 3.7334719220233437, + "tokens_seen": 854523904 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037427281845536607, + "loss": 1.3752, + "theoretical_loss": 3.7334440906682493, + "tokens_seen": 854589440 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003742627883650953, + "loss": 1.3104, + "theoretical_loss": 3.7334162620449334, + "tokens_seen": 854654976 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003742527582748245, + "loss": 1.4314, + "theoretical_loss": 3.733388436152918, + "tokens_seen": 854720512 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037424272818455367, + "loss": 1.3729, + "theoretical_loss": 3.733360612991725, + "tokens_seen": 854786048 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037423269809428285, + "loss": 1.3923, + "theoretical_loss": 3.733332792560878, + "tokens_seen": 854851584 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037422266800401203, + "loss": 1.4005, + "theoretical_loss": 3.7333049748599, + "tokens_seen": 854917120 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003742126379137412, + "loss": 1.4139, + "theoretical_loss": 3.7332771598883134, + "tokens_seen": 854982656 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037420260782347045, + "loss": 1.3298, + "theoretical_loss": 3.7332493476456405, + "tokens_seen": 855048192 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003741925777331996, + "loss": 1.3243, + "theoretical_loss": 3.7332215381314056, + "tokens_seen": 855113728 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003741825476429288, + "loss": 1.3624, + "theoretical_loss": 3.733193731345131, + "tokens_seen": 855179264 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 288715, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3886699676513672, + "objective/train/theoretical_loss": 3.733165927286341, + "objective/train/tokens_used": 855244800, + "theoretical_loss": 3.733165927286341, + "tokens_seen": 855244800 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037417251755265794, + "loss": 1.4307, + "theoretical_loss": 3.733165927286341, + "tokens_seen": 855244800 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003741624874623872, + "loss": 1.3597, + "theoretical_loss": 3.7331381259545586, + "tokens_seen": 855310336 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037415245737211635, + "loss": 1.414, + "theoretical_loss": 3.7331103273493076, + "tokens_seen": 855375872 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037414242728184554, + "loss": 1.4203, + "theoretical_loss": 3.733082531470111, + "tokens_seen": 855441408 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003741323971915747, + "loss": 1.4526, + "theoretical_loss": 3.7330547383164943, + "tokens_seen": 855506944 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037412236710130395, + "loss": 1.4289, + "theoretical_loss": 3.7330269478879794, + "tokens_seen": 855572480 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003741123370110331, + "loss": 1.4328, + "theoretical_loss": 3.732999160184092, + "tokens_seen": 855638016 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003741023069207623, + "loss": 1.4788, + "theoretical_loss": 3.7329713752043556, + "tokens_seen": 855703552 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037409227683049144, + "loss": 1.4091, + "theoretical_loss": 3.7329435929482946, + "tokens_seen": 855769088 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003740822467402207, + "loss": 1.3688, + "theoretical_loss": 3.732915813415434, + "tokens_seen": 855834624 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037407221664994986, + "loss": 1.411, + "theoretical_loss": 3.732888036605298, + "tokens_seen": 855900160 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037406218655967904, + "loss": 1.347, + "theoretical_loss": 3.7328602625174105, + "tokens_seen": 855965696 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003740521564694082, + "loss": 1.325, + "theoretical_loss": 3.7328324911512976, + "tokens_seen": 856031232 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003740421263791374, + "loss": 1.3607, + "theoretical_loss": 3.7328047225064838, + "tokens_seen": 856096768 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003740320962888666, + "loss": 1.3524, + "theoretical_loss": 3.7327769565824935, + "tokens_seen": 856162304 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003740220661985958, + "loss": 1.3804, + "theoretical_loss": 3.7327491933788526, + "tokens_seen": 856227840 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037401203610832494, + "loss": 1.3645, + "theoretical_loss": 3.7327214328950857, + "tokens_seen": 856293376 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003740020060180542, + "loss": 1.3441, + "theoretical_loss": 3.7326936751307196, + "tokens_seen": 856358912 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037399197592778336, + "loss": 1.4991, + "theoretical_loss": 3.732665920085278, + "tokens_seen": 856424448 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037398194583751254, + "loss": 1.4616, + "theoretical_loss": 3.732638167758288, + "tokens_seen": 856489984 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003739719157472417, + "loss": 1.3093, + "theoretical_loss": 3.7326104181492745, + "tokens_seen": 856555520 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003739618856569709, + "loss": 1.2839, + "theoretical_loss": 3.732582671257764, + "tokens_seen": 856621056 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003739518555667001, + "loss": 1.4156, + "theoretical_loss": 3.732554927083282, + "tokens_seen": 856686592 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003739418254764293, + "loss": 1.4188, + "theoretical_loss": 3.7325271856253552, + "tokens_seen": 856752128 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037393179538615845, + "loss": 1.2887, + "theoretical_loss": 3.732499446883509, + "tokens_seen": 856817664 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003739217652958877, + "loss": 1.3948, + "theoretical_loss": 3.7324717108572703, + "tokens_seen": 856883200 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003739117352056168, + "loss": 1.3189, + "theoretical_loss": 3.7324439775461657, + "tokens_seen": 856948736 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037390170511534605, + "loss": 1.4446, + "theoretical_loss": 3.7324162469497217, + "tokens_seen": 857014272 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003738916750250752, + "loss": 1.4382, + "theoretical_loss": 3.732388519067465, + "tokens_seen": 857079808 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003738816449348044, + "loss": 1.291, + "theoretical_loss": 3.732360793898922, + "tokens_seen": 857145344 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037387161484453364, + "loss": 1.4018, + "theoretical_loss": 3.7323330714436205, + "tokens_seen": 857210880 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037386158475426277, + "loss": 1.364, + "theoretical_loss": 3.7323053517010862, + "tokens_seen": 857276416 + }, + { + "epoch": 0.26, + "learning_rate": 0.000373851554663992, + "loss": 1.4384, + "theoretical_loss": 3.732277634670848, + "tokens_seen": 857341952 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003738415245737212, + "loss": 1.3362, + "theoretical_loss": 3.732249920352432, + "tokens_seen": 857407488 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037383149448345037, + "loss": 1.3776, + "theoretical_loss": 3.732222208745366, + "tokens_seen": 857473024 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037382146439317955, + "loss": 1.2966, + "theoretical_loss": 3.732194499849178, + "tokens_seen": 857538560 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003738114343029088, + "loss": 1.2902, + "theoretical_loss": 3.732166793663395, + "tokens_seen": 857604096 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003738014042126379, + "loss": 1.3522, + "theoretical_loss": 3.7321390901875455, + "tokens_seen": 857669632 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037379137412236715, + "loss": 1.3825, + "theoretical_loss": 3.7321113894211564, + "tokens_seen": 857735168 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037378134403209627, + "loss": 1.399, + "theoretical_loss": 3.732083691363756, + "tokens_seen": 857800704 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003737713139418255, + "loss": 1.3997, + "theoretical_loss": 3.732055996014873, + "tokens_seen": 857866240 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003737612838515547, + "loss": 1.4265, + "theoretical_loss": 3.732028303374035, + "tokens_seen": 857931776 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037375125376128387, + "loss": 1.3835, + "theoretical_loss": 3.7320006134407713, + "tokens_seen": 857997312 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037374122367101305, + "loss": 1.4538, + "theoretical_loss": 3.7319729262146093, + "tokens_seen": 858062848 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037373119358074223, + "loss": 1.5169, + "theoretical_loss": 3.7319452416950782, + "tokens_seen": 858128384 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003737211634904714, + "loss": 1.2942, + "theoretical_loss": 3.731917559881707, + "tokens_seen": 858193920 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037371113340020065, + "loss": 1.3817, + "theoretical_loss": 3.7318898807740237, + "tokens_seen": 858259456 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003737011033099298, + "loss": 1.3825, + "theoretical_loss": 3.7318622043715584, + "tokens_seen": 858324992 + }, + { + "epoch": 0.26, + "learning_rate": 0.000373691073219659, + "loss": 1.4159, + "theoretical_loss": 3.731834530673839, + "tokens_seen": 858390528 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037368104312938814, + "loss": 1.4254, + "theoretical_loss": 3.731806859680395, + "tokens_seen": 858456064 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 289863, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4462603330612183, + "objective/train/theoretical_loss": 3.7317791913907565, + "objective/train/tokens_used": 858521600, + "theoretical_loss": 3.7317791913907565, + "tokens_seen": 858521600 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003736710130391174, + "loss": 1.4063, + "theoretical_loss": 3.7317791913907565, + "tokens_seen": 858521600 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037366098294884655, + "loss": 1.3993, + "theoretical_loss": 3.7317515258044525, + "tokens_seen": 858587136 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037365095285857574, + "loss": 1.3758, + "theoretical_loss": 3.731723862921012, + "tokens_seen": 858652672 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003736409227683049, + "loss": 1.4398, + "theoretical_loss": 3.7316962027399656, + "tokens_seen": 858718208 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037363089267803415, + "loss": 1.3249, + "theoretical_loss": 3.731668545260842, + "tokens_seen": 858783744 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003736208625877633, + "loss": 1.3411, + "theoretical_loss": 3.731640890483172, + "tokens_seen": 858849280 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003736108324974925, + "loss": 1.303, + "theoretical_loss": 3.7316132384064855, + "tokens_seen": 858914816 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037360080240722164, + "loss": 1.2965, + "theoretical_loss": 3.7315855890303125, + "tokens_seen": 858980352 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003735907723169509, + "loss": 1.359, + "theoretical_loss": 3.7315579423541836, + "tokens_seen": 859045888 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037358074222668006, + "loss": 1.3704, + "theoretical_loss": 3.731530298377628, + "tokens_seen": 859111424 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037357071213640924, + "loss": 1.2665, + "theoretical_loss": 3.7315026571001777, + "tokens_seen": 859176960 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003735606820461384, + "loss": 1.3537, + "theoretical_loss": 3.731475018521362, + "tokens_seen": 859242496 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003735506519558676, + "loss": 1.3613, + "theoretical_loss": 3.731447382640713, + "tokens_seen": 859308032 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003735406218655968, + "loss": 1.3513, + "theoretical_loss": 3.73141974945776, + "tokens_seen": 859373568 + }, + { + "epoch": 0.26, + "learning_rate": 0.000373530591775326, + "loss": 1.4568, + "theoretical_loss": 3.7313921189720354, + "tokens_seen": 859439104 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037352056168505514, + "loss": 1.3048, + "theoretical_loss": 3.7313644911830695, + "tokens_seen": 859504640 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003735105315947844, + "loss": 1.4356, + "theoretical_loss": 3.731336866090394, + "tokens_seen": 859570176 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037350050150451356, + "loss": 1.4043, + "theoretical_loss": 3.7313092436935396, + "tokens_seen": 859635712 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037349047141424274, + "loss": 1.3475, + "theoretical_loss": 3.7312816239920377, + "tokens_seen": 859701248 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003734804413239719, + "loss": 1.4042, + "theoretical_loss": 3.731254006985421, + "tokens_seen": 859766784 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003734704112337011, + "loss": 1.3705, + "theoretical_loss": 3.7312263926732196, + "tokens_seen": 859832320 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003734603811434303, + "loss": 1.3121, + "theoretical_loss": 3.731198781054966, + "tokens_seen": 859897856 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003734503510531595, + "loss": 1.3535, + "theoretical_loss": 3.7311711721301926, + "tokens_seen": 859963392 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037344032096288865, + "loss": 1.3941, + "theoretical_loss": 3.7311435658984307, + "tokens_seen": 860028928 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003734302908726179, + "loss": 1.3193, + "theoretical_loss": 3.7311159623592127, + "tokens_seen": 860094464 + }, + { + "epoch": 0.26, + "learning_rate": 0.000373420260782347, + "loss": 1.2438, + "theoretical_loss": 3.7310883615120707, + "tokens_seen": 860160000 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037341023069207625, + "loss": 1.4433, + "theoretical_loss": 3.7310607633565374, + "tokens_seen": 860225536 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003734002006018054, + "loss": 1.3733, + "theoretical_loss": 3.7310331678921456, + "tokens_seen": 860291072 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003733901705115346, + "loss": 1.4224, + "theoretical_loss": 3.731005575118427, + "tokens_seen": 860356608 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003733801404212638, + "loss": 1.4043, + "theoretical_loss": 3.7309779850349143, + "tokens_seen": 860422144 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037337011033099297, + "loss": 1.3059, + "theoretical_loss": 3.730950397641141, + "tokens_seen": 860487680 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037336008024072215, + "loss": 1.3339, + "theoretical_loss": 3.73092281293664, + "tokens_seen": 860553216 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003733500501504514, + "loss": 1.2635, + "theoretical_loss": 3.730895230920944, + "tokens_seen": 860618752 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003733400200601805, + "loss": 1.4708, + "theoretical_loss": 3.730867651593586, + "tokens_seen": 860684288 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037332998996990975, + "loss": 1.3776, + "theoretical_loss": 3.7308400749541004, + "tokens_seen": 860749824 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037331995987963893, + "loss": 1.4193, + "theoretical_loss": 3.7308125010020197, + "tokens_seen": 860815360 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003733099297893681, + "loss": 1.4211, + "theoretical_loss": 3.7307849297368776, + "tokens_seen": 860880896 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003732998996990973, + "loss": 1.3507, + "theoretical_loss": 3.730757361158208, + "tokens_seen": 860946432 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003732898696088265, + "loss": 1.4241, + "theoretical_loss": 3.7307297952655443, + "tokens_seen": 861011968 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037327983951855565, + "loss": 1.3572, + "theoretical_loss": 3.7307022320584204, + "tokens_seen": 861077504 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003732698094282849, + "loss": 1.3229, + "theoretical_loss": 3.730674671536371, + "tokens_seen": 861143040 + }, + { + "epoch": 0.26, + "learning_rate": 0.000373259779338014, + "loss": 1.3885, + "theoretical_loss": 3.7306471136989288, + "tokens_seen": 861208576 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037324974924774325, + "loss": 1.3856, + "theoretical_loss": 3.7306195585456297, + "tokens_seen": 861274112 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003732397191574724, + "loss": 1.372, + "theoretical_loss": 3.730592006076007, + "tokens_seen": 861339648 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003732296890672016, + "loss": 1.4498, + "theoretical_loss": 3.730564456289595, + "tokens_seen": 861405184 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003732196589769308, + "loss": 1.3217, + "theoretical_loss": 3.730536909185929, + "tokens_seen": 861470720 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037320962888666, + "loss": 1.4292, + "theoretical_loss": 3.7305093647645435, + "tokens_seen": 861536256 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037319959879638916, + "loss": 1.382, + "theoretical_loss": 3.730481823024973, + "tokens_seen": 861601792 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037318956870611834, + "loss": 1.4748, + "theoretical_loss": 3.730454283966753, + "tokens_seen": 861667328 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003731795386158475, + "loss": 1.3911, + "theoretical_loss": 3.7304267475894184, + "tokens_seen": 861732864 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 291104, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4459387063980103, + "objective/train/theoretical_loss": 3.730399213892504, + "objective/train/tokens_used": 861798400, + "theoretical_loss": 3.730399213892504, + "tokens_seen": 861798400 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037316950852557675, + "loss": 1.382, + "theoretical_loss": 3.730399213892504, + "tokens_seen": 861798400 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003731594784353059, + "loss": 1.3746, + "theoretical_loss": 3.730371682875545, + "tokens_seen": 861863936 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003731494483450351, + "loss": 1.4296, + "theoretical_loss": 3.730344154538077, + "tokens_seen": 861929472 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003731394182547643, + "loss": 1.3336, + "theoretical_loss": 3.7303166288796357, + "tokens_seen": 861995008 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003731293881644935, + "loss": 1.3335, + "theoretical_loss": 3.7302891058997565, + "tokens_seen": 862060544 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003731193580742227, + "loss": 1.4797, + "theoretical_loss": 3.730261585597976, + "tokens_seen": 862126080 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037310932798395184, + "loss": 1.3475, + "theoretical_loss": 3.7302340679738286, + "tokens_seen": 862191616 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003730992978936811, + "loss": 1.3613, + "theoretical_loss": 3.730206553026851, + "tokens_seen": 862257152 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037308926780341026, + "loss": 1.3727, + "theoretical_loss": 3.730179040756579, + "tokens_seen": 862322688 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037307923771313944, + "loss": 1.2781, + "theoretical_loss": 3.7301515311625497, + "tokens_seen": 862388224 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003730692076228686, + "loss": 1.2654, + "theoretical_loss": 3.730124024244298, + "tokens_seen": 862453760 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003730591775325978, + "loss": 1.3409, + "theoretical_loss": 3.730096520001362, + "tokens_seen": 862519296 + }, + { + "epoch": 0.26, + "learning_rate": 0.000373049147442327, + "loss": 1.3095, + "theoretical_loss": 3.730069018433277, + "tokens_seen": 862584832 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003730391173520562, + "loss": 1.2867, + "theoretical_loss": 3.73004151953958, + "tokens_seen": 862650368 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037302908726178534, + "loss": 1.4824, + "theoretical_loss": 3.730014023319808, + "tokens_seen": 862715904 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003730190571715146, + "loss": 1.3593, + "theoretical_loss": 3.7299865297734973, + "tokens_seen": 862781440 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037300902708124376, + "loss": 1.3724, + "theoretical_loss": 3.7299590389001858, + "tokens_seen": 862846976 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037299899699097294, + "loss": 1.4865, + "theoretical_loss": 3.72993155069941, + "tokens_seen": 862912512 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003729889669007021, + "loss": 1.3967, + "theoretical_loss": 3.7299040651707074, + "tokens_seen": 862978048 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003729789368104313, + "loss": 1.3513, + "theoretical_loss": 3.7298765823136155, + "tokens_seen": 863043584 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003729689067201605, + "loss": 1.3863, + "theoretical_loss": 3.7298491021276714, + "tokens_seen": 863109120 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003729588766298897, + "loss": 1.3526, + "theoretical_loss": 3.7298216246124127, + "tokens_seen": 863174656 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037294884653961885, + "loss": 1.4697, + "theoretical_loss": 3.729794149767377, + "tokens_seen": 863240192 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003729388164493481, + "loss": 1.3825, + "theoretical_loss": 3.729766677592103, + "tokens_seen": 863305728 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003729287863590772, + "loss": 1.396, + "theoretical_loss": 3.7297392080861282, + "tokens_seen": 863371264 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037291875626880645, + "loss": 1.4643, + "theoretical_loss": 3.7297117412489897, + "tokens_seen": 863436800 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003729087261785356, + "loss": 1.3415, + "theoretical_loss": 3.729684277080227, + "tokens_seen": 863502336 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003728986960882648, + "loss": 1.495, + "theoretical_loss": 3.7296568155793777, + "tokens_seen": 863567872 + }, + { + "epoch": 0.26, + "learning_rate": 0.000372888665997994, + "loss": 1.331, + "theoretical_loss": 3.7296293567459804, + "tokens_seen": 863633408 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037287863590772317, + "loss": 1.308, + "theoretical_loss": 3.7296019005795733, + "tokens_seen": 863698944 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037286860581745235, + "loss": 1.3363, + "theoretical_loss": 3.7295744470796954, + "tokens_seen": 863764480 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003728585757271816, + "loss": 1.3666, + "theoretical_loss": 3.729546996245885, + "tokens_seen": 863830016 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003728485456369107, + "loss": 1.3265, + "theoretical_loss": 3.7295195480776817, + "tokens_seen": 863895552 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037283851554663995, + "loss": 1.3177, + "theoretical_loss": 3.729492102574624, + "tokens_seen": 863961088 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037282848545636913, + "loss": 1.3709, + "theoretical_loss": 3.7294646597362506, + "tokens_seen": 864026624 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003728184553660983, + "loss": 1.408, + "theoretical_loss": 3.729437219562101, + "tokens_seen": 864092160 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003728084252758275, + "loss": 1.3421, + "theoretical_loss": 3.7294097820517154, + "tokens_seen": 864157696 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003727983951855567, + "loss": 1.41, + "theoretical_loss": 3.7293823472046315, + "tokens_seen": 864223232 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037278836509528585, + "loss": 1.271, + "theoretical_loss": 3.729354915020391, + "tokens_seen": 864288768 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003727783350050151, + "loss": 1.3165, + "theoretical_loss": 3.729327485498531, + "tokens_seen": 864354304 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003727683049147442, + "loss": 1.3276, + "theoretical_loss": 3.7293000586385934, + "tokens_seen": 864419840 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037275827482447345, + "loss": 1.391, + "theoretical_loss": 3.7292726344401173, + "tokens_seen": 864485376 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003727482447342026, + "loss": 1.389, + "theoretical_loss": 3.7292452129026428, + "tokens_seen": 864550912 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003727382146439318, + "loss": 1.3993, + "theoretical_loss": 3.7292177940257094, + "tokens_seen": 864616448 + }, + { + "epoch": 0.26, + "learning_rate": 0.000372728184553661, + "loss": 1.3515, + "theoretical_loss": 3.7291903778088584, + "tokens_seen": 864681984 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003727181544633902, + "loss": 1.3698, + "theoretical_loss": 3.7291629642516293, + "tokens_seen": 864747520 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037270812437311936, + "loss": 1.4291, + "theoretical_loss": 3.7291355533535633, + "tokens_seen": 864813056 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037269809428284854, + "loss": 1.3867, + "theoretical_loss": 3.7291081451142003, + "tokens_seen": 864878592 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003726880641925777, + "loss": 1.4432, + "theoretical_loss": 3.729080739533081, + "tokens_seen": 864944128 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037267803410230696, + "loss": 1.3092, + "theoretical_loss": 3.7290533366097467, + "tokens_seen": 865009664 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 292007, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.261814832687378, + "objective/train/theoretical_loss": 3.7290259363437377, + "objective/train/tokens_used": 865075200, + "theoretical_loss": 3.7290259363437377, + "tokens_seen": 865075200 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003726680040120361, + "loss": 1.3687, + "theoretical_loss": 3.7290259363437377, + "tokens_seen": 865075200 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003726579739217653, + "loss": 1.3129, + "theoretical_loss": 3.7289985387345963, + "tokens_seen": 865140736 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003726479438314945, + "loss": 1.3347, + "theoretical_loss": 3.7289711437818616, + "tokens_seen": 865206272 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003726379137412237, + "loss": 1.3375, + "theoretical_loss": 3.7289437514850765, + "tokens_seen": 865271808 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037262788365095286, + "loss": 1.386, + "theoretical_loss": 3.728916361843782, + "tokens_seen": 865337344 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037261785356068204, + "loss": 1.3748, + "theoretical_loss": 3.728888974857519, + "tokens_seen": 865402880 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003726078234704112, + "loss": 1.3622, + "theoretical_loss": 3.7288615905258298, + "tokens_seen": 865468416 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037259779338014046, + "loss": 1.5036, + "theoretical_loss": 3.728834208848256, + "tokens_seen": 865533952 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003725877632898696, + "loss": 1.2666, + "theoretical_loss": 3.7288068298243386, + "tokens_seen": 865599488 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003725777331995988, + "loss": 1.3785, + "theoretical_loss": 3.7287794534536207, + "tokens_seen": 865665024 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037256770310932795, + "loss": 1.3884, + "theoretical_loss": 3.728752079735644, + "tokens_seen": 865730560 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003725576730190572, + "loss": 1.3272, + "theoretical_loss": 3.728724708669951, + "tokens_seen": 865796096 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037254764292878636, + "loss": 1.3604, + "theoretical_loss": 3.7286973402560823, + "tokens_seen": 865861632 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037253761283851554, + "loss": 1.3717, + "theoretical_loss": 3.7286699744935827, + "tokens_seen": 865927168 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003725275827482447, + "loss": 1.5013, + "theoretical_loss": 3.728642611381993, + "tokens_seen": 865992704 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037251755265797396, + "loss": 1.3721, + "theoretical_loss": 3.728615250920856, + "tokens_seen": 866058240 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003725075225677031, + "loss": 1.4101, + "theoretical_loss": 3.728587893109715, + "tokens_seen": 866123776 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003724974924774323, + "loss": 1.4211, + "theoretical_loss": 3.7285605379481126, + "tokens_seen": 866189312 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037248746238716145, + "loss": 1.3996, + "theoretical_loss": 3.728533185435592, + "tokens_seen": 866254848 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003724774322968907, + "loss": 1.3752, + "theoretical_loss": 3.7285058355716965, + "tokens_seen": 866320384 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037246740220661987, + "loss": 1.3423, + "theoretical_loss": 3.728478488355968, + "tokens_seen": 866385920 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037245737211634905, + "loss": 1.3588, + "theoretical_loss": 3.728451143787951, + "tokens_seen": 866451456 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037244734202607823, + "loss": 1.3411, + "theoretical_loss": 3.7284238018671885, + "tokens_seen": 866516992 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003724373119358074, + "loss": 1.3813, + "theoretical_loss": 3.728396462593224, + "tokens_seen": 866582528 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003724272818455366, + "loss": 1.3786, + "theoretical_loss": 3.7283691259656018, + "tokens_seen": 866648064 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003724172517552658, + "loss": 1.2643, + "theoretical_loss": 3.7283417919838646, + "tokens_seen": 866713600 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037240722166499495, + "loss": 1.2498, + "theoretical_loss": 3.7283144606475576, + "tokens_seen": 866779136 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003723971915747242, + "loss": 1.3802, + "theoretical_loss": 3.7282871319562227, + "tokens_seen": 866844672 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003723871614844533, + "loss": 1.3604, + "theoretical_loss": 3.7282598059094063, + "tokens_seen": 866910208 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037237713139418255, + "loss": 1.2952, + "theoretical_loss": 3.728232482506651, + "tokens_seen": 866975744 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003723671013039118, + "loss": 1.3081, + "theoretical_loss": 3.7282051617475016, + "tokens_seen": 867041280 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003723570712136409, + "loss": 1.3011, + "theoretical_loss": 3.728177843631503, + "tokens_seen": 867106816 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037234704112337015, + "loss": 1.3123, + "theoretical_loss": 3.7281505281581993, + "tokens_seen": 867172352 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037233701103309933, + "loss": 1.3413, + "theoretical_loss": 3.7281232153271353, + "tokens_seen": 867237888 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003723269809428285, + "loss": 1.4873, + "theoretical_loss": 3.728095905137856, + "tokens_seen": 867303424 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003723169508525577, + "loss": 1.4893, + "theoretical_loss": 3.728068597589905, + "tokens_seen": 867368960 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003723069207622869, + "loss": 1.4395, + "theoretical_loss": 3.7280412926828292, + "tokens_seen": 867434496 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037229689067201605, + "loss": 1.3306, + "theoretical_loss": 3.7280139904161724, + "tokens_seen": 867500032 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003722868605817453, + "loss": 1.4272, + "theoretical_loss": 3.7279866907894803, + "tokens_seen": 867565568 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003722768304914744, + "loss": 1.3585, + "theoretical_loss": 3.7279593938022986, + "tokens_seen": 867631104 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037226680040120365, + "loss": 1.3323, + "theoretical_loss": 3.7279320994541716, + "tokens_seen": 867696640 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003722567703109328, + "loss": 1.383, + "theoretical_loss": 3.727904807744646, + "tokens_seen": 867762176 + }, + { + "epoch": 0.26, + "learning_rate": 0.000372246740220662, + "loss": 1.4233, + "theoretical_loss": 3.7278775186732664, + "tokens_seen": 867827712 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003722367101303912, + "loss": 1.3611, + "theoretical_loss": 3.7278502322395797, + "tokens_seen": 867893248 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003722266800401204, + "loss": 1.3722, + "theoretical_loss": 3.727822948443132, + "tokens_seen": 867958784 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037221664994984956, + "loss": 1.3705, + "theoretical_loss": 3.727795667283468, + "tokens_seen": 868024320 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037220661985957874, + "loss": 1.4215, + "theoretical_loss": 3.7277683887601345, + "tokens_seen": 868089856 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003721965897693079, + "loss": 1.4173, + "theoretical_loss": 3.7277411128726774, + "tokens_seen": 868155392 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037218655967903716, + "loss": 1.3288, + "theoretical_loss": 3.7277138396206437, + "tokens_seen": 868220928 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003721765295887663, + "loss": 1.3314, + "theoretical_loss": 3.7276865690035796, + "tokens_seen": 868286464 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 293130, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2583335638046265, + "objective/train/theoretical_loss": 3.7276593010210313, + "objective/train/tokens_used": 868352000, + "theoretical_loss": 3.7276593010210313, + "tokens_seen": 868352000 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003721664994984955, + "loss": 1.3385, + "theoretical_loss": 3.7276593010210313, + "tokens_seen": 868352000 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003721564694082247, + "loss": 1.3542, + "theoretical_loss": 3.727632035672546, + "tokens_seen": 868417536 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003721464393179539, + "loss": 1.3488, + "theoretical_loss": 3.7276047729576707, + "tokens_seen": 868483072 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037213640922768306, + "loss": 1.3091, + "theoretical_loss": 3.727577512875951, + "tokens_seen": 868548608 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037212637913741224, + "loss": 1.4316, + "theoretical_loss": 3.7275502554269355, + "tokens_seen": 868614144 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003721163490471414, + "loss": 1.3825, + "theoretical_loss": 3.7275230006101703, + "tokens_seen": 868679680 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037210631895687066, + "loss": 1.475, + "theoretical_loss": 3.7274957484252034, + "tokens_seen": 868745216 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003720962888665998, + "loss": 1.3952, + "theoretical_loss": 3.7274684988715814, + "tokens_seen": 868810752 + }, + { + "epoch": 0.26, + "learning_rate": 0.000372086258776329, + "loss": 1.4317, + "theoretical_loss": 3.7274412519488522, + "tokens_seen": 868876288 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037207622868605815, + "loss": 1.3632, + "theoretical_loss": 3.727414007656564, + "tokens_seen": 868941824 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003720661985957874, + "loss": 1.3775, + "theoretical_loss": 3.7273867659942628, + "tokens_seen": 869007360 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037205616850551656, + "loss": 1.4539, + "theoretical_loss": 3.7273595269614974, + "tokens_seen": 869072896 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037204613841524575, + "loss": 1.5113, + "theoretical_loss": 3.727332290557816, + "tokens_seen": 869138432 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003720361083249749, + "loss": 1.2781, + "theoretical_loss": 3.7273050567827664, + "tokens_seen": 869203968 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037202607823470416, + "loss": 1.341, + "theoretical_loss": 3.7272778256358974, + "tokens_seen": 869269504 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003720160481444333, + "loss": 1.2517, + "theoretical_loss": 3.7272505971167558, + "tokens_seen": 869335040 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003720060180541625, + "loss": 1.3991, + "theoretical_loss": 3.7272233712248903, + "tokens_seen": 869400576 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037199598796389165, + "loss": 1.4513, + "theoretical_loss": 3.7271961479598508, + "tokens_seen": 869466112 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003719859578736209, + "loss": 1.413, + "theoretical_loss": 3.727168927321184, + "tokens_seen": 869531648 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037197592778335007, + "loss": 1.3821, + "theoretical_loss": 3.72714170930844, + "tokens_seen": 869597184 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037196589769307925, + "loss": 1.3814, + "theoretical_loss": 3.7271144939211673, + "tokens_seen": 869662720 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037195586760280843, + "loss": 1.3838, + "theoretical_loss": 3.727087281158914, + "tokens_seen": 869728256 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003719458375125376, + "loss": 1.3072, + "theoretical_loss": 3.7270600710212305, + "tokens_seen": 869793792 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003719358074222668, + "loss": 1.3872, + "theoretical_loss": 3.7270328635076644, + "tokens_seen": 869859328 + }, + { + "epoch": 0.26, + "learning_rate": 0.000371925777331996, + "loss": 1.4747, + "theoretical_loss": 3.7270056586177662, + "tokens_seen": 869924864 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037191574724172515, + "loss": 1.331, + "theoretical_loss": 3.726978456351085, + "tokens_seen": 869990400 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003719057171514544, + "loss": 1.4125, + "theoretical_loss": 3.72695125670717, + "tokens_seen": 870055936 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003718956870611835, + "loss": 1.4041, + "theoretical_loss": 3.7269240596855706, + "tokens_seen": 870121472 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037188565697091275, + "loss": 1.5462, + "theoretical_loss": 3.7268968652858367, + "tokens_seen": 870187008 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037187562688064193, + "loss": 1.3529, + "theoretical_loss": 3.726869673507519, + "tokens_seen": 870252544 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003718655967903711, + "loss": 1.4244, + "theoretical_loss": 3.726842484350166, + "tokens_seen": 870318080 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003718555667001003, + "loss": 1.4482, + "theoretical_loss": 3.726815297813329, + "tokens_seen": 870383616 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037184553660982953, + "loss": 1.3649, + "theoretical_loss": 3.7267881138965566, + "tokens_seen": 870449152 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037183550651955866, + "loss": 1.3753, + "theoretical_loss": 3.726760932599401, + "tokens_seen": 870514688 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003718254764292879, + "loss": 1.4188, + "theoretical_loss": 3.726733753921411, + "tokens_seen": 870580224 + }, + { + "epoch": 0.26, + "learning_rate": 0.000371815446339017, + "loss": 1.3009, + "theoretical_loss": 3.7267065778621378, + "tokens_seen": 870645760 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037180541624874625, + "loss": 1.3854, + "theoretical_loss": 3.726679404421131, + "tokens_seen": 870711296 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037179538615847544, + "loss": 1.4394, + "theoretical_loss": 3.7266522335979433, + "tokens_seen": 870776832 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003717853560682046, + "loss": 1.3375, + "theoretical_loss": 3.726625065392124, + "tokens_seen": 870842368 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003717753259779338, + "loss": 1.4545, + "theoretical_loss": 3.726597899803224, + "tokens_seen": 870907904 + }, + { + "epoch": 0.26, + "learning_rate": 0.000371765295887663, + "loss": 1.4417, + "theoretical_loss": 3.726570736830795, + "tokens_seen": 870973440 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037175526579739216, + "loss": 1.3943, + "theoretical_loss": 3.7265435764743877, + "tokens_seen": 871038976 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003717452357071214, + "loss": 1.3615, + "theoretical_loss": 3.7265164187335538, + "tokens_seen": 871104512 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003717352056168505, + "loss": 1.3981, + "theoretical_loss": 3.7264892636078444, + "tokens_seen": 871170048 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037172517552657976, + "loss": 1.372, + "theoretical_loss": 3.7264621110968106, + "tokens_seen": 871235584 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003717151454363089, + "loss": 1.498, + "theoretical_loss": 3.726434961200005, + "tokens_seen": 871301120 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003717051153460381, + "loss": 1.4048, + "theoretical_loss": 3.7264078139169783, + "tokens_seen": 871366656 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003716950852557673, + "loss": 1.4189, + "theoretical_loss": 3.7263806692472823, + "tokens_seen": 871432192 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003716850551654965, + "loss": 1.3259, + "theoretical_loss": 3.7263535271904695, + "tokens_seen": 871497728 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037167502507522566, + "loss": 1.4776, + "theoretical_loss": 3.726326387746092, + "tokens_seen": 871563264 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 294217, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3148512840270996, + "objective/train/theoretical_loss": 3.7262992509137014, + "objective/train/tokens_used": 871628800, + "theoretical_loss": 3.7262992509137014, + "tokens_seen": 871628800 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003716649949849549, + "loss": 1.3594, + "theoretical_loss": 3.7262992509137014, + "tokens_seen": 871628800 + }, + { + "epoch": 0.26, + "learning_rate": 0.000371654964894684, + "loss": 1.3627, + "theoretical_loss": 3.72627211669285, + "tokens_seen": 871694336 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037164493480441326, + "loss": 1.347, + "theoretical_loss": 3.7262449850830905, + "tokens_seen": 871759872 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003716349047141424, + "loss": 1.2548, + "theoretical_loss": 3.7262178560839754, + "tokens_seen": 871825408 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003716248746238716, + "loss": 1.3701, + "theoretical_loss": 3.726190729695057, + "tokens_seen": 871890944 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037161484453360086, + "loss": 1.3907, + "theoretical_loss": 3.7261636059158887, + "tokens_seen": 871956480 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037160481444333, + "loss": 1.3042, + "theoretical_loss": 3.726136484746022, + "tokens_seen": 872022016 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003715947843530592, + "loss": 1.3509, + "theoretical_loss": 3.7261093661850104, + "tokens_seen": 872087552 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037158475426278835, + "loss": 1.4103, + "theoretical_loss": 3.7260822502324076, + "tokens_seen": 872153088 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003715747241725176, + "loss": 1.4327, + "theoretical_loss": 3.726055136887766, + "tokens_seen": 872218624 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037156469408224676, + "loss": 1.3356, + "theoretical_loss": 3.7260280261506393, + "tokens_seen": 872284160 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037155466399197595, + "loss": 1.2977, + "theoretical_loss": 3.7260009180205804, + "tokens_seen": 872349696 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003715446339017051, + "loss": 1.3201, + "theoretical_loss": 3.7259738124971427, + "tokens_seen": 872415232 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037153460381143436, + "loss": 1.4214, + "theoretical_loss": 3.7259467095798806, + "tokens_seen": 872480768 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003715245737211635, + "loss": 1.3489, + "theoretical_loss": 3.7259196092683466, + "tokens_seen": 872546304 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003715145436308927, + "loss": 1.4214, + "theoretical_loss": 3.7258925115620953, + "tokens_seen": 872611840 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037150451354062185, + "loss": 1.458, + "theoretical_loss": 3.725865416460681, + "tokens_seen": 872677376 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003714944834503511, + "loss": 1.3547, + "theoretical_loss": 3.7258383239636563, + "tokens_seen": 872742912 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037148445336008027, + "loss": 1.4125, + "theoretical_loss": 3.7258112340705765, + "tokens_seen": 872808448 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037148445336008027, + "loss": 1.3405, + "theoretical_loss": 3.7257841467809953, + "tokens_seen": 872873984 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037147442326980945, + "loss": 1.2863, + "theoretical_loss": 3.725757062094467, + "tokens_seen": 872939520 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037146439317953863, + "loss": 1.371, + "theoretical_loss": 3.7257299800105472, + "tokens_seen": 873005056 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003714543630892678, + "loss": 1.4401, + "theoretical_loss": 3.7257029005287894, + "tokens_seen": 873070592 + }, + { + "epoch": 0.26, + "learning_rate": 0.000371444332998997, + "loss": 1.3649, + "theoretical_loss": 3.7256758236487477, + "tokens_seen": 873136128 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037143430290872623, + "loss": 1.4182, + "theoretical_loss": 3.725648749369978, + "tokens_seen": 873201664 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037142427281845535, + "loss": 1.4192, + "theoretical_loss": 3.725621677692035, + "tokens_seen": 873267200 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003714142427281846, + "loss": 1.4109, + "theoretical_loss": 3.725594608614473, + "tokens_seen": 873332736 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003714042126379137, + "loss": 1.2934, + "theoretical_loss": 3.725567542136848, + "tokens_seen": 873398272 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037139418254764295, + "loss": 1.3662, + "theoretical_loss": 3.725540478258715, + "tokens_seen": 873463808 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037138415245737213, + "loss": 1.355, + "theoretical_loss": 3.725513416979629, + "tokens_seen": 873529344 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003713741223671013, + "loss": 1.4562, + "theoretical_loss": 3.725486358299145, + "tokens_seen": 873594880 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003713640922768305, + "loss": 1.3591, + "theoretical_loss": 3.7254593022168203, + "tokens_seen": 873660416 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037135406218655973, + "loss": 1.3474, + "theoretical_loss": 3.7254322487322087, + "tokens_seen": 873725952 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037134403209628886, + "loss": 1.4605, + "theoretical_loss": 3.7254051978448666, + "tokens_seen": 873791488 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003713340020060181, + "loss": 1.3929, + "theoretical_loss": 3.7253781495543503, + "tokens_seen": 873857024 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003713239719157472, + "loss": 1.3074, + "theoretical_loss": 3.7253511038602154, + "tokens_seen": 873922560 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037131394182547645, + "loss": 1.3349, + "theoretical_loss": 3.7253240607620177, + "tokens_seen": 873988096 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037130391173520564, + "loss": 1.4507, + "theoretical_loss": 3.7252970202593136, + "tokens_seen": 874053632 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003712938816449348, + "loss": 1.4238, + "theoretical_loss": 3.7252699823516604, + "tokens_seen": 874119168 + }, + { + "epoch": 0.26, + "learning_rate": 0.000371283851554664, + "loss": 1.3316, + "theoretical_loss": 3.7252429470386135, + "tokens_seen": 874184704 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003712738214643932, + "loss": 1.3723, + "theoretical_loss": 3.7252159143197288, + "tokens_seen": 874250240 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037126379137412236, + "loss": 1.4632, + "theoretical_loss": 3.7251888841945644, + "tokens_seen": 874315776 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003712537612838516, + "loss": 1.3547, + "theoretical_loss": 3.7251618566626767, + "tokens_seen": 874381312 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003712437311935807, + "loss": 1.3808, + "theoretical_loss": 3.7251348317236217, + "tokens_seen": 874446848 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037123370110330996, + "loss": 1.4592, + "theoretical_loss": 3.725107809376957, + "tokens_seen": 874512384 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003712236710130391, + "loss": 1.3732, + "theoretical_loss": 3.7250807896222398, + "tokens_seen": 874577920 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003712136409227683, + "loss": 1.3569, + "theoretical_loss": 3.725053772459027, + "tokens_seen": 874643456 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003712036108324975, + "loss": 1.3625, + "theoretical_loss": 3.7250267578868757, + "tokens_seen": 874708992 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003711935807422267, + "loss": 1.3666, + "theoretical_loss": 3.724999745905344, + "tokens_seen": 874774528 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037118355065195586, + "loss": 1.3075, + "theoretical_loss": 3.724972736513989, + "tokens_seen": 874840064 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 295442, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3968369960784912, + "objective/train/theoretical_loss": 3.7249457297123687, + "objective/train/tokens_used": 874905600, + "theoretical_loss": 3.7249457297123687, + "tokens_seen": 874905600 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003711735205616851, + "loss": 1.4094, + "theoretical_loss": 3.7249457297123687, + "tokens_seen": 874905600 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003711634904714142, + "loss": 1.3593, + "theoretical_loss": 3.7249187255000393, + "tokens_seen": 874971136 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037115346038114346, + "loss": 1.1844, + "theoretical_loss": 3.724891723876561, + "tokens_seen": 875036672 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003711434302908726, + "loss": 1.4837, + "theoretical_loss": 3.7248647248414897, + "tokens_seen": 875102208 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003711334002006018, + "loss": 1.2382, + "theoretical_loss": 3.7248377283943848, + "tokens_seen": 875167744 + }, + { + "epoch": 0.27, + "learning_rate": 0.000371123370110331, + "loss": 1.3835, + "theoretical_loss": 3.7248107345348034, + "tokens_seen": 875233280 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003711133400200602, + "loss": 1.3719, + "theoretical_loss": 3.7247837432623054, + "tokens_seen": 875298816 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037110330992978937, + "loss": 1.3285, + "theoretical_loss": 3.724756754576447, + "tokens_seen": 875364352 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037109327983951855, + "loss": 1.3838, + "theoretical_loss": 3.7247297684767884, + "tokens_seen": 875429888 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037108324974924773, + "loss": 1.3669, + "theoretical_loss": 3.7247027849628873, + "tokens_seen": 875495424 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037107321965897696, + "loss": 1.3757, + "theoretical_loss": 3.7246758040343027, + "tokens_seen": 875560960 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003710631895687061, + "loss": 1.3094, + "theoretical_loss": 3.724648825690594, + "tokens_seen": 875626496 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003710531594784353, + "loss": 1.3121, + "theoretical_loss": 3.724621849931319, + "tokens_seen": 875692032 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037104312938816445, + "loss": 1.3235, + "theoretical_loss": 3.7245948767560373, + "tokens_seen": 875757568 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003710330992978937, + "loss": 1.3684, + "theoretical_loss": 3.724567906164308, + "tokens_seen": 875823104 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037102306920762287, + "loss": 1.3276, + "theoretical_loss": 3.72454093815569, + "tokens_seen": 875888640 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037101303911735205, + "loss": 1.3269, + "theoretical_loss": 3.724513972729744, + "tokens_seen": 875954176 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037100300902708123, + "loss": 1.3034, + "theoretical_loss": 3.724487009886028, + "tokens_seen": 876019712 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037099297893681047, + "loss": 1.308, + "theoretical_loss": 3.724460049624102, + "tokens_seen": 876085248 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003709829488465396, + "loss": 1.3438, + "theoretical_loss": 3.7244330919435256, + "tokens_seen": 876150784 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037097291875626883, + "loss": 1.4401, + "theoretical_loss": 3.724406136843859, + "tokens_seen": 876216320 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037096288866599796, + "loss": 1.3881, + "theoretical_loss": 3.7243791843246616, + "tokens_seen": 876281856 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003709528585757272, + "loss": 1.3757, + "theoretical_loss": 3.724352234385494, + "tokens_seen": 876347392 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003709428284854564, + "loss": 1.4731, + "theoretical_loss": 3.7243252870259154, + "tokens_seen": 876412928 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037093279839518555, + "loss": 1.376, + "theoretical_loss": 3.7242983422454867, + "tokens_seen": 876478464 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037092276830491474, + "loss": 1.3707, + "theoretical_loss": 3.724271400043768, + "tokens_seen": 876544000 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003709127382146439, + "loss": 1.3653, + "theoretical_loss": 3.72424446042032, + "tokens_seen": 876609536 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003709027081243731, + "loss": 1.3992, + "theoretical_loss": 3.724217523374703, + "tokens_seen": 876675072 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037089267803410233, + "loss": 1.3539, + "theoretical_loss": 3.7241905889064784, + "tokens_seen": 876740608 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037088264794383146, + "loss": 1.3013, + "theoretical_loss": 3.7241636570152052, + "tokens_seen": 876806144 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003708726178535607, + "loss": 1.2843, + "theoretical_loss": 3.724136727700446, + "tokens_seen": 876871680 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037086258776328993, + "loss": 1.4795, + "theoretical_loss": 3.7241098009617604, + "tokens_seen": 876937216 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037085255767301906, + "loss": 1.412, + "theoretical_loss": 3.724082876798711, + "tokens_seen": 877002752 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003708425275827483, + "loss": 1.4483, + "theoretical_loss": 3.724055955210858, + "tokens_seen": 877068288 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003708324974924774, + "loss": 1.3899, + "theoretical_loss": 3.7240290361977624, + "tokens_seen": 877133824 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037082246740220665, + "loss": 1.4586, + "theoretical_loss": 3.7240021197589863, + "tokens_seen": 877199360 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037081243731193584, + "loss": 1.4013, + "theoretical_loss": 3.723975205894091, + "tokens_seen": 877264896 + }, + { + "epoch": 0.27, + "learning_rate": 0.000370802407221665, + "loss": 1.4531, + "theoretical_loss": 3.7239482946026383, + "tokens_seen": 877330432 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003707923771313942, + "loss": 1.3224, + "theoretical_loss": 3.7239213858841893, + "tokens_seen": 877395968 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003707823470411234, + "loss": 1.3862, + "theoretical_loss": 3.723894479738307, + "tokens_seen": 877461504 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037077231695085256, + "loss": 1.5357, + "theoretical_loss": 3.723867576164552, + "tokens_seen": 877527040 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003707622868605818, + "loss": 1.4013, + "theoretical_loss": 3.723840675162487, + "tokens_seen": 877592576 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003707522567703109, + "loss": 1.327, + "theoretical_loss": 3.7238137767316744, + "tokens_seen": 877658112 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037074222668004016, + "loss": 1.441, + "theoretical_loss": 3.723786880871676, + "tokens_seen": 877723648 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003707321965897693, + "loss": 1.3047, + "theoretical_loss": 3.7237599875820546, + "tokens_seen": 877789184 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003707221664994985, + "loss": 1.3187, + "theoretical_loss": 3.723733096862372, + "tokens_seen": 877854720 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003707121364092277, + "loss": 1.3381, + "theoretical_loss": 3.7237062087121915, + "tokens_seen": 877920256 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003707021063189569, + "loss": 1.4182, + "theoretical_loss": 3.7236793231310754, + "tokens_seen": 877985792 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037069207622868606, + "loss": 1.3988, + "theoretical_loss": 3.7236524401185864, + "tokens_seen": 878051328 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003706820461384153, + "loss": 1.4572, + "theoretical_loss": 3.7236255596742884, + "tokens_seen": 878116864 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 296390, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5886049270629883, + "objective/train/theoretical_loss": 3.7235986817977427, + "objective/train/tokens_used": 878182400, + "theoretical_loss": 3.7235986817977427, + "tokens_seen": 878182400 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003706720160481444, + "loss": 1.4577, + "theoretical_loss": 3.7235986817977427, + "tokens_seen": 878182400 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037066198595787366, + "loss": 1.3922, + "theoretical_loss": 3.723571806488514, + "tokens_seen": 878247936 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003706519558676028, + "loss": 1.3315, + "theoretical_loss": 3.7235449337461644, + "tokens_seen": 878313472 + }, + { + "epoch": 0.27, + "learning_rate": 0.000370641925777332, + "loss": 1.3854, + "theoretical_loss": 3.723518063570258, + "tokens_seen": 878379008 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003706318956870612, + "loss": 1.3593, + "theoretical_loss": 3.7234911959603574, + "tokens_seen": 878444544 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003706218655967904, + "loss": 1.353, + "theoretical_loss": 3.7234643309160274, + "tokens_seen": 878510080 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037061183550651957, + "loss": 1.342, + "theoretical_loss": 3.7234374684368303, + "tokens_seen": 878575616 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037060180541624875, + "loss": 1.4496, + "theoretical_loss": 3.7234106085223306, + "tokens_seen": 878641152 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037059177532597793, + "loss": 1.4893, + "theoretical_loss": 3.7233837511720926, + "tokens_seen": 878706688 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037058174523570716, + "loss": 1.4388, + "theoretical_loss": 3.7233568963856793, + "tokens_seen": 878772224 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003705717151454363, + "loss": 1.3919, + "theoretical_loss": 3.7233300441626556, + "tokens_seen": 878837760 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003705616850551655, + "loss": 1.4709, + "theoretical_loss": 3.7233031945025847, + "tokens_seen": 878903296 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037055165496489465, + "loss": 1.2995, + "theoretical_loss": 3.723276347405032, + "tokens_seen": 878968832 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003705416248746239, + "loss": 1.4574, + "theoretical_loss": 3.723249502869561, + "tokens_seen": 879034368 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037053159478435307, + "loss": 1.3541, + "theoretical_loss": 3.723222660895737, + "tokens_seen": 879099904 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037052156469408225, + "loss": 1.3439, + "theoretical_loss": 3.7231958214831242, + "tokens_seen": 879165440 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037051153460381143, + "loss": 1.4275, + "theoretical_loss": 3.723168984631287, + "tokens_seen": 879230976 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037050150451354067, + "loss": 1.376, + "theoretical_loss": 3.7231421503397906, + "tokens_seen": 879296512 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003704914744232698, + "loss": 1.3889, + "theoretical_loss": 3.7231153186082, + "tokens_seen": 879362048 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037048144433299903, + "loss": 1.3154, + "theoretical_loss": 3.7230884894360807, + "tokens_seen": 879427584 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037047141424272816, + "loss": 1.3135, + "theoretical_loss": 3.723061662822997, + "tokens_seen": 879493120 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003704613841524574, + "loss": 1.45, + "theoretical_loss": 3.7230348387685144, + "tokens_seen": 879558656 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003704513540621866, + "loss": 1.3545, + "theoretical_loss": 3.723008017272198, + "tokens_seen": 879624192 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037044132397191575, + "loss": 1.382, + "theoretical_loss": 3.7229811983336134, + "tokens_seen": 879689728 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037043129388164494, + "loss": 1.3632, + "theoretical_loss": 3.7229543819523268, + "tokens_seen": 879755264 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003704212637913741, + "loss": 1.3778, + "theoretical_loss": 3.7229275681279033, + "tokens_seen": 879820800 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003704112337011033, + "loss": 1.3723, + "theoretical_loss": 3.722900756859909, + "tokens_seen": 879886336 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037040120361083253, + "loss": 1.4218, + "theoretical_loss": 3.722873948147909, + "tokens_seen": 879951872 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037039117352056166, + "loss": 1.4252, + "theoretical_loss": 3.72284714199147, + "tokens_seen": 880017408 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003703811434302909, + "loss": 1.3767, + "theoretical_loss": 3.722820338390158, + "tokens_seen": 880082944 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003703711133400201, + "loss": 1.3613, + "theoretical_loss": 3.7227935373435397, + "tokens_seen": 880148480 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037036108324974926, + "loss": 1.395, + "theoretical_loss": 3.7227667388511807, + "tokens_seen": 880214016 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037035105315947844, + "loss": 1.4109, + "theoretical_loss": 3.7227399429126473, + "tokens_seen": 880279552 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003703410230692076, + "loss": 1.4319, + "theoretical_loss": 3.7227131495275065, + "tokens_seen": 880345088 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003703309929789368, + "loss": 1.284, + "theoretical_loss": 3.7226863586953245, + "tokens_seen": 880410624 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037032096288866604, + "loss": 1.3903, + "theoretical_loss": 3.7226595704156686, + "tokens_seen": 880476160 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037031093279839516, + "loss": 1.3435, + "theoretical_loss": 3.7226327846881055, + "tokens_seen": 880541696 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003703009027081244, + "loss": 1.3435, + "theoretical_loss": 3.722606001512202, + "tokens_seen": 880607232 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003702908726178535, + "loss": 1.4438, + "theoretical_loss": 3.722579220887525, + "tokens_seen": 880672768 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037028084252758276, + "loss": 1.2884, + "theoretical_loss": 3.722552442813641, + "tokens_seen": 880738304 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037027081243731194, + "loss": 1.4502, + "theoretical_loss": 3.7225256672901192, + "tokens_seen": 880803840 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003702607823470411, + "loss": 1.3792, + "theoretical_loss": 3.7224988943165256, + "tokens_seen": 880869376 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003702507522567703, + "loss": 1.3188, + "theoretical_loss": 3.7224721238924277, + "tokens_seen": 880934912 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003702407221664995, + "loss": 1.4425, + "theoretical_loss": 3.722445356017393, + "tokens_seen": 881000448 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037023069207622867, + "loss": 1.4477, + "theoretical_loss": 3.7224185906909897, + "tokens_seen": 881065984 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003702206619859579, + "loss": 1.2998, + "theoretical_loss": 3.7223918279127854, + "tokens_seen": 881131520 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037021063189568703, + "loss": 1.3101, + "theoretical_loss": 3.7223650676823485, + "tokens_seen": 881197056 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037020060180541626, + "loss": 1.4664, + "theoretical_loss": 3.7223383099992455, + "tokens_seen": 881262592 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037019057171514544, + "loss": 1.3484, + "theoretical_loss": 3.7223115548630465, + "tokens_seen": 881328128 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003701805416248746, + "loss": 1.3265, + "theoretical_loss": 3.722284802273318, + "tokens_seen": 881393664 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 297442, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4217342138290405, + "objective/train/theoretical_loss": 3.722258052229629, + "objective/train/tokens_used": 881459200, + "theoretical_loss": 3.722258052229629, + "tokens_seen": 881459200 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003701705115346038, + "loss": 1.3611, + "theoretical_loss": 3.722258052229629, + "tokens_seen": 881459200 + }, + { + "epoch": 0.27, + "learning_rate": 0.000370160481444333, + "loss": 1.4025, + "theoretical_loss": 3.7222313047315483, + "tokens_seen": 881524736 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037015045135406217, + "loss": 1.322, + "theoretical_loss": 3.722204559778644, + "tokens_seen": 881590272 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003701404212637914, + "loss": 1.3865, + "theoretical_loss": 3.7221778173704845, + "tokens_seen": 881655808 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037013039117352053, + "loss": 1.2741, + "theoretical_loss": 3.722151077506639, + "tokens_seen": 881721344 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037012036108324977, + "loss": 1.3535, + "theoretical_loss": 3.722124340186676, + "tokens_seen": 881786880 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037011033099297895, + "loss": 1.3747, + "theoretical_loss": 3.7220976054101653, + "tokens_seen": 881852416 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037010030090270813, + "loss": 1.3286, + "theoretical_loss": 3.7220708731766745, + "tokens_seen": 881917952 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037009027081243736, + "loss": 1.4066, + "theoretical_loss": 3.722044143485774, + "tokens_seen": 881983488 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003700802407221665, + "loss": 1.3555, + "theoretical_loss": 3.7220174163370325, + "tokens_seen": 882049024 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003700702106318957, + "loss": 1.3541, + "theoretical_loss": 3.72199069173002, + "tokens_seen": 882114560 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037006018054162485, + "loss": 1.4831, + "theoretical_loss": 3.721963969664305, + "tokens_seen": 882180096 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003700501504513541, + "loss": 1.4443, + "theoretical_loss": 3.721937250139457, + "tokens_seen": 882245632 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037004012036108327, + "loss": 1.2376, + "theoretical_loss": 3.721910533155047, + "tokens_seen": 882311168 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037003009027081245, + "loss": 1.4253, + "theoretical_loss": 3.721883818710644, + "tokens_seen": 882376704 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037002006018054163, + "loss": 1.3889, + "theoretical_loss": 3.7218571068058175, + "tokens_seen": 882442240 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037001003009027087, + "loss": 1.3076, + "theoretical_loss": 3.7218303974401383, + "tokens_seen": 882507776 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037, + "loss": 1.5055, + "theoretical_loss": 3.7218036906131755, + "tokens_seen": 882573312 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036998996990972923, + "loss": 1.3884, + "theoretical_loss": 3.7217769863245005, + "tokens_seen": 882638848 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036997993981945836, + "loss": 1.4724, + "theoretical_loss": 3.721750284573683, + "tokens_seen": 882704384 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003699699097291876, + "loss": 1.2474, + "theoretical_loss": 3.7217235853602935, + "tokens_seen": 882769920 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003699598796389168, + "loss": 1.3668, + "theoretical_loss": 3.721696888683902, + "tokens_seen": 882835456 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036994984954864595, + "loss": 1.2997, + "theoretical_loss": 3.72167019454408, + "tokens_seen": 882900992 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036993981945837514, + "loss": 1.3428, + "theoretical_loss": 3.7216435029403976, + "tokens_seen": 882966528 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003699297893681043, + "loss": 1.3928, + "theoretical_loss": 3.721616813872426, + "tokens_seen": 883032064 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003699197592778335, + "loss": 1.3164, + "theoretical_loss": 3.721590127339736, + "tokens_seen": 883097600 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036990972918756273, + "loss": 1.3443, + "theoretical_loss": 3.721563443341899, + "tokens_seen": 883163136 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036989969909729186, + "loss": 1.3602, + "theoretical_loss": 3.721536761878485, + "tokens_seen": 883228672 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003698896690070211, + "loss": 1.3616, + "theoretical_loss": 3.7215100829490666, + "tokens_seen": 883294208 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003698796389167503, + "loss": 1.373, + "theoretical_loss": 3.7214834065532147, + "tokens_seen": 883359744 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036986960882647946, + "loss": 1.2463, + "theoretical_loss": 3.7214567326905006, + "tokens_seen": 883425280 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036985957873620864, + "loss": 1.3689, + "theoretical_loss": 3.721430061360496, + "tokens_seen": 883490816 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003698495486459378, + "loss": 1.3492, + "theoretical_loss": 3.721403392562772, + "tokens_seen": 883556352 + }, + { + "epoch": 0.27, + "learning_rate": 0.000369839518555667, + "loss": 1.4123, + "theoretical_loss": 3.7213767262969015, + "tokens_seen": 883621888 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036982948846539624, + "loss": 1.4287, + "theoretical_loss": 3.721350062562456, + "tokens_seen": 883687424 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036981945837512536, + "loss": 1.2646, + "theoretical_loss": 3.7213234013590064, + "tokens_seen": 883752960 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003698094282848546, + "loss": 1.3919, + "theoretical_loss": 3.7212967426861265, + "tokens_seen": 883818496 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003697993981945837, + "loss": 1.3311, + "theoretical_loss": 3.721270086543387, + "tokens_seen": 883884032 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036978936810431296, + "loss": 1.3189, + "theoretical_loss": 3.721243432930361, + "tokens_seen": 883949568 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036977933801404214, + "loss": 1.3994, + "theoretical_loss": 3.721216781846621, + "tokens_seen": 884015104 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003697693079237713, + "loss": 1.3509, + "theoretical_loss": 3.721190133291739, + "tokens_seen": 884080640 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003697592778335005, + "loss": 1.3567, + "theoretical_loss": 3.7211634872652875, + "tokens_seen": 884146176 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003697492477432297, + "loss": 1.3292, + "theoretical_loss": 3.72113684376684, + "tokens_seen": 884211712 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036973921765295887, + "loss": 1.3595, + "theoretical_loss": 3.721110202795969, + "tokens_seen": 884277248 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003697291875626881, + "loss": 1.3843, + "theoretical_loss": 3.721083564352247, + "tokens_seen": 884342784 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036971915747241723, + "loss": 1.3488, + "theoretical_loss": 3.7210569284352473, + "tokens_seen": 884408320 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036970912738214646, + "loss": 1.4554, + "theoretical_loss": 3.7210302950445433, + "tokens_seen": 884473856 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036969909729187565, + "loss": 1.4198, + "theoretical_loss": 3.7210036641797073, + "tokens_seen": 884539392 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003696890672016048, + "loss": 1.3791, + "theoretical_loss": 3.7209770358403143, + "tokens_seen": 884604928 + }, + { + "epoch": 0.27, + "learning_rate": 0.000369679037111334, + "loss": 1.3723, + "theoretical_loss": 3.720950410025936, + "tokens_seen": 884670464 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 298461, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2768340110778809, + "objective/train/theoretical_loss": 3.7209237867361473, + "objective/train/tokens_used": 884736000, + "theoretical_loss": 3.7209237867361473, + "tokens_seen": 884736000 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003696690070210632, + "loss": 1.331, + "theoretical_loss": 3.7209237867361473, + "tokens_seen": 884736000 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036965897693079237, + "loss": 1.3345, + "theoretical_loss": 3.720897165970521, + "tokens_seen": 884801536 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003696489468405216, + "loss": 1.3751, + "theoretical_loss": 3.7208705477286306, + "tokens_seen": 884867072 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036963891675025073, + "loss": 1.3256, + "theoretical_loss": 3.720843932010051, + "tokens_seen": 884932608 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036962888665997997, + "loss": 1.3539, + "theoretical_loss": 3.720817318814355, + "tokens_seen": 884998144 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003696188565697091, + "loss": 1.3134, + "theoretical_loss": 3.720790708141118, + "tokens_seen": 885063680 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036960882647943833, + "loss": 1.266, + "theoretical_loss": 3.7207640999899128, + "tokens_seen": 885129216 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003695987963891675, + "loss": 1.254, + "theoretical_loss": 3.720737494360314, + "tokens_seen": 885194752 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003695887662988967, + "loss": 1.4036, + "theoretical_loss": 3.720710891251897, + "tokens_seen": 885260288 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036957873620862587, + "loss": 1.3091, + "theoretical_loss": 3.7206842906642352, + "tokens_seen": 885325824 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036956870611835505, + "loss": 1.341, + "theoretical_loss": 3.7206576925969035, + "tokens_seen": 885391360 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036955867602808424, + "loss": 1.3824, + "theoretical_loss": 3.720631097049477, + "tokens_seen": 885456896 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036954864593781347, + "loss": 1.2654, + "theoretical_loss": 3.720604504021529, + "tokens_seen": 885522432 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003695386158475426, + "loss": 1.4876, + "theoretical_loss": 3.7205779135126367, + "tokens_seen": 885587968 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036952858575727183, + "loss": 1.3824, + "theoretical_loss": 3.720551325522373, + "tokens_seen": 885653504 + }, + { + "epoch": 0.27, + "learning_rate": 0.000369518555667001, + "loss": 1.3964, + "theoretical_loss": 3.720524740050314, + "tokens_seen": 885719040 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003695085255767302, + "loss": 1.3481, + "theoretical_loss": 3.720498157096035, + "tokens_seen": 885784576 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003694984954864594, + "loss": 1.4037, + "theoretical_loss": 3.7204715766591105, + "tokens_seen": 885850112 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036948846539618856, + "loss": 1.3461, + "theoretical_loss": 3.720444998739117, + "tokens_seen": 885915648 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036947843530591774, + "loss": 1.3417, + "theoretical_loss": 3.720418423335629, + "tokens_seen": 885981184 + }, + { + "epoch": 0.27, + "learning_rate": 0.000369468405215647, + "loss": 1.3157, + "theoretical_loss": 3.7203918504482227, + "tokens_seen": 886046720 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003694583751253761, + "loss": 1.3647, + "theoretical_loss": 3.7203652800764737, + "tokens_seen": 886112256 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036944834503510534, + "loss": 1.4579, + "theoretical_loss": 3.7203387122199576, + "tokens_seen": 886177792 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036943831494483446, + "loss": 1.3878, + "theoretical_loss": 3.7203121468782507, + "tokens_seen": 886243328 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003694282848545637, + "loss": 1.3821, + "theoretical_loss": 3.720285584050928, + "tokens_seen": 886308864 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003694182547642929, + "loss": 1.3202, + "theoretical_loss": 3.7202590237375674, + "tokens_seen": 886374400 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036940822467402206, + "loss": 1.3951, + "theoretical_loss": 3.7202324659377437, + "tokens_seen": 886439936 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036939819458375124, + "loss": 1.2858, + "theoretical_loss": 3.7202059106510337, + "tokens_seen": 886505472 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003693881644934805, + "loss": 1.354, + "theoretical_loss": 3.720179357877014, + "tokens_seen": 886571008 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003693781344032096, + "loss": 1.3472, + "theoretical_loss": 3.7201528076152606, + "tokens_seen": 886636544 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036936810431293884, + "loss": 1.4167, + "theoretical_loss": 3.7201262598653506, + "tokens_seen": 886702080 + }, + { + "epoch": 0.27, + "learning_rate": 0.000369358074222668, + "loss": 1.3342, + "theoretical_loss": 3.720099714626861, + "tokens_seen": 886767616 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003693480441323972, + "loss": 1.4252, + "theoretical_loss": 3.720073171899368, + "tokens_seen": 886833152 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036933801404212644, + "loss": 1.3791, + "theoretical_loss": 3.720046631682448, + "tokens_seen": 886898688 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036932798395185556, + "loss": 1.298, + "theoretical_loss": 3.7200200939756796, + "tokens_seen": 886964224 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003693179538615848, + "loss": 1.3809, + "theoretical_loss": 3.719993558778639, + "tokens_seen": 887029760 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003693079237713139, + "loss": 1.3641, + "theoretical_loss": 3.7199670260909037, + "tokens_seen": 887095296 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036929789368104316, + "loss": 1.4223, + "theoretical_loss": 3.7199404959120512, + "tokens_seen": 887160832 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036928786359077234, + "loss": 1.2882, + "theoretical_loss": 3.7199139682416584, + "tokens_seen": 887226368 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003692778335005015, + "loss": 1.423, + "theoretical_loss": 3.719887443079303, + "tokens_seen": 887291904 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003692678034102307, + "loss": 1.3428, + "theoretical_loss": 3.719860920424563, + "tokens_seen": 887357440 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003692577733199599, + "loss": 1.2977, + "theoretical_loss": 3.7198344002770165, + "tokens_seen": 887422976 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036924774322968907, + "loss": 1.3955, + "theoretical_loss": 3.719807882636241, + "tokens_seen": 887488512 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003692377131394183, + "loss": 1.2757, + "theoretical_loss": 3.719781367501813, + "tokens_seen": 887554048 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036922768304914743, + "loss": 1.3908, + "theoretical_loss": 3.719754854873313, + "tokens_seen": 887619584 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036921765295887666, + "loss": 1.3313, + "theoretical_loss": 3.719728344750318, + "tokens_seen": 887685120 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036920762286860585, + "loss": 1.2556, + "theoretical_loss": 3.7197018371324067, + "tokens_seen": 887750656 + }, + { + "epoch": 0.27, + "learning_rate": 0.000369197592778335, + "loss": 1.3802, + "theoretical_loss": 3.7196753320191567, + "tokens_seen": 887816192 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003691875626880642, + "loss": 1.3438, + "theoretical_loss": 3.7196488294101466, + "tokens_seen": 887881728 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003691775325977934, + "loss": 1.2629, + "theoretical_loss": 3.719622329304956, + "tokens_seen": 887947264 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 299383, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5326988697052002, + "objective/train/theoretical_loss": 3.7195958317031623, + "objective/train/tokens_used": 888012800, + "theoretical_loss": 3.7195958317031623, + "tokens_seen": 888012800 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036916750250752257, + "loss": 1.4723, + "theoretical_loss": 3.7195958317031623, + "tokens_seen": 888012800 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003691574724172518, + "loss": 1.3638, + "theoretical_loss": 3.719569336604345, + "tokens_seen": 888078336 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036914744232698093, + "loss": 1.3696, + "theoretical_loss": 3.7195428440080827, + "tokens_seen": 888143872 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036913741223671017, + "loss": 1.3619, + "theoretical_loss": 3.7195163539139546, + "tokens_seen": 888209408 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003691273821464393, + "loss": 1.381, + "theoretical_loss": 3.71948986632154, + "tokens_seen": 888274944 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036911735205616853, + "loss": 1.3, + "theoretical_loss": 3.719463381230418, + "tokens_seen": 888340480 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003691073219658977, + "loss": 1.366, + "theoretical_loss": 3.7194368986401676, + "tokens_seen": 888406016 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003690972918756269, + "loss": 1.3513, + "theoretical_loss": 3.719410418550368, + "tokens_seen": 888471552 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003690872617853561, + "loss": 1.4009, + "theoretical_loss": 3.7193839409605993, + "tokens_seen": 888537088 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036907723169508525, + "loss": 1.3218, + "theoretical_loss": 3.7193574658704405, + "tokens_seen": 888602624 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036906720160481444, + "loss": 1.2547, + "theoretical_loss": 3.719330993279472, + "tokens_seen": 888668160 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036905717151454367, + "loss": 1.2559, + "theoretical_loss": 3.7193045231872732, + "tokens_seen": 888733696 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003690471414242728, + "loss": 1.3876, + "theoretical_loss": 3.7192780555934237, + "tokens_seen": 888799232 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036903711133400203, + "loss": 1.3301, + "theoretical_loss": 3.719251590497504, + "tokens_seen": 888864768 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003690270812437312, + "loss": 1.4927, + "theoretical_loss": 3.7192251278990938, + "tokens_seen": 888930304 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003690170511534604, + "loss": 1.3462, + "theoretical_loss": 3.7191986677977735, + "tokens_seen": 888995840 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003690070210631896, + "loss": 1.3478, + "theoretical_loss": 3.7191722101931237, + "tokens_seen": 889061376 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036899699097291876, + "loss": 1.3968, + "theoretical_loss": 3.7191457550847247, + "tokens_seen": 889126912 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036898696088264794, + "loss": 1.4129, + "theoretical_loss": 3.7191193024721567, + "tokens_seen": 889192448 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003689769307923772, + "loss": 1.412, + "theoretical_loss": 3.719092852355, + "tokens_seen": 889257984 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003689669007021063, + "loss": 1.2673, + "theoretical_loss": 3.7190664047328363, + "tokens_seen": 889323520 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036895687061183554, + "loss": 1.3999, + "theoretical_loss": 3.719039959605246, + "tokens_seen": 889389056 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036894684052156466, + "loss": 1.3546, + "theoretical_loss": 3.7190135169718097, + "tokens_seen": 889454592 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003689368104312939, + "loss": 1.3713, + "theoretical_loss": 3.718987076832109, + "tokens_seen": 889520128 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003689267803410231, + "loss": 1.324, + "theoretical_loss": 3.718960639185724, + "tokens_seen": 889585664 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036891675025075226, + "loss": 1.3034, + "theoretical_loss": 3.7189342040322373, + "tokens_seen": 889651200 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036890672016048144, + "loss": 1.2935, + "theoretical_loss": 3.718907771371229, + "tokens_seen": 889716736 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003688966900702107, + "loss": 1.3746, + "theoretical_loss": 3.718881341202281, + "tokens_seen": 889782272 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003688866599799398, + "loss": 1.5201, + "theoretical_loss": 3.7188549135249755, + "tokens_seen": 889847808 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036887662988966904, + "loss": 1.2731, + "theoretical_loss": 3.718828488338893, + "tokens_seen": 889913344 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036886659979939817, + "loss": 1.3814, + "theoretical_loss": 3.718802065643616, + "tokens_seen": 889978880 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003688565697091274, + "loss": 1.2584, + "theoretical_loss": 3.7187756454387255, + "tokens_seen": 890044416 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003688465396188566, + "loss": 1.2684, + "theoretical_loss": 3.7187492277238046, + "tokens_seen": 890109952 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036883650952858576, + "loss": 1.319, + "theoretical_loss": 3.7187228124984344, + "tokens_seen": 890175488 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036882647943831494, + "loss": 1.3584, + "theoretical_loss": 3.7186963997621976, + "tokens_seen": 890241024 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003688164493480441, + "loss": 1.4131, + "theoretical_loss": 3.7186699895146758, + "tokens_seen": 890306560 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003688064192577733, + "loss": 1.3216, + "theoretical_loss": 3.718643581755452, + "tokens_seen": 890372096 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036879638916750254, + "loss": 1.4197, + "theoretical_loss": 3.7186171764841083, + "tokens_seen": 890437632 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036878635907723167, + "loss": 1.3345, + "theoretical_loss": 3.7185907737002273, + "tokens_seen": 890503168 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003687763289869609, + "loss": 1.2441, + "theoretical_loss": 3.7185643734033915, + "tokens_seen": 890568704 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036876629889669003, + "loss": 1.3206, + "theoretical_loss": 3.7185379755931836, + "tokens_seen": 890634240 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036875626880641927, + "loss": 1.3137, + "theoretical_loss": 3.718511580269187, + "tokens_seen": 890699776 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036874623871614845, + "loss": 1.2806, + "theoretical_loss": 3.718485187430984, + "tokens_seen": 890765312 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036873620862587763, + "loss": 1.3552, + "theoretical_loss": 3.7184587970781573, + "tokens_seen": 890830848 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003687261785356068, + "loss": 1.3736, + "theoretical_loss": 3.7184324092102914, + "tokens_seen": 890896384 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036871614844533605, + "loss": 1.4911, + "theoretical_loss": 3.7184060238269683, + "tokens_seen": 890961920 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036870611835506517, + "loss": 1.216, + "theoretical_loss": 3.718379640927772, + "tokens_seen": 891027456 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003686960882647944, + "loss": 1.3235, + "theoretical_loss": 3.718353260512286, + "tokens_seen": 891092992 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036868605817452353, + "loss": 1.3931, + "theoretical_loss": 3.718326882580093, + "tokens_seen": 891158528 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036867602808425277, + "loss": 1.2824, + "theoretical_loss": 3.7183005071307775, + "tokens_seen": 891224064 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 300545, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4590158462524414, + "objective/train/theoretical_loss": 3.7182741341639227, + "objective/train/tokens_used": 891289600, + "theoretical_loss": 3.7182741341639227, + "tokens_seen": 891289600 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036866599799398195, + "loss": 1.394, + "theoretical_loss": 3.7182741341639227, + "tokens_seen": 891289600 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036865596790371113, + "loss": 1.4006, + "theoretical_loss": 3.7182477636791127, + "tokens_seen": 891355136 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003686459378134403, + "loss": 1.434, + "theoretical_loss": 3.718221395675932, + "tokens_seen": 891420672 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003686359077231695, + "loss": 1.3337, + "theoretical_loss": 3.7181950301539635, + "tokens_seen": 891486208 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003686258776328987, + "loss": 1.2516, + "theoretical_loss": 3.718168667112792, + "tokens_seen": 891551744 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003686158475426279, + "loss": 1.3189, + "theoretical_loss": 3.718142306552002, + "tokens_seen": 891617280 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003686058174523571, + "loss": 1.2793, + "theoretical_loss": 3.718115948471177, + "tokens_seen": 891682816 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003685957873620863, + "loss": 1.2685, + "theoretical_loss": 3.718089592869902, + "tokens_seen": 891748352 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036858575727181545, + "loss": 1.3728, + "theoretical_loss": 3.718063239747762, + "tokens_seen": 891813888 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036857572718154464, + "loss": 1.3857, + "theoretical_loss": 3.718036889104341, + "tokens_seen": 891879424 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036856569709127387, + "loss": 1.3426, + "theoretical_loss": 3.7180105409392237, + "tokens_seen": 891944960 + }, + { + "epoch": 0.27, + "learning_rate": 0.000368555667001003, + "loss": 1.3626, + "theoretical_loss": 3.7179841952519945, + "tokens_seen": 892010496 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036854563691073223, + "loss": 1.4097, + "theoretical_loss": 3.7179578520422396, + "tokens_seen": 892076032 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003685356068204614, + "loss": 1.2725, + "theoretical_loss": 3.7179315113095437, + "tokens_seen": 892141568 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003685255767301906, + "loss": 1.422, + "theoretical_loss": 3.7179051730534916, + "tokens_seen": 892207104 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003685155466399198, + "loss": 1.3163, + "theoretical_loss": 3.717878837273668, + "tokens_seen": 892272640 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036850551654964896, + "loss": 1.4173, + "theoretical_loss": 3.7178525039696595, + "tokens_seen": 892338176 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036849548645937814, + "loss": 1.426, + "theoretical_loss": 3.7178261731410505, + "tokens_seen": 892403712 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003684854563691074, + "loss": 1.3082, + "theoretical_loss": 3.717799844787427, + "tokens_seen": 892469248 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003684754262788365, + "loss": 1.3156, + "theoretical_loss": 3.7177735189083743, + "tokens_seen": 892534784 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036846539618856574, + "loss": 1.2848, + "theoretical_loss": 3.717747195503479, + "tokens_seen": 892600320 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036845536609829486, + "loss": 1.2654, + "theoretical_loss": 3.7177208745723256, + "tokens_seen": 892665856 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003684453360080241, + "loss": 1.2893, + "theoretical_loss": 3.717694556114502, + "tokens_seen": 892731392 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003684353059177533, + "loss": 1.3231, + "theoretical_loss": 3.717668240129592, + "tokens_seen": 892796928 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036842527582748246, + "loss": 1.2677, + "theoretical_loss": 3.717641926617183, + "tokens_seen": 892862464 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036841524573721164, + "loss": 1.4403, + "theoretical_loss": 3.717615615576861, + "tokens_seen": 892928000 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003684052156469409, + "loss": 1.4058, + "theoretical_loss": 3.717589307008213, + "tokens_seen": 892993536 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036839518555667, + "loss": 1.3237, + "theoretical_loss": 3.717563000910824, + "tokens_seen": 893059072 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036838515546639924, + "loss": 1.2576, + "theoretical_loss": 3.717536697284282, + "tokens_seen": 893124608 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036837512537612837, + "loss": 1.3819, + "theoretical_loss": 3.717510396128173, + "tokens_seen": 893190144 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003683650952858576, + "loss": 1.3231, + "theoretical_loss": 3.7174840974420826, + "tokens_seen": 893255680 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003683550651955868, + "loss": 1.3451, + "theoretical_loss": 3.7174578012256, + "tokens_seen": 893321216 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036834503510531596, + "loss": 1.2952, + "theoretical_loss": 3.7174315074783104, + "tokens_seen": 893386752 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036833500501504514, + "loss": 1.3535, + "theoretical_loss": 3.7174052161998015, + "tokens_seen": 893452288 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003683249749247743, + "loss": 1.3916, + "theoretical_loss": 3.7173789273896602, + "tokens_seen": 893517824 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003683149448345035, + "loss": 1.2589, + "theoretical_loss": 3.717352641047474, + "tokens_seen": 893583360 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036830491474423274, + "loss": 1.2186, + "theoretical_loss": 3.7173263571728303, + "tokens_seen": 893648896 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036829488465396187, + "loss": 1.3182, + "theoretical_loss": 3.7173000757653156, + "tokens_seen": 893714432 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003682848545636911, + "loss": 1.348, + "theoretical_loss": 3.717273796824519, + "tokens_seen": 893779968 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036827482447342023, + "loss": 1.3433, + "theoretical_loss": 3.7172475203500266, + "tokens_seen": 893845504 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036826479438314947, + "loss": 1.3297, + "theoretical_loss": 3.717221246341427, + "tokens_seen": 893911040 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036825476429287865, + "loss": 1.363, + "theoretical_loss": 3.717194974798308, + "tokens_seen": 893976576 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036824473420260783, + "loss": 1.3529, + "theoretical_loss": 3.7171687057202574, + "tokens_seen": 894042112 + }, + { + "epoch": 0.27, + "learning_rate": 0.000368234704112337, + "loss": 1.4381, + "theoretical_loss": 3.717142439106863, + "tokens_seen": 894107648 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036822467402206625, + "loss": 1.3378, + "theoretical_loss": 3.717116174957713, + "tokens_seen": 894173184 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036821464393179537, + "loss": 1.438, + "theoretical_loss": 3.717089913272396, + "tokens_seen": 894238720 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003682046138415246, + "loss": 1.3016, + "theoretical_loss": 3.7170636540504995, + "tokens_seen": 894304256 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036819458375125373, + "loss": 1.3089, + "theoretical_loss": 3.7170373972916133, + "tokens_seen": 894369792 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036818455366098297, + "loss": 1.2246, + "theoretical_loss": 3.7170111429953248, + "tokens_seen": 894435328 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036817452357071215, + "loss": 1.3331, + "theoretical_loss": 3.716984891161223, + "tokens_seen": 894500864 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 301446, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3855623006820679, + "objective/train/theoretical_loss": 3.7169586417888967, + "objective/train/tokens_used": 894566400, + "theoretical_loss": 3.7169586417888967, + "tokens_seen": 894566400 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036816449348044133, + "loss": 1.3355, + "theoretical_loss": 3.7169586417888967, + "tokens_seen": 894566400 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003681544633901705, + "loss": 1.2818, + "theoretical_loss": 3.7169323948779343, + "tokens_seen": 894631936 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003681444332998997, + "loss": 1.3416, + "theoretical_loss": 3.716906150427925, + "tokens_seen": 894697472 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003681344032096289, + "loss": 1.3195, + "theoretical_loss": 3.7168799084384583, + "tokens_seen": 894763008 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003681243731193581, + "loss": 1.3869, + "theoretical_loss": 3.7168536689091223, + "tokens_seen": 894828544 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036811434302908724, + "loss": 1.3884, + "theoretical_loss": 3.716827431839507, + "tokens_seen": 894894080 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003681043129388165, + "loss": 1.3672, + "theoretical_loss": 3.7168011972292017, + "tokens_seen": 894959616 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003680942828485456, + "loss": 1.2736, + "theoretical_loss": 3.7167749650777955, + "tokens_seen": 895025152 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036808425275827484, + "loss": 1.3581, + "theoretical_loss": 3.7167487353848774, + "tokens_seen": 895090688 + }, + { + "epoch": 0.27, + "learning_rate": 0.000368074222668004, + "loss": 1.2095, + "theoretical_loss": 3.7167225081500384, + "tokens_seen": 895156224 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003680641925777332, + "loss": 1.3957, + "theoretical_loss": 3.716696283372867, + "tokens_seen": 895221760 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003680541624874624, + "loss": 1.4421, + "theoretical_loss": 3.716670061052954, + "tokens_seen": 895287296 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003680441323971916, + "loss": 1.4229, + "theoretical_loss": 3.7166438411898883, + "tokens_seen": 895352832 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036803410230692074, + "loss": 1.3365, + "theoretical_loss": 3.7166176237832604, + "tokens_seen": 895418368 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036802407221665, + "loss": 1.351, + "theoretical_loss": 3.7165914088326604, + "tokens_seen": 895483904 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003680140421263791, + "loss": 1.2445, + "theoretical_loss": 3.7165651963376787, + "tokens_seen": 895549440 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036800401203610834, + "loss": 1.3526, + "theoretical_loss": 3.716538986297905, + "tokens_seen": 895614976 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003679939819458375, + "loss": 1.3865, + "theoretical_loss": 3.7165127787129304, + "tokens_seen": 895680512 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003679839518555667, + "loss": 1.3052, + "theoretical_loss": 3.716486573582345, + "tokens_seen": 895746048 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003679739217652959, + "loss": 1.3027, + "theoretical_loss": 3.7164603709057387, + "tokens_seen": 895811584 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036796389167502506, + "loss": 1.3736, + "theoretical_loss": 3.716434170682704, + "tokens_seen": 895877120 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036795386158475424, + "loss": 1.3572, + "theoretical_loss": 3.7164079729128296, + "tokens_seen": 895942656 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003679438314944835, + "loss": 1.3085, + "theoretical_loss": 3.7163817775957084, + "tokens_seen": 896008192 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003679338014042126, + "loss": 1.3308, + "theoretical_loss": 3.71635558473093, + "tokens_seen": 896073728 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036792377131394184, + "loss": 1.337, + "theoretical_loss": 3.7163293943180857, + "tokens_seen": 896139264 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036791374122367097, + "loss": 1.4096, + "theoretical_loss": 3.7163032063567676, + "tokens_seen": 896204800 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003679037111334002, + "loss": 1.2782, + "theoretical_loss": 3.7162770208465656, + "tokens_seen": 896270336 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003678936810431294, + "loss": 1.3538, + "theoretical_loss": 3.716250837787072, + "tokens_seen": 896335872 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036788365095285857, + "loss": 1.3805, + "theoretical_loss": 3.716224657177878, + "tokens_seen": 896401408 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036787362086258775, + "loss": 1.2846, + "theoretical_loss": 3.716198479018575, + "tokens_seen": 896466944 + }, + { + "epoch": 0.27, + "learning_rate": 0.000367863590772317, + "loss": 1.2438, + "theoretical_loss": 3.7161723033087553, + "tokens_seen": 896532480 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036785356068204616, + "loss": 1.4527, + "theoretical_loss": 3.71614613004801, + "tokens_seen": 896598016 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036784353059177535, + "loss": 1.226, + "theoretical_loss": 3.716119959235931, + "tokens_seen": 896663552 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003678335005015045, + "loss": 1.3983, + "theoretical_loss": 3.716093790872111, + "tokens_seen": 896729088 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003678234704112337, + "loss": 1.4205, + "theoretical_loss": 3.716067624956141, + "tokens_seen": 896794624 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036781344032096294, + "loss": 1.3628, + "theoretical_loss": 3.716041461487614, + "tokens_seen": 896860160 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036780341023069207, + "loss": 1.4182, + "theoretical_loss": 3.716015300466122, + "tokens_seen": 896925696 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003677933801404213, + "loss": 1.3021, + "theoretical_loss": 3.7159891418912574, + "tokens_seen": 896991232 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036778335005015043, + "loss": 1.3268, + "theoretical_loss": 3.715962985762612, + "tokens_seen": 897056768 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036777331995987967, + "loss": 1.31, + "theoretical_loss": 3.7159368320797794, + "tokens_seen": 897122304 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036776328986960885, + "loss": 1.233, + "theoretical_loss": 3.7159106808423523, + "tokens_seen": 897187840 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036775325977933803, + "loss": 1.3729, + "theoretical_loss": 3.7158845320499223, + "tokens_seen": 897253376 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003677432296890672, + "loss": 1.3183, + "theoretical_loss": 3.7158583857020826, + "tokens_seen": 897318912 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036773319959879645, + "loss": 1.2815, + "theoretical_loss": 3.7158322417984273, + "tokens_seen": 897384448 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036772316950852557, + "loss": 1.2387, + "theoretical_loss": 3.715806100338548, + "tokens_seen": 897449984 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003677131394182548, + "loss": 1.299, + "theoretical_loss": 3.7157799613220384, + "tokens_seen": 897515520 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036770310932798393, + "loss": 1.2786, + "theoretical_loss": 3.715753824748492, + "tokens_seen": 897581056 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036769307923771317, + "loss": 1.4176, + "theoretical_loss": 3.715727690617502, + "tokens_seen": 897646592 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036768304914744235, + "loss": 1.3602, + "theoretical_loss": 3.7157015589286617, + "tokens_seen": 897712128 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036767301905717153, + "loss": 1.304, + "theoretical_loss": 3.715675429681564, + "tokens_seen": 897777664 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 302678, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3054497241973877, + "objective/train/theoretical_loss": 3.7156493028758035, + "objective/train/tokens_used": 897843200, + "theoretical_loss": 3.7156493028758035, + "tokens_seen": 897843200 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003676629889669007, + "loss": 1.3236, + "theoretical_loss": 3.7156493028758035, + "tokens_seen": 897843200 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003676529588766299, + "loss": 1.3057, + "theoretical_loss": 3.715623178510974, + "tokens_seen": 897908736 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003676429287863591, + "loss": 1.3351, + "theoretical_loss": 3.7155970565866685, + "tokens_seen": 897974272 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003676328986960883, + "loss": 1.4161, + "theoretical_loss": 3.7155709371024814, + "tokens_seen": 898039808 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036762286860581744, + "loss": 1.3355, + "theoretical_loss": 3.715544820058007, + "tokens_seen": 898105344 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003676128385155467, + "loss": 1.2959, + "theoretical_loss": 3.7155187054528387, + "tokens_seen": 898170880 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003676028084252758, + "loss": 1.4059, + "theoretical_loss": 3.715492593286571, + "tokens_seen": 898236416 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036759277833500504, + "loss": 1.3179, + "theoretical_loss": 3.715466483558799, + "tokens_seen": 898301952 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003675827482447342, + "loss": 1.4266, + "theoretical_loss": 3.7154403762691155, + "tokens_seen": 898367488 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003675727181544634, + "loss": 1.262, + "theoretical_loss": 3.7154142714171168, + "tokens_seen": 898433024 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003675626880641926, + "loss": 1.2862, + "theoretical_loss": 3.715388169002396, + "tokens_seen": 898498560 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003675526579739218, + "loss": 1.3784, + "theoretical_loss": 3.715362069024548, + "tokens_seen": 898564096 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036754262788365094, + "loss": 1.3645, + "theoretical_loss": 3.7153359714831686, + "tokens_seen": 898629632 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003675325977933802, + "loss": 1.3015, + "theoretical_loss": 3.7153098763778516, + "tokens_seen": 898695168 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003675225677031093, + "loss": 1.3518, + "theoretical_loss": 3.715283783708193, + "tokens_seen": 898760704 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036751253761283854, + "loss": 1.2989, + "theoretical_loss": 3.715257693473787, + "tokens_seen": 898826240 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003675025075225677, + "loss": 1.2066, + "theoretical_loss": 3.715231605674229, + "tokens_seen": 898891776 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003674924774322969, + "loss": 1.1953, + "theoretical_loss": 3.7152055203091146, + "tokens_seen": 898957312 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003674824473420261, + "loss": 1.2654, + "theoretical_loss": 3.7151794373780387, + "tokens_seen": 899022848 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036747241725175526, + "loss": 1.2747, + "theoretical_loss": 3.715153356880597, + "tokens_seen": 899088384 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036746238716148444, + "loss": 1.3633, + "theoretical_loss": 3.715127278816386, + "tokens_seen": 899153920 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003674523570712137, + "loss": 1.4664, + "theoretical_loss": 3.7151012031849993, + "tokens_seen": 899219456 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003674423269809428, + "loss": 1.3411, + "theoretical_loss": 3.715075129986034, + "tokens_seen": 899284992 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036743229689067204, + "loss": 1.2678, + "theoretical_loss": 3.715049059219086, + "tokens_seen": 899350528 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036742226680040117, + "loss": 1.3776, + "theoretical_loss": 3.7150229908837513, + "tokens_seen": 899416064 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003674122367101304, + "loss": 1.3256, + "theoretical_loss": 3.7149969249796255, + "tokens_seen": 899481600 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003674022066198596, + "loss": 1.4269, + "theoretical_loss": 3.7149708615063046, + "tokens_seen": 899547136 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036739217652958877, + "loss": 1.2176, + "theoretical_loss": 3.7149448004633854, + "tokens_seen": 899612672 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036738214643931795, + "loss": 1.3353, + "theoretical_loss": 3.714918741850464, + "tokens_seen": 899678208 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003673721163490472, + "loss": 1.4133, + "theoretical_loss": 3.7148926856671367, + "tokens_seen": 899743744 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003673620862587763, + "loss": 1.4018, + "theoretical_loss": 3.7148666319130004, + "tokens_seen": 899809280 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036735205616850555, + "loss": 1.3803, + "theoretical_loss": 3.7148405805876514, + "tokens_seen": 899874816 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036734202607823467, + "loss": 1.2277, + "theoretical_loss": 3.7148145316906866, + "tokens_seen": 899940352 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003673319959879639, + "loss": 1.2791, + "theoretical_loss": 3.7147884852217024, + "tokens_seen": 900005888 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003673219658976931, + "loss": 1.2794, + "theoretical_loss": 3.714762441180296, + "tokens_seen": 900071424 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036731193580742227, + "loss": 1.3001, + "theoretical_loss": 3.714736399566065, + "tokens_seen": 900136960 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036730190571715145, + "loss": 1.289, + "theoretical_loss": 3.7147103603786054, + "tokens_seen": 900202496 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036729187562688063, + "loss": 1.3708, + "theoretical_loss": 3.7146843236175155, + "tokens_seen": 900268032 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003672818455366098, + "loss": 1.2774, + "theoretical_loss": 3.714658289282392, + "tokens_seen": 900333568 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036727181544633905, + "loss": 1.3457, + "theoretical_loss": 3.7146322573728323, + "tokens_seen": 900399104 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003672617853560682, + "loss": 1.3652, + "theoretical_loss": 3.7146062278884338, + "tokens_seen": 900464640 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003672517552657974, + "loss": 1.425, + "theoretical_loss": 3.7145802008287943, + "tokens_seen": 900530176 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036724172517552654, + "loss": 1.3879, + "theoretical_loss": 3.7145541761935115, + "tokens_seen": 900595712 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036723169508525577, + "loss": 1.2909, + "theoretical_loss": 3.714528153982183, + "tokens_seen": 900661248 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036722166499498495, + "loss": 1.2536, + "theoretical_loss": 3.714502134194407, + "tokens_seen": 900726784 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036721163490471414, + "loss": 1.3175, + "theoretical_loss": 3.7144761168297817, + "tokens_seen": 900792320 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003672016048144433, + "loss": 1.2993, + "theoretical_loss": 3.714450101887904, + "tokens_seen": 900857856 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036719157472417255, + "loss": 1.4294, + "theoretical_loss": 3.7144240893683733, + "tokens_seen": 900923392 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003671815446339017, + "loss": 1.2991, + "theoretical_loss": 3.7143980792707874, + "tokens_seen": 900988928 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003671715145436309, + "loss": 1.3118, + "theoretical_loss": 3.7143720715947444, + "tokens_seen": 901054464 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 303654, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3526173830032349, + "objective/train/theoretical_loss": 3.7143460663398433, + "objective/train/tokens_used": 901120000, + "theoretical_loss": 3.7143460663398433, + "tokens_seen": 901120000 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036716148445336004, + "loss": 1.3314, + "theoretical_loss": 3.7143460663398433, + "tokens_seen": 901120000 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003671514543630893, + "loss": 1.3745, + "theoretical_loss": 3.714320063505682, + "tokens_seen": 901185536 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036714142427281846, + "loss": 1.3114, + "theoretical_loss": 3.7142940630918595, + "tokens_seen": 901251072 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036713139418254764, + "loss": 1.3421, + "theoretical_loss": 3.7142680650979747, + "tokens_seen": 901316608 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003671213640922768, + "loss": 1.4311, + "theoretical_loss": 3.7142420695236265, + "tokens_seen": 901382144 + }, + { + "epoch": 0.27, + "learning_rate": 0.000367111334002006, + "loss": 1.3936, + "theoretical_loss": 3.7142160763684133, + "tokens_seen": 901447680 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036710130391173524, + "loss": 1.2344, + "theoretical_loss": 3.7141900856319343, + "tokens_seen": 901513216 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003670912738214644, + "loss": 1.3443, + "theoretical_loss": 3.714164097313789, + "tokens_seen": 901578752 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003670812437311936, + "loss": 1.2923, + "theoretical_loss": 3.7141381114135763, + "tokens_seen": 901644288 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003670712136409228, + "loss": 1.4261, + "theoretical_loss": 3.7141121279308957, + "tokens_seen": 901709824 + }, + { + "epoch": 0.27, + "learning_rate": 0.000367061183550652, + "loss": 1.4919, + "theoretical_loss": 3.7140861468653465, + "tokens_seen": 901775360 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036705115346038114, + "loss": 1.3889, + "theoretical_loss": 3.7140601682165277, + "tokens_seen": 901840896 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003670411233701104, + "loss": 1.4158, + "theoretical_loss": 3.71403419198404, + "tokens_seen": 901906432 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003670310932798395, + "loss": 1.3284, + "theoretical_loss": 3.714008218167482, + "tokens_seen": 901971968 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036702106318956874, + "loss": 1.3381, + "theoretical_loss": 3.7139822467664545, + "tokens_seen": 902037504 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003670110330992979, + "loss": 1.4209, + "theoretical_loss": 3.713956277780557, + "tokens_seen": 902103040 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003670010030090271, + "loss": 1.3028, + "theoretical_loss": 3.7139303112093884, + "tokens_seen": 902168576 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003669909729187563, + "loss": 1.2783, + "theoretical_loss": 3.7139043470525506, + "tokens_seen": 902234112 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036698094282848546, + "loss": 1.3434, + "theoretical_loss": 3.713878385309643, + "tokens_seen": 902299648 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036697091273821464, + "loss": 1.3705, + "theoretical_loss": 3.7138524259802654, + "tokens_seen": 902365184 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003669608826479439, + "loss": 1.3299, + "theoretical_loss": 3.7138264690640184, + "tokens_seen": 902430720 + }, + { + "epoch": 0.27, + "learning_rate": 0.000366950852557673, + "loss": 1.4204, + "theoretical_loss": 3.7138005145605026, + "tokens_seen": 902496256 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036694082246740224, + "loss": 1.4341, + "theoretical_loss": 3.713774562469319, + "tokens_seen": 902561792 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036693079237713137, + "loss": 1.302, + "theoretical_loss": 3.7137486127900674, + "tokens_seen": 902627328 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003669207622868606, + "loss": 1.274, + "theoretical_loss": 3.713722665522349, + "tokens_seen": 902692864 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003669107321965898, + "loss": 1.3553, + "theoretical_loss": 3.713696720665765, + "tokens_seen": 902758400 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036690070210631897, + "loss": 1.4612, + "theoretical_loss": 3.7136707782199156, + "tokens_seen": 902823936 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036689067201604815, + "loss": 1.3859, + "theoretical_loss": 3.7136448381844023, + "tokens_seen": 902889472 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003668806419257774, + "loss": 1.3355, + "theoretical_loss": 3.713618900558826, + "tokens_seen": 902955008 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003668706118355065, + "loss": 1.3356, + "theoretical_loss": 3.7135929653427877, + "tokens_seen": 903020544 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036686058174523575, + "loss": 1.3757, + "theoretical_loss": 3.7135670325358894, + "tokens_seen": 903086080 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036685055165496487, + "loss": 1.3554, + "theoretical_loss": 3.7135411021377323, + "tokens_seen": 903151616 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003668405215646941, + "loss": 1.3366, + "theoretical_loss": 3.713515174147917, + "tokens_seen": 903217152 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003668304914744233, + "loss": 1.3815, + "theoretical_loss": 3.7134892485660465, + "tokens_seen": 903282688 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036682046138415247, + "loss": 1.322, + "theoretical_loss": 3.7134633253917215, + "tokens_seen": 903348224 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036681043129388165, + "loss": 1.3695, + "theoretical_loss": 3.713437404624544, + "tokens_seen": 903413760 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036680040120361083, + "loss": 1.3326, + "theoretical_loss": 3.7134114862641163, + "tokens_seen": 903479296 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036679037111334, + "loss": 1.3797, + "theoretical_loss": 3.71338557031004, + "tokens_seen": 903544832 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036678034102306925, + "loss": 1.182, + "theoretical_loss": 3.713359656761917, + "tokens_seen": 903610368 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003667703109327984, + "loss": 1.4023, + "theoretical_loss": 3.71333374561935, + "tokens_seen": 903675904 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003667602808425276, + "loss": 1.3143, + "theoretical_loss": 3.713307836881941, + "tokens_seen": 903741440 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036675025075225674, + "loss": 1.2833, + "theoretical_loss": 3.713281930549292, + "tokens_seen": 903806976 + }, + { + "epoch": 0.27, + "learning_rate": 0.000366740220661986, + "loss": 1.2875, + "theoretical_loss": 3.7132560266210057, + "tokens_seen": 903872512 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036673019057171515, + "loss": 1.3209, + "theoretical_loss": 3.713230125096685, + "tokens_seen": 903938048 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036672016048144434, + "loss": 1.5184, + "theoretical_loss": 3.7132042259759324, + "tokens_seen": 904003584 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003667101303911735, + "loss": 1.3098, + "theoretical_loss": 3.71317832925835, + "tokens_seen": 904069120 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036670010030090275, + "loss": 1.3763, + "theoretical_loss": 3.7131524349435416, + "tokens_seen": 904134656 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003666900702106319, + "loss": 1.2891, + "theoretical_loss": 3.7131265430311093, + "tokens_seen": 904200192 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003666800401203611, + "loss": 1.2601, + "theoretical_loss": 3.713100653520657, + "tokens_seen": 904265728 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036667001003009024, + "loss": 1.3743, + "theoretical_loss": 3.7130747664117862, + "tokens_seen": 904331264 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 304879, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3840272426605225, + "objective/train/theoretical_loss": 3.7130488817041023, + "objective/train/tokens_used": 904396800, + "theoretical_loss": 3.7130488817041023, + "tokens_seen": 904396800 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003666599799398195, + "loss": 1.3983, + "theoretical_loss": 3.7130488817041023, + "tokens_seen": 904396800 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036664994984954866, + "loss": 1.3302, + "theoretical_loss": 3.7130229993972073, + "tokens_seen": 904462336 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036663991975927784, + "loss": 1.286, + "theoretical_loss": 3.712997119490704, + "tokens_seen": 904527872 + }, + { + "epoch": 0.27, + "learning_rate": 0.000366629889669007, + "loss": 1.3346, + "theoretical_loss": 3.7129712419841976, + "tokens_seen": 904593408 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003666198595787362, + "loss": 1.3909, + "theoretical_loss": 3.7129453668772903, + "tokens_seen": 904658944 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003666098294884654, + "loss": 1.3994, + "theoretical_loss": 3.7129194941695864, + "tokens_seen": 904724480 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003665997993981946, + "loss": 1.2843, + "theoretical_loss": 3.7128936238606896, + "tokens_seen": 904790016 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036658976930792374, + "loss": 1.3786, + "theoretical_loss": 3.712867755950203, + "tokens_seen": 904855552 + }, + { + "epoch": 0.27, + "learning_rate": 0.000366579739217653, + "loss": 1.2923, + "theoretical_loss": 3.7128418904377316, + "tokens_seen": 904921088 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036656970912738216, + "loss": 1.3767, + "theoretical_loss": 3.712816027322879, + "tokens_seen": 904986624 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036655967903711134, + "loss": 1.4439, + "theoretical_loss": 3.71279016660525, + "tokens_seen": 905052160 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003665496489468405, + "loss": 1.2956, + "theoretical_loss": 3.712764308284448, + "tokens_seen": 905117696 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003665396188565697, + "loss": 1.3395, + "theoretical_loss": 3.7127384523600773, + "tokens_seen": 905183232 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003665295887662989, + "loss": 1.4127, + "theoretical_loss": 3.712712598831743, + "tokens_seen": 905248768 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003665195586760281, + "loss": 1.4602, + "theoretical_loss": 3.712686747699049, + "tokens_seen": 905314304 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036650952858575725, + "loss": 1.3882, + "theoretical_loss": 3.7126608989616, + "tokens_seen": 905379840 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003664994984954865, + "loss": 1.3528, + "theoretical_loss": 3.7126350526190013, + "tokens_seen": 905445376 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003664894684052156, + "loss": 1.3391, + "theoretical_loss": 3.712609208670857, + "tokens_seen": 905510912 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036647943831494484, + "loss": 1.3609, + "theoretical_loss": 3.712583367116773, + "tokens_seen": 905576448 + }, + { + "epoch": 0.27, + "learning_rate": 0.000366469408224674, + "loss": 1.2268, + "theoretical_loss": 3.7125575279563527, + "tokens_seen": 905641984 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003664593781344032, + "loss": 1.3936, + "theoretical_loss": 3.7125316911892026, + "tokens_seen": 905707520 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003664493480441324, + "loss": 1.4506, + "theoretical_loss": 3.7125058568149267, + "tokens_seen": 905773056 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036643931795386157, + "loss": 1.3831, + "theoretical_loss": 3.7124800248331318, + "tokens_seen": 905838592 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036642928786359075, + "loss": 1.3823, + "theoretical_loss": 3.712454195243422, + "tokens_seen": 905904128 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036641925777332, + "loss": 1.2919, + "theoretical_loss": 3.712428368045403, + "tokens_seen": 905969664 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003664092276830491, + "loss": 1.3016, + "theoretical_loss": 3.7124025432386807, + "tokens_seen": 906035200 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036639919759277835, + "loss": 1.4423, + "theoretical_loss": 3.712376720822861, + "tokens_seen": 906100736 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036638916750250753, + "loss": 1.3142, + "theoretical_loss": 3.7123509007975484, + "tokens_seen": 906166272 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003663791374122367, + "loss": 1.3662, + "theoretical_loss": 3.7123250831623498, + "tokens_seen": 906231808 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003663691073219659, + "loss": 1.324, + "theoretical_loss": 3.712299267916871, + "tokens_seen": 906297344 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036635907723169507, + "loss": 1.4988, + "theoretical_loss": 3.7122734550607177, + "tokens_seen": 906362880 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003663490471414243, + "loss": 1.4049, + "theoretical_loss": 3.712247644593497, + "tokens_seen": 906428416 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003663390170511535, + "loss": 1.2701, + "theoretical_loss": 3.7122218365148134, + "tokens_seen": 906493952 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036632898696088267, + "loss": 1.3346, + "theoretical_loss": 3.7121960308242743, + "tokens_seen": 906559488 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036631895687061185, + "loss": 1.2989, + "theoretical_loss": 3.7121702275214865, + "tokens_seen": 906625024 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036630892678034103, + "loss": 1.3517, + "theoretical_loss": 3.7121444266060553, + "tokens_seen": 906690560 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003662988966900702, + "loss": 1.3527, + "theoretical_loss": 3.712118628077588, + "tokens_seen": 906756096 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036628886659979945, + "loss": 1.4098, + "theoretical_loss": 3.712092831935691, + "tokens_seen": 906821632 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003662788365095286, + "loss": 1.3143, + "theoretical_loss": 3.712067038179972, + "tokens_seen": 906887168 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003662688064192578, + "loss": 1.4955, + "theoretical_loss": 3.7120412468100366, + "tokens_seen": 906952704 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036625877632898694, + "loss": 1.3699, + "theoretical_loss": 3.7120154578254922, + "tokens_seen": 907018240 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003662487462387162, + "loss": 1.4199, + "theoretical_loss": 3.711989671225946, + "tokens_seen": 907083776 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036623871614844535, + "loss": 1.3399, + "theoretical_loss": 3.711963887011005, + "tokens_seen": 907149312 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036622868605817454, + "loss": 1.3316, + "theoretical_loss": 3.7119381051802764, + "tokens_seen": 907214848 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003662186559679037, + "loss": 1.2515, + "theoretical_loss": 3.7119123257333677, + "tokens_seen": 907280384 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036620862587763295, + "loss": 1.3242, + "theoretical_loss": 3.711886548669886, + "tokens_seen": 907345920 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003661985957873621, + "loss": 1.3558, + "theoretical_loss": 3.7118607739894394, + "tokens_seen": 907411456 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003661885656970913, + "loss": 1.3459, + "theoretical_loss": 3.711835001691635, + "tokens_seen": 907476992 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036617853560682044, + "loss": 1.286, + "theoretical_loss": 3.7118092317760807, + "tokens_seen": 907542528 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003661685055165497, + "loss": 1.4048, + "theoretical_loss": 3.7117834642423846, + "tokens_seen": 907608064 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 305792, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3697357177734375, + "objective/train/theoretical_loss": 3.7117576990901537, + "objective/train/tokens_used": 907673600, + "theoretical_loss": 3.7117576990901537, + "tokens_seen": 907673600 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036615847542627886, + "loss": 1.35, + "theoretical_loss": 3.7117576990901537, + "tokens_seen": 907673600 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036614844533600804, + "loss": 1.4014, + "theoretical_loss": 3.711731936318997, + "tokens_seen": 907739136 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003661384152457372, + "loss": 1.3665, + "theoretical_loss": 3.711706175928522, + "tokens_seen": 907804672 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003661283851554664, + "loss": 1.4163, + "theoretical_loss": 3.7116804179183367, + "tokens_seen": 907870208 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003661183550651956, + "loss": 1.4162, + "theoretical_loss": 3.7116546622880495, + "tokens_seen": 907935744 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003661083249749248, + "loss": 1.453, + "theoretical_loss": 3.711628909037269, + "tokens_seen": 908001280 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036609829488465394, + "loss": 1.4793, + "theoretical_loss": 3.7116031581656035, + "tokens_seen": 908066816 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003660882647943832, + "loss": 1.4303, + "theoretical_loss": 3.7115774096726617, + "tokens_seen": 908132352 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036607823470411236, + "loss": 1.4247, + "theoretical_loss": 3.711551663558052, + "tokens_seen": 908197888 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036606820461384154, + "loss": 1.3951, + "theoretical_loss": 3.7115259198213835, + "tokens_seen": 908263424 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003660581745235707, + "loss": 1.3408, + "theoretical_loss": 3.7115001784622637, + "tokens_seen": 908328960 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003660481444332999, + "loss": 1.4293, + "theoretical_loss": 3.7114744394803036, + "tokens_seen": 908394496 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003660381143430291, + "loss": 1.4685, + "theoretical_loss": 3.711448702875111, + "tokens_seen": 908460032 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003660280842527583, + "loss": 1.4373, + "theoretical_loss": 3.7114229686462945, + "tokens_seen": 908525568 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036601805416248745, + "loss": 1.34, + "theoretical_loss": 3.711397236793464, + "tokens_seen": 908591104 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003660080240722167, + "loss": 1.2344, + "theoretical_loss": 3.711371507316229, + "tokens_seen": 908656640 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003659979939819458, + "loss": 1.3653, + "theoretical_loss": 3.7113457802141983, + "tokens_seen": 908722176 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036598796389167504, + "loss": 1.4022, + "theoretical_loss": 3.7113200554869814, + "tokens_seen": 908787712 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003659779338014042, + "loss": 1.3421, + "theoretical_loss": 3.7112943331341883, + "tokens_seen": 908853248 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003659679037111334, + "loss": 1.4389, + "theoretical_loss": 3.711268613155428, + "tokens_seen": 908918784 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003659578736208626, + "loss": 1.3659, + "theoretical_loss": 3.7112428955503116, + "tokens_seen": 908984320 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036594784353059177, + "loss": 1.3889, + "theoretical_loss": 3.711217180318447, + "tokens_seen": 909049856 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036593781344032095, + "loss": 1.4246, + "theoretical_loss": 3.7111914674594457, + "tokens_seen": 909115392 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003659277833500502, + "loss": 1.4167, + "theoretical_loss": 3.7111657569729166, + "tokens_seen": 909180928 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003659177532597793, + "loss": 1.4035, + "theoretical_loss": 3.7111400488584705, + "tokens_seen": 909246464 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036590772316950855, + "loss": 1.3675, + "theoretical_loss": 3.7111143431157174, + "tokens_seen": 909312000 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036589769307923773, + "loss": 1.3008, + "theoretical_loss": 3.7110886397442675, + "tokens_seen": 909377536 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003658876629889669, + "loss": 1.3794, + "theoretical_loss": 3.7110629387437317, + "tokens_seen": 909443072 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003658776328986961, + "loss": 1.3353, + "theoretical_loss": 3.7110372401137193, + "tokens_seen": 909508608 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036586760280842527, + "loss": 1.3103, + "theoretical_loss": 3.711011543853842, + "tokens_seen": 909574144 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036585757271815445, + "loss": 1.4509, + "theoretical_loss": 3.71098584996371, + "tokens_seen": 909639680 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003658475426278837, + "loss": 1.3723, + "theoretical_loss": 3.7109601584429344, + "tokens_seen": 909705216 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003658375125376128, + "loss": 1.3269, + "theoretical_loss": 3.7109344692911255, + "tokens_seen": 909770752 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036582748244734205, + "loss": 1.38, + "theoretical_loss": 3.710908782507895, + "tokens_seen": 909836288 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003658174523570712, + "loss": 1.4105, + "theoretical_loss": 3.710883098092853, + "tokens_seen": 909901824 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003658074222668004, + "loss": 1.3713, + "theoretical_loss": 3.7108574160456107, + "tokens_seen": 909967360 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003657973921765296, + "loss": 1.2696, + "theoretical_loss": 3.7108317363657797, + "tokens_seen": 910032896 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003657873620862588, + "loss": 1.4201, + "theoretical_loss": 3.710806059052972, + "tokens_seen": 910098432 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036577733199598796, + "loss": 1.3776, + "theoretical_loss": 3.7107803841067977, + "tokens_seen": 910163968 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036576730190571714, + "loss": 1.3341, + "theoretical_loss": 3.710754711526869, + "tokens_seen": 910229504 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003657572718154463, + "loss": 1.3547, + "theoretical_loss": 3.7107290413127973, + "tokens_seen": 910295040 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036574724172517555, + "loss": 1.387, + "theoretical_loss": 3.7107033734641943, + "tokens_seen": 910360576 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003657372116349047, + "loss": 1.3852, + "theoretical_loss": 3.710677707980672, + "tokens_seen": 910426112 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003657271815446339, + "loss": 1.3265, + "theoretical_loss": 3.710652044861842, + "tokens_seen": 910491648 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003657171514543631, + "loss": 1.3577, + "theoretical_loss": 3.7106263841073153, + "tokens_seen": 910557184 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003657071213640923, + "loss": 1.4106, + "theoretical_loss": 3.7106007257167057, + "tokens_seen": 910622720 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036569709127382146, + "loss": 1.416, + "theoretical_loss": 3.7105750696896247, + "tokens_seen": 910688256 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036568706118355064, + "loss": 1.3248, + "theoretical_loss": 3.710549416025684, + "tokens_seen": 910753792 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003656770310932798, + "loss": 1.3652, + "theoretical_loss": 3.7105237647244964, + "tokens_seen": 910819328 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036566700100300906, + "loss": 1.3294, + "theoretical_loss": 3.710498115785674, + "tokens_seen": 910884864 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 307019, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3524775505065918, + "objective/train/theoretical_loss": 3.7104724692088293, + "objective/train/tokens_used": 910950400, + "theoretical_loss": 3.7104724692088293, + "tokens_seen": 910950400 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003656569709127382, + "loss": 1.3738, + "theoretical_loss": 3.7104724692088293, + "tokens_seen": 910950400 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003656469408224674, + "loss": 1.3532, + "theoretical_loss": 3.7104468249935754, + "tokens_seen": 911015936 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036563691073219655, + "loss": 1.3189, + "theoretical_loss": 3.710421183139524, + "tokens_seen": 911081472 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003656268806419258, + "loss": 1.3596, + "theoretical_loss": 3.710395543646289, + "tokens_seen": 911147008 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036561685055165496, + "loss": 1.3707, + "theoretical_loss": 3.710369906513483, + "tokens_seen": 911212544 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036560682046138414, + "loss": 1.3831, + "theoretical_loss": 3.710344271740718, + "tokens_seen": 911278080 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003655967903711134, + "loss": 1.3466, + "theoretical_loss": 3.710318639327608, + "tokens_seen": 911343616 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036558676028084256, + "loss": 1.3341, + "theoretical_loss": 3.7102930092737663, + "tokens_seen": 911409152 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036557673019057174, + "loss": 1.3074, + "theoretical_loss": 3.710267381578805, + "tokens_seen": 911474688 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003655667001003009, + "loss": 1.3021, + "theoretical_loss": 3.710241756242339, + "tokens_seen": 911540224 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003655566700100301, + "loss": 1.3524, + "theoretical_loss": 3.7102161332639794, + "tokens_seen": 911605760 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003655466399197593, + "loss": 1.3922, + "theoretical_loss": 3.7101905126433423, + "tokens_seen": 911671296 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003655366098294885, + "loss": 1.4272, + "theoretical_loss": 3.7101648943800396, + "tokens_seen": 911736832 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036552657973921765, + "loss": 1.3298, + "theoretical_loss": 3.7101392784736857, + "tokens_seen": 911802368 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003655165496489469, + "loss": 1.4441, + "theoretical_loss": 3.7101136649238944, + "tokens_seen": 911867904 + }, + { + "epoch": 0.28, + "learning_rate": 0.000365506519558676, + "loss": 1.3299, + "theoretical_loss": 3.7100880537302787, + "tokens_seen": 911933440 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036549648946840525, + "loss": 1.3497, + "theoretical_loss": 3.7100624448924533, + "tokens_seen": 911998976 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003654864593781344, + "loss": 1.3565, + "theoretical_loss": 3.710036838410032, + "tokens_seen": 912064512 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003654764292878636, + "loss": 1.2968, + "theoretical_loss": 3.7100112342826295, + "tokens_seen": 912130048 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003654663991975928, + "loss": 1.3547, + "theoretical_loss": 3.7099856325098592, + "tokens_seen": 912195584 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036545636910732197, + "loss": 1.4128, + "theoretical_loss": 3.7099600330913356, + "tokens_seen": 912261120 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036544633901705115, + "loss": 1.4812, + "theoretical_loss": 3.709934436026674, + "tokens_seen": 912326656 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003654363089267804, + "loss": 1.2751, + "theoretical_loss": 3.709908841315488, + "tokens_seen": 912392192 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003654262788365095, + "loss": 1.4176, + "theoretical_loss": 3.709883248957392, + "tokens_seen": 912457728 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036541624874623875, + "loss": 1.4448, + "theoretical_loss": 3.709857658952001, + "tokens_seen": 912523264 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036540621865596793, + "loss": 1.4512, + "theoretical_loss": 3.7098320712989303, + "tokens_seen": 912588800 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003653961885656971, + "loss": 1.2565, + "theoretical_loss": 3.7098064859977944, + "tokens_seen": 912654336 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003653861584754263, + "loss": 1.3499, + "theoretical_loss": 3.7097809030482076, + "tokens_seen": 912719872 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036537612838515547, + "loss": 1.3801, + "theoretical_loss": 3.709755322449786, + "tokens_seen": 912785408 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036536609829488465, + "loss": 1.3021, + "theoretical_loss": 3.7097297442021437, + "tokens_seen": 912850944 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003653560682046139, + "loss": 1.2635, + "theoretical_loss": 3.7097041683048966, + "tokens_seen": 912916480 + }, + { + "epoch": 0.28, + "learning_rate": 0.000365346038114343, + "loss": 1.3521, + "theoretical_loss": 3.7096785947576603, + "tokens_seen": 912982016 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036533600802407225, + "loss": 1.3144, + "theoretical_loss": 3.7096530235600493, + "tokens_seen": 913047552 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003653259779338014, + "loss": 1.3655, + "theoretical_loss": 3.70962745471168, + "tokens_seen": 913113088 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003653159478435306, + "loss": 1.4414, + "theoretical_loss": 3.709601888212167, + "tokens_seen": 913178624 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003653059177532598, + "loss": 1.4182, + "theoretical_loss": 3.7095763240611275, + "tokens_seen": 913244160 + }, + { + "epoch": 0.28, + "learning_rate": 0.000365295887662989, + "loss": 1.4068, + "theoretical_loss": 3.7095507622581754, + "tokens_seen": 913309696 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036528585757271816, + "loss": 1.4573, + "theoretical_loss": 3.7095252028029275, + "tokens_seen": 913375232 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036527582748244734, + "loss": 1.431, + "theoretical_loss": 3.7094996456950002, + "tokens_seen": 913440768 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003652657973921765, + "loss": 1.3551, + "theoretical_loss": 3.709474090934009, + "tokens_seen": 913506304 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036525576730190575, + "loss": 1.4343, + "theoretical_loss": 3.70944853851957, + "tokens_seen": 913571840 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003652457372116349, + "loss": 1.3797, + "theoretical_loss": 3.7094229884512995, + "tokens_seen": 913637376 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003652357071213641, + "loss": 1.3277, + "theoretical_loss": 3.7093974407288135, + "tokens_seen": 913702912 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003652256770310933, + "loss": 1.3626, + "theoretical_loss": 3.7093718953517287, + "tokens_seen": 913768448 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003652156469408225, + "loss": 1.308, + "theoretical_loss": 3.7093463523196624, + "tokens_seen": 913833984 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036520561685055166, + "loss": 1.4422, + "theoretical_loss": 3.70932081163223, + "tokens_seen": 913899520 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036519558676028084, + "loss": 1.3666, + "theoretical_loss": 3.709295273289048, + "tokens_seen": 913965056 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036518555667001, + "loss": 1.3789, + "theoretical_loss": 3.709269737289734, + "tokens_seen": 914030592 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036517552657973926, + "loss": 1.3282, + "theoretical_loss": 3.7092442036339044, + "tokens_seen": 914096128 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003651654964894684, + "loss": 1.3908, + "theoretical_loss": 3.7092186723211773, + "tokens_seen": 914161664 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 308189, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2112423181533813, + "objective/train/theoretical_loss": 3.7091931433511673, + "objective/train/tokens_used": 914227200, + "theoretical_loss": 3.7091931433511673, + "tokens_seen": 914227200 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003651554663991976, + "loss": 1.2765, + "theoretical_loss": 3.7091931433511673, + "tokens_seen": 914227200 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036514543630892675, + "loss": 1.3103, + "theoretical_loss": 3.7091676167234944, + "tokens_seen": 914292736 + }, + { + "epoch": 0.28, + "learning_rate": 0.000365135406218656, + "loss": 1.327, + "theoretical_loss": 3.7091420924377734, + "tokens_seen": 914358272 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036512537612838516, + "loss": 1.4549, + "theoretical_loss": 3.709116570493623, + "tokens_seen": 914423808 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036511534603811434, + "loss": 1.3826, + "theoretical_loss": 3.70909105089066, + "tokens_seen": 914489344 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003651053159478435, + "loss": 1.4439, + "theoretical_loss": 3.709065533628502, + "tokens_seen": 914554880 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036509528585757276, + "loss": 1.395, + "theoretical_loss": 3.7090400187067667, + "tokens_seen": 914620416 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003650852557673019, + "loss": 1.3896, + "theoretical_loss": 3.709014506125072, + "tokens_seen": 914685952 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003650752256770311, + "loss": 1.3442, + "theoretical_loss": 3.708988995883035, + "tokens_seen": 914751488 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036506519558676025, + "loss": 1.3447, + "theoretical_loss": 3.7089634879802738, + "tokens_seen": 914817024 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003650551654964895, + "loss": 1.4181, + "theoretical_loss": 3.7089379824164066, + "tokens_seen": 914882560 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036504513540621867, + "loss": 1.4224, + "theoretical_loss": 3.708912479191051, + "tokens_seen": 914948096 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036503510531594785, + "loss": 1.4079, + "theoretical_loss": 3.708886978303825, + "tokens_seen": 915013632 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036502507522567703, + "loss": 1.4617, + "theoretical_loss": 3.708861479754348, + "tokens_seen": 915079168 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003650150451354062, + "loss": 1.3526, + "theoretical_loss": 3.708835983542237, + "tokens_seen": 915144704 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003650050150451354, + "loss": 1.4087, + "theoretical_loss": 3.708810489667111, + "tokens_seen": 915210240 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003649949849548646, + "loss": 1.4224, + "theoretical_loss": 3.7087849981285883, + "tokens_seen": 915275776 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036498495486459375, + "loss": 1.3225, + "theoretical_loss": 3.7087595089262875, + "tokens_seen": 915341312 + }, + { + "epoch": 0.28, + "learning_rate": 0.000364974924774323, + "loss": 1.4898, + "theoretical_loss": 3.7087340220598266, + "tokens_seen": 915406848 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003649648946840521, + "loss": 1.302, + "theoretical_loss": 3.7087085375288256, + "tokens_seen": 915472384 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036495486459378135, + "loss": 1.3786, + "theoretical_loss": 3.708683055332902, + "tokens_seen": 915537920 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036494483450351053, + "loss": 1.4098, + "theoretical_loss": 3.7086575754716757, + "tokens_seen": 915603456 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003649348044132397, + "loss": 1.2417, + "theoretical_loss": 3.7086320979447653, + "tokens_seen": 915668992 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003649247743229689, + "loss": 1.2599, + "theoretical_loss": 3.70860662275179, + "tokens_seen": 915734528 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036491474423269813, + "loss": 1.3513, + "theoretical_loss": 3.7085811498923693, + "tokens_seen": 915800064 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036490471414242726, + "loss": 1.3659, + "theoretical_loss": 3.7085556793661216, + "tokens_seen": 915865600 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003648946840521565, + "loss": 1.4093, + "theoretical_loss": 3.708530211172667, + "tokens_seen": 915931136 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003648846539618856, + "loss": 1.3822, + "theoretical_loss": 3.7085047453116244, + "tokens_seen": 915996672 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036487462387161485, + "loss": 1.3718, + "theoretical_loss": 3.7084792817826147, + "tokens_seen": 916062208 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003648645937813441, + "loss": 1.3343, + "theoretical_loss": 3.7084538205852553, + "tokens_seen": 916127744 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003648545636910732, + "loss": 1.3291, + "theoretical_loss": 3.708428361719168, + "tokens_seen": 916193280 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036484453360080245, + "loss": 1.3047, + "theoretical_loss": 3.7084029051839713, + "tokens_seen": 916258816 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003648345035105316, + "loss": 1.3723, + "theoretical_loss": 3.7083774509792855, + "tokens_seen": 916324352 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003648244734202608, + "loss": 1.4085, + "theoretical_loss": 3.708351999104731, + "tokens_seen": 916389888 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036481444332999, + "loss": 1.3433, + "theoretical_loss": 3.7083265495599274, + "tokens_seen": 916455424 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003648044132397192, + "loss": 1.3883, + "theoretical_loss": 3.708301102344495, + "tokens_seen": 916520960 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036479438314944836, + "loss": 1.4129, + "theoretical_loss": 3.7082756574580538, + "tokens_seen": 916586496 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036478435305917754, + "loss": 1.3024, + "theoretical_loss": 3.7082502149002243, + "tokens_seen": 916652032 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003647743229689067, + "loss": 1.2843, + "theoretical_loss": 3.7082247746706276, + "tokens_seen": 916717568 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036476429287863595, + "loss": 1.3916, + "theoretical_loss": 3.7081993367688835, + "tokens_seen": 916783104 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003647542627883651, + "loss": 1.2733, + "theoretical_loss": 3.708173901194612, + "tokens_seen": 916848640 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003647442326980943, + "loss": 1.4154, + "theoretical_loss": 3.708148467947435, + "tokens_seen": 916914176 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003647342026078235, + "loss": 1.3592, + "theoretical_loss": 3.708123037026973, + "tokens_seen": 916979712 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003647241725175527, + "loss": 1.3461, + "theoretical_loss": 3.7080976084328467, + "tokens_seen": 917045248 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036471414242728186, + "loss": 1.4086, + "theoretical_loss": 3.7080721821646767, + "tokens_seen": 917110784 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036470411233701104, + "loss": 1.3881, + "theoretical_loss": 3.7080467582220855, + "tokens_seen": 917176320 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003646940822467402, + "loss": 1.3167, + "theoretical_loss": 3.708021336604692, + "tokens_seen": 917241856 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036468405215646946, + "loss": 1.364, + "theoretical_loss": 3.707995917312119, + "tokens_seen": 917307392 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003646740220661986, + "loss": 1.3879, + "theoretical_loss": 3.7079705003439876, + "tokens_seen": 917372928 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003646639919759278, + "loss": 1.3201, + "theoretical_loss": 3.7079450856999188, + "tokens_seen": 917438464 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 309045, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4410195350646973, + "objective/train/theoretical_loss": 3.7079196733795348, + "objective/train/tokens_used": 917504000, + "theoretical_loss": 3.7079196733795348, + "tokens_seen": 917504000 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036465396188565695, + "loss": 1.3893, + "theoretical_loss": 3.7079196733795348, + "tokens_seen": 917504000 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003646439317953862, + "loss": 1.4109, + "theoretical_loss": 3.7078942633824563, + "tokens_seen": 917569536 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036463390170511536, + "loss": 1.4172, + "theoretical_loss": 3.7078688557083055, + "tokens_seen": 917635072 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036462387161484454, + "loss": 1.3911, + "theoretical_loss": 3.7078434503567044, + "tokens_seen": 917700608 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003646138415245737, + "loss": 1.3216, + "theoretical_loss": 3.707818047327274, + "tokens_seen": 917766144 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036460381143430296, + "loss": 1.3999, + "theoretical_loss": 3.7077926466196374, + "tokens_seen": 917831680 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003645937813440321, + "loss": 1.2723, + "theoretical_loss": 3.7077672482334156, + "tokens_seen": 917897216 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003645837512537613, + "loss": 1.2812, + "theoretical_loss": 3.7077418521682315, + "tokens_seen": 917962752 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036457372116349045, + "loss": 1.3315, + "theoretical_loss": 3.7077164584237066, + "tokens_seen": 918028288 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003645636910732197, + "loss": 1.3397, + "theoretical_loss": 3.7076910669994634, + "tokens_seen": 918093824 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036455366098294887, + "loss": 1.386, + "theoretical_loss": 3.7076656778951245, + "tokens_seen": 918159360 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036454363089267805, + "loss": 1.3752, + "theoretical_loss": 3.7076402911103123, + "tokens_seen": 918224896 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036453360080240723, + "loss": 1.2846, + "theoretical_loss": 3.7076149066446495, + "tokens_seen": 918290432 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003645235707121364, + "loss": 1.4486, + "theoretical_loss": 3.707589524497759, + "tokens_seen": 918355968 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003645135406218656, + "loss": 1.4732, + "theoretical_loss": 3.707564144669263, + "tokens_seen": 918421504 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003645035105315948, + "loss": 1.4298, + "theoretical_loss": 3.707538767158784, + "tokens_seen": 918487040 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036449348044132395, + "loss": 1.3625, + "theoretical_loss": 3.7075133919659455, + "tokens_seen": 918552576 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003644834503510532, + "loss": 1.4243, + "theoretical_loss": 3.7074880190903707, + "tokens_seen": 918618112 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003644734202607823, + "loss": 1.4067, + "theoretical_loss": 3.707462648531682, + "tokens_seen": 918683648 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036446339017051155, + "loss": 1.3373, + "theoretical_loss": 3.7074372802895033, + "tokens_seen": 918749184 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036445336008024073, + "loss": 1.3129, + "theoretical_loss": 3.7074119143634574, + "tokens_seen": 918814720 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003644433299899699, + "loss": 1.3344, + "theoretical_loss": 3.707386550753168, + "tokens_seen": 918880256 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003644332998996991, + "loss": 1.4363, + "theoretical_loss": 3.7073611894582585, + "tokens_seen": 918945792 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036442326980942833, + "loss": 1.2608, + "theoretical_loss": 3.707335830478352, + "tokens_seen": 919011328 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036441323971915746, + "loss": 1.4083, + "theoretical_loss": 3.707310473813072, + "tokens_seen": 919076864 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003644032096288867, + "loss": 1.4192, + "theoretical_loss": 3.707285119462043, + "tokens_seen": 919142400 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003643931795386158, + "loss": 1.2706, + "theoretical_loss": 3.7072597674248886, + "tokens_seen": 919207936 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036438314944834505, + "loss": 1.3267, + "theoretical_loss": 3.707234417701232, + "tokens_seen": 919273472 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036437311935807424, + "loss": 1.3523, + "theoretical_loss": 3.707209070290698, + "tokens_seen": 919339008 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003643630892678034, + "loss": 1.3985, + "theoretical_loss": 3.70718372519291, + "tokens_seen": 919404544 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003643530591775326, + "loss": 1.3127, + "theoretical_loss": 3.707158382407493, + "tokens_seen": 919470080 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003643430290872618, + "loss": 1.4093, + "theoretical_loss": 3.7071330419340702, + "tokens_seen": 919535616 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036433299899699096, + "loss": 1.2634, + "theoretical_loss": 3.707107703772267, + "tokens_seen": 919601152 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003643229689067202, + "loss": 1.4433, + "theoretical_loss": 3.7070823679217066, + "tokens_seen": 919666688 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003643129388164493, + "loss": 1.342, + "theoretical_loss": 3.7070570343820144, + "tokens_seen": 919732224 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036430290872617856, + "loss": 1.349, + "theoretical_loss": 3.707031703152815, + "tokens_seen": 919797760 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003642928786359077, + "loss": 1.4058, + "theoretical_loss": 3.7070063742337322, + "tokens_seen": 919863296 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003642828485456369, + "loss": 1.3262, + "theoretical_loss": 3.706981047624392, + "tokens_seen": 919928832 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003642728184553661, + "loss": 1.2331, + "theoretical_loss": 3.706955723324418, + "tokens_seen": 919994368 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003642627883650953, + "loss": 1.3549, + "theoretical_loss": 3.706930401333436, + "tokens_seen": 920059904 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036425275827482446, + "loss": 1.4131, + "theoretical_loss": 3.7069050816510707, + "tokens_seen": 920125440 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003642427281845537, + "loss": 1.3398, + "theoretical_loss": 3.7068797642769478, + "tokens_seen": 920190976 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003642326980942828, + "loss": 1.3403, + "theoretical_loss": 3.7068544492106916, + "tokens_seen": 920256512 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036422266800401206, + "loss": 1.3695, + "theoretical_loss": 3.706829136451928, + "tokens_seen": 920322048 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003642126379137412, + "loss": 1.3686, + "theoretical_loss": 3.7068038260002814, + "tokens_seen": 920387584 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003642026078234704, + "loss": 1.3962, + "theoretical_loss": 3.706778517855379, + "tokens_seen": 920453120 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003641925777331996, + "loss": 1.3884, + "theoretical_loss": 3.706753212016845, + "tokens_seen": 920518656 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003641825476429288, + "loss": 1.393, + "theoretical_loss": 3.7067279084843054, + "tokens_seen": 920584192 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036417251755265797, + "loss": 1.3701, + "theoretical_loss": 3.7067026072573857, + "tokens_seen": 920649728 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036416248746238715, + "loss": 1.381, + "theoretical_loss": 3.7066773083357125, + "tokens_seen": 920715264 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 310434, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3575263023376465, + "objective/train/theoretical_loss": 3.7066520117189103, + "objective/train/tokens_used": 920780800, + "theoretical_loss": 3.7066520117189103, + "tokens_seen": 920780800 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036415245737211633, + "loss": 1.4175, + "theoretical_loss": 3.7066520117189103, + "tokens_seen": 920780800 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036414242728184556, + "loss": 1.3432, + "theoretical_loss": 3.706626717406607, + "tokens_seen": 920846336 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003641323971915747, + "loss": 1.2421, + "theoretical_loss": 3.7066014253984267, + "tokens_seen": 920911872 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003641223671013039, + "loss": 1.3462, + "theoretical_loss": 3.7065761356939966, + "tokens_seen": 920977408 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036411233701103316, + "loss": 1.3286, + "theoretical_loss": 3.7065508482929435, + "tokens_seen": 921042944 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003641023069207623, + "loss": 1.3774, + "theoretical_loss": 3.7065255631948926, + "tokens_seen": 921108480 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003640922768304915, + "loss": 1.43, + "theoretical_loss": 3.706500280399471, + "tokens_seen": 921174016 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036408224674022065, + "loss": 1.3724, + "theoretical_loss": 3.7064749999063045, + "tokens_seen": 921239552 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003640722166499499, + "loss": 1.2766, + "theoretical_loss": 3.7064497217150207, + "tokens_seen": 921305088 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036406218655967907, + "loss": 1.313, + "theoretical_loss": 3.7064244458252453, + "tokens_seen": 921370624 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036405215646940825, + "loss": 1.4371, + "theoretical_loss": 3.706399172236606, + "tokens_seen": 921436160 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036404212637913743, + "loss": 1.3977, + "theoretical_loss": 3.706373900948729, + "tokens_seen": 921501696 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003640320962888666, + "loss": 1.3872, + "theoretical_loss": 3.7063486319612418, + "tokens_seen": 921567232 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003640220661985958, + "loss": 1.3269, + "theoretical_loss": 3.7063233652737706, + "tokens_seen": 921632768 + }, + { + "epoch": 0.28, + "learning_rate": 0.000364012036108325, + "loss": 1.4494, + "theoretical_loss": 3.7062981008859435, + "tokens_seen": 921698304 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036400200601805415, + "loss": 1.2987, + "theoretical_loss": 3.7062728387973873, + "tokens_seen": 921763840 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003639919759277834, + "loss": 1.421, + "theoretical_loss": 3.7062475790077283, + "tokens_seen": 921829376 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003639819458375125, + "loss": 1.3668, + "theoretical_loss": 3.706222321516596, + "tokens_seen": 921894912 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036397191574724175, + "loss": 1.3509, + "theoretical_loss": 3.7061970663236155, + "tokens_seen": 921960448 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036396188565697093, + "loss": 1.2961, + "theoretical_loss": 3.7061718134284165, + "tokens_seen": 922025984 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003639518555667001, + "loss": 1.374, + "theoretical_loss": 3.706146562830625, + "tokens_seen": 922091520 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003639418254764293, + "loss": 1.3403, + "theoretical_loss": 3.70612131452987, + "tokens_seen": 922157056 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036393179538615853, + "loss": 1.4112, + "theoretical_loss": 3.7060960685257784, + "tokens_seen": 922222592 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036392176529588766, + "loss": 1.4226, + "theoretical_loss": 3.7060708248179783, + "tokens_seen": 922288128 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003639117352056169, + "loss": 1.2191, + "theoretical_loss": 3.7060455834060977, + "tokens_seen": 922353664 + }, + { + "epoch": 0.28, + "learning_rate": 0.000363901705115346, + "loss": 1.5023, + "theoretical_loss": 3.7060203442897652, + "tokens_seen": 922419200 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036389167502507525, + "loss": 1.4041, + "theoretical_loss": 3.7059951074686084, + "tokens_seen": 922484736 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036388164493480444, + "loss": 1.3752, + "theoretical_loss": 3.705969872942256, + "tokens_seen": 922550272 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003638716148445336, + "loss": 1.2625, + "theoretical_loss": 3.705944640710335, + "tokens_seen": 922615808 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003638615847542628, + "loss": 1.3914, + "theoretical_loss": 3.7059194107724758, + "tokens_seen": 922681344 + }, + { + "epoch": 0.28, + "learning_rate": 0.000363851554663992, + "loss": 1.3572, + "theoretical_loss": 3.7058941831283057, + "tokens_seen": 922746880 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036384152457372116, + "loss": 1.3435, + "theoretical_loss": 3.7058689577774535, + "tokens_seen": 922812416 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003638314944834504, + "loss": 1.4561, + "theoretical_loss": 3.705843734719548, + "tokens_seen": 922877952 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003638214643931795, + "loss": 1.3717, + "theoretical_loss": 3.7058185139542177, + "tokens_seen": 922943488 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036381143430290876, + "loss": 1.4056, + "theoretical_loss": 3.705793295481092, + "tokens_seen": 923009024 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003638014042126379, + "loss": 1.3128, + "theoretical_loss": 3.7057680792997996, + "tokens_seen": 923074560 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003637913741223671, + "loss": 1.3956, + "theoretical_loss": 3.7057428654099693, + "tokens_seen": 923140096 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003637813440320963, + "loss": 1.3468, + "theoretical_loss": 3.70571765381123, + "tokens_seen": 923205632 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003637713139418255, + "loss": 1.3841, + "theoretical_loss": 3.705692444503211, + "tokens_seen": 923271168 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036376128385155466, + "loss": 1.3408, + "theoretical_loss": 3.7056672374855424, + "tokens_seen": 923336704 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003637512537612839, + "loss": 1.313, + "theoretical_loss": 3.7056420327578534, + "tokens_seen": 923402240 + }, + { + "epoch": 0.28, + "learning_rate": 0.000363741223671013, + "loss": 1.3778, + "theoretical_loss": 3.705616830319773, + "tokens_seen": 923467776 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036373119358074226, + "loss": 1.2764, + "theoretical_loss": 3.7055916301709297, + "tokens_seen": 923533312 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003637211634904714, + "loss": 1.3337, + "theoretical_loss": 3.705566432310955, + "tokens_seen": 923598848 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003637111334002006, + "loss": 1.412, + "theoretical_loss": 3.705541236739478, + "tokens_seen": 923664384 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003637011033099298, + "loss": 1.3232, + "theoretical_loss": 3.7055160434561283, + "tokens_seen": 923729920 + }, + { + "epoch": 0.28, + "learning_rate": 0.000363691073219659, + "loss": 1.3104, + "theoretical_loss": 3.7054908524605357, + "tokens_seen": 923795456 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036368104312938817, + "loss": 1.3981, + "theoretical_loss": 3.70546566375233, + "tokens_seen": 923860992 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036367101303911735, + "loss": 1.4554, + "theoretical_loss": 3.705440477331142, + "tokens_seen": 923926528 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036366098294884653, + "loss": 1.4305, + "theoretical_loss": 3.7054152931966016, + "tokens_seen": 923992064 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 311361, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3047106266021729, + "objective/train/theoretical_loss": 3.705390111348339, + "objective/train/tokens_used": 924057600, + "theoretical_loss": 3.705390111348339, + "tokens_seen": 924057600 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036365095285857576, + "loss": 1.3011, + "theoretical_loss": 3.705390111348339, + "tokens_seen": 924057600 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003636409227683049, + "loss": 1.3445, + "theoretical_loss": 3.705364931785984, + "tokens_seen": 924123136 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003636308926780341, + "loss": 1.4185, + "theoretical_loss": 3.7053397545091675, + "tokens_seen": 924188672 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036362086258776325, + "loss": 1.3593, + "theoretical_loss": 3.7053145795175197, + "tokens_seen": 924254208 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003636108324974925, + "loss": 1.2903, + "theoretical_loss": 3.705289406810672, + "tokens_seen": 924319744 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036360080240722167, + "loss": 1.3737, + "theoretical_loss": 3.7052642363882535, + "tokens_seen": 924385280 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036359077231695085, + "loss": 1.3196, + "theoretical_loss": 3.7052390682498966, + "tokens_seen": 924450816 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036358074222668003, + "loss": 1.3611, + "theoretical_loss": 3.705213902395231, + "tokens_seen": 924516352 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036357071213640927, + "loss": 1.312, + "theoretical_loss": 3.705188738823889, + "tokens_seen": 924581888 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003635606820461384, + "loss": 1.4287, + "theoretical_loss": 3.7051635775354996, + "tokens_seen": 924647424 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036355065195586763, + "loss": 1.3705, + "theoretical_loss": 3.705138418529695, + "tokens_seen": 924712960 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036354062186559676, + "loss": 1.4215, + "theoretical_loss": 3.7051132618061073, + "tokens_seen": 924778496 + }, + { + "epoch": 0.28, + "learning_rate": 0.000363530591775326, + "loss": 1.2692, + "theoretical_loss": 3.705088107364366, + "tokens_seen": 924844032 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036352056168505517, + "loss": 1.3804, + "theoretical_loss": 3.7050629552041032, + "tokens_seen": 924909568 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036351053159478435, + "loss": 1.363, + "theoretical_loss": 3.7050378053249506, + "tokens_seen": 924975104 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036350050150451353, + "loss": 1.3499, + "theoretical_loss": 3.70501265772654, + "tokens_seen": 925040640 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003634904714142427, + "loss": 1.3027, + "theoretical_loss": 3.7049875124085014, + "tokens_seen": 925106176 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003634804413239719, + "loss": 1.4168, + "theoretical_loss": 3.7049623693704685, + "tokens_seen": 925171712 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036347041123370113, + "loss": 1.3584, + "theoretical_loss": 3.7049372286120716, + "tokens_seen": 925237248 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036346038114343026, + "loss": 1.3907, + "theoretical_loss": 3.704912090132944, + "tokens_seen": 925302784 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003634503510531595, + "loss": 1.321, + "theoretical_loss": 3.704886953932716, + "tokens_seen": 925368320 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003634403209628886, + "loss": 1.4285, + "theoretical_loss": 3.704861820011021, + "tokens_seen": 925433856 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036343029087261786, + "loss": 1.4027, + "theoretical_loss": 3.70483668836749, + "tokens_seen": 925499392 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036342026078234704, + "loss": 1.3564, + "theoretical_loss": 3.7048115590017563, + "tokens_seen": 925564928 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003634102306920762, + "loss": 1.4266, + "theoretical_loss": 3.704786431913451, + "tokens_seen": 925630464 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003634002006018054, + "loss": 1.383, + "theoretical_loss": 3.7047613071022076, + "tokens_seen": 925696000 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036339017051153464, + "loss": 1.4159, + "theoretical_loss": 3.7047361845676585, + "tokens_seen": 925761536 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036338014042126376, + "loss": 1.3375, + "theoretical_loss": 3.704711064309435, + "tokens_seen": 925827072 + }, + { + "epoch": 0.28, + "learning_rate": 0.000363370110330993, + "loss": 1.2907, + "theoretical_loss": 3.704685946327171, + "tokens_seen": 925892608 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003633600802407222, + "loss": 1.3905, + "theoretical_loss": 3.704660830620499, + "tokens_seen": 925958144 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036335005015045136, + "loss": 1.3144, + "theoretical_loss": 3.704635717189051, + "tokens_seen": 926023680 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003633400200601806, + "loss": 1.389, + "theoretical_loss": 3.7046106060324613, + "tokens_seen": 926089216 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003633299899699097, + "loss": 1.3507, + "theoretical_loss": 3.7045854971503616, + "tokens_seen": 926154752 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036331995987963896, + "loss": 1.3077, + "theoretical_loss": 3.704560390542385, + "tokens_seen": 926220288 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003633099297893681, + "loss": 1.2299, + "theoretical_loss": 3.7045352862081655, + "tokens_seen": 926285824 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003632998996990973, + "loss": 1.4366, + "theoretical_loss": 3.7045101841473365, + "tokens_seen": 926351360 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003632898696088265, + "loss": 1.4167, + "theoretical_loss": 3.70448508435953, + "tokens_seen": 926416896 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003632798395185557, + "loss": 1.4473, + "theoretical_loss": 3.7044599868443804, + "tokens_seen": 926482432 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036326980942828486, + "loss": 1.3657, + "theoretical_loss": 3.7044348916015206, + "tokens_seen": 926547968 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003632597793380141, + "loss": 1.3958, + "theoretical_loss": 3.704409798630585, + "tokens_seen": 926613504 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003632497492477432, + "loss": 1.3406, + "theoretical_loss": 3.7043847079312058, + "tokens_seen": 926679040 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036323971915747246, + "loss": 1.3602, + "theoretical_loss": 3.704359619503018, + "tokens_seen": 926744576 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003632296890672016, + "loss": 1.2826, + "theoretical_loss": 3.704334533345656, + "tokens_seen": 926810112 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003632196589769308, + "loss": 1.467, + "theoretical_loss": 3.704309449458752, + "tokens_seen": 926875648 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036320962888666, + "loss": 1.3148, + "theoretical_loss": 3.7042843678419404, + "tokens_seen": 926941184 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003631995987963892, + "loss": 1.3884, + "theoretical_loss": 3.7042592884948564, + "tokens_seen": 927006720 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036318956870611837, + "loss": 1.2863, + "theoretical_loss": 3.704234211417133, + "tokens_seen": 927072256 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036317953861584755, + "loss": 1.2437, + "theoretical_loss": 3.704209136608405, + "tokens_seen": 927137792 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036316950852557673, + "loss": 1.3358, + "theoretical_loss": 3.704184064068307, + "tokens_seen": 927203328 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036315947843530596, + "loss": 1.3869, + "theoretical_loss": 3.704158993796473, + "tokens_seen": 927268864 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 312491, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.246079683303833, + "objective/train/theoretical_loss": 3.7041339257925374, + "objective/train/tokens_used": 927334400, + "theoretical_loss": 3.7041339257925374, + "tokens_seen": 927334400 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003631494483450351, + "loss": 1.2815, + "theoretical_loss": 3.7041339257925374, + "tokens_seen": 927334400 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003631394182547643, + "loss": 1.2322, + "theoretical_loss": 3.7041088600561345, + "tokens_seen": 927399936 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036312938816449345, + "loss": 1.3737, + "theoretical_loss": 3.7040837965869002, + "tokens_seen": 927465472 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003631193580742227, + "loss": 1.3747, + "theoretical_loss": 3.704058735384468, + "tokens_seen": 927531008 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036310932798395187, + "loss": 1.2686, + "theoretical_loss": 3.704033676448473, + "tokens_seen": 927596544 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036309929789368105, + "loss": 1.3745, + "theoretical_loss": 3.7040086197785502, + "tokens_seen": 927662080 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036308926780341023, + "loss": 1.4421, + "theoretical_loss": 3.7039835653743354, + "tokens_seen": 927727616 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036307923771313947, + "loss": 1.2995, + "theoretical_loss": 3.703958513235463, + "tokens_seen": 927793152 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003630692076228686, + "loss": 1.28, + "theoretical_loss": 3.703933463361568, + "tokens_seen": 927858688 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036305917753259783, + "loss": 1.3519, + "theoretical_loss": 3.7039084157522857, + "tokens_seen": 927924224 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036304914744232696, + "loss": 1.3087, + "theoretical_loss": 3.703883370407252, + "tokens_seen": 927989760 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003630391173520562, + "loss": 1.4149, + "theoretical_loss": 3.7038583273261017, + "tokens_seen": 928055296 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036302908726178537, + "loss": 1.4417, + "theoretical_loss": 3.7038332865084707, + "tokens_seen": 928120832 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036301905717151455, + "loss": 1.4071, + "theoretical_loss": 3.703808247953994, + "tokens_seen": 928186368 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036300902708124374, + "loss": 1.4252, + "theoretical_loss": 3.7037832116623086, + "tokens_seen": 928251904 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003629989969909729, + "loss": 1.5384, + "theoretical_loss": 3.7037581776330493, + "tokens_seen": 928317440 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003629889669007021, + "loss": 1.3603, + "theoretical_loss": 3.7037331458658516, + "tokens_seen": 928382976 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036297893681043133, + "loss": 1.3562, + "theoretical_loss": 3.7037081163603522, + "tokens_seen": 928448512 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036296890672016046, + "loss": 1.402, + "theoretical_loss": 3.703683089116187, + "tokens_seen": 928514048 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003629588766298897, + "loss": 1.3518, + "theoretical_loss": 3.7036580641329917, + "tokens_seen": 928579584 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003629488465396188, + "loss": 1.4203, + "theoretical_loss": 3.703633041410403, + "tokens_seen": 928645120 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036293881644934806, + "loss": 1.2942, + "theoretical_loss": 3.7036080209480566, + "tokens_seen": 928710656 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036292878635907724, + "loss": 1.3357, + "theoretical_loss": 3.7035830027455896, + "tokens_seen": 928776192 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003629187562688064, + "loss": 1.3738, + "theoretical_loss": 3.7035579868026383, + "tokens_seen": 928841728 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003629087261785356, + "loss": 1.3314, + "theoretical_loss": 3.703532973118839, + "tokens_seen": 928907264 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036289869608826484, + "loss": 1.4569, + "theoretical_loss": 3.703507961693828, + "tokens_seen": 928972800 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036288866599799396, + "loss": 1.293, + "theoretical_loss": 3.703482952527242, + "tokens_seen": 929038336 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003628786359077232, + "loss": 1.428, + "theoretical_loss": 3.7034579456187187, + "tokens_seen": 929103872 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003628686058174523, + "loss": 1.3332, + "theoretical_loss": 3.7034329409678937, + "tokens_seen": 929169408 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036285857572718156, + "loss": 1.3764, + "theoretical_loss": 3.703407938574405, + "tokens_seen": 929234944 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036284854563691074, + "loss": 1.3047, + "theoretical_loss": 3.703382938437889, + "tokens_seen": 929300480 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003628385155466399, + "loss": 1.4539, + "theoretical_loss": 3.703357940557983, + "tokens_seen": 929366016 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003628284854563691, + "loss": 1.4255, + "theoretical_loss": 3.7033329449343246, + "tokens_seen": 929431552 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003628184553660983, + "loss": 1.3277, + "theoretical_loss": 3.7033079515665506, + "tokens_seen": 929497088 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036280842527582747, + "loss": 1.3984, + "theoretical_loss": 3.703282960454299, + "tokens_seen": 929562624 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003627983951855567, + "loss": 1.3763, + "theoretical_loss": 3.703257971597206, + "tokens_seen": 929628160 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036278836509528583, + "loss": 1.3236, + "theoretical_loss": 3.70323298499491, + "tokens_seen": 929693696 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036277833500501506, + "loss": 1.2881, + "theoretical_loss": 3.7032080006470487, + "tokens_seen": 929759232 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036276830491474424, + "loss": 1.3818, + "theoretical_loss": 3.7031830185532595, + "tokens_seen": 929824768 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003627582748244734, + "loss": 1.4711, + "theoretical_loss": 3.7031580387131804, + "tokens_seen": 929890304 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003627482447342026, + "loss": 1.3684, + "theoretical_loss": 3.7031330611264495, + "tokens_seen": 929955840 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003627382146439318, + "loss": 1.3719, + "theoretical_loss": 3.703108085792704, + "tokens_seen": 930021376 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036272818455366097, + "loss": 1.3774, + "theoretical_loss": 3.7030831127115826, + "tokens_seen": 930086912 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003627181544633902, + "loss": 1.3979, + "theoretical_loss": 3.7030581418827233, + "tokens_seen": 930152448 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036270812437311933, + "loss": 1.3621, + "theoretical_loss": 3.703033173305764, + "tokens_seen": 930217984 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036269809428284857, + "loss": 1.3197, + "theoretical_loss": 3.703008206980343, + "tokens_seen": 930283520 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003626880641925777, + "loss": 1.3313, + "theoretical_loss": 3.7029832429060994, + "tokens_seen": 930349056 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036267803410230693, + "loss": 1.3168, + "theoretical_loss": 3.7029582810826707, + "tokens_seen": 930414592 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003626680040120361, + "loss": 1.5098, + "theoretical_loss": 3.702933321509696, + "tokens_seen": 930480128 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003626579739217653, + "loss": 1.4582, + "theoretical_loss": 3.702908364186814, + "tokens_seen": 930545664 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 313439, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4109883308410645, + "objective/train/theoretical_loss": 3.702883409113663, + "objective/train/tokens_used": 930611200, + "theoretical_loss": 3.702883409113663, + "tokens_seen": 930611200 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036264794383149447, + "loss": 1.3723, + "theoretical_loss": 3.702883409113663, + "tokens_seen": 930611200 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036263791374122365, + "loss": 1.2662, + "theoretical_loss": 3.7028584562898823, + "tokens_seen": 930676736 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036262788365095283, + "loss": 1.3644, + "theoretical_loss": 3.7028335057151103, + "tokens_seen": 930742272 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036261785356068207, + "loss": 1.3539, + "theoretical_loss": 3.7028085573889857, + "tokens_seen": 930807808 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036260782347041125, + "loss": 1.3744, + "theoretical_loss": 3.7027836113111485, + "tokens_seen": 930873344 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036259779338014043, + "loss": 1.4172, + "theoretical_loss": 3.7027586674812376, + "tokens_seen": 930938880 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036258776328986967, + "loss": 1.3622, + "theoretical_loss": 3.7027337258988915, + "tokens_seen": 931004416 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003625777331995988, + "loss": 1.3363, + "theoretical_loss": 3.7027087865637496, + "tokens_seen": 931069952 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036256770310932803, + "loss": 1.3763, + "theoretical_loss": 3.702683849475452, + "tokens_seen": 931135488 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036255767301905716, + "loss": 1.3768, + "theoretical_loss": 3.7026589146336377, + "tokens_seen": 931201024 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003625476429287864, + "loss": 1.4431, + "theoretical_loss": 3.702633982037946, + "tokens_seen": 931266560 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003625376128385156, + "loss": 1.376, + "theoretical_loss": 3.7026090516880172, + "tokens_seen": 931332096 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036252758274824475, + "loss": 1.3368, + "theoretical_loss": 3.7025841235834904, + "tokens_seen": 931397632 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036251755265797394, + "loss": 1.3474, + "theoretical_loss": 3.702559197724006, + "tokens_seen": 931463168 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003625075225677031, + "loss": 1.4948, + "theoretical_loss": 3.702534274109203, + "tokens_seen": 931528704 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003624974924774323, + "loss": 1.5828, + "theoretical_loss": 3.702509352738722, + "tokens_seen": 931594240 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036248746238716153, + "loss": 1.3951, + "theoretical_loss": 3.7024844336122027, + "tokens_seen": 931659776 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036247743229689066, + "loss": 1.2294, + "theoretical_loss": 3.7024595167292853, + "tokens_seen": 931725312 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003624674022066199, + "loss": 1.3637, + "theoretical_loss": 3.7024346020896104, + "tokens_seen": 931790848 + }, + { + "epoch": 0.28, + "learning_rate": 0.000362457372116349, + "loss": 1.2645, + "theoretical_loss": 3.7024096896928183, + "tokens_seen": 931856384 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036244734202607826, + "loss": 1.3787, + "theoretical_loss": 3.7023847795385487, + "tokens_seen": 931921920 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036243731193580744, + "loss": 1.3615, + "theoretical_loss": 3.7023598716264425, + "tokens_seen": 931987456 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003624272818455366, + "loss": 1.3895, + "theoretical_loss": 3.7023349659561395, + "tokens_seen": 932052992 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003624172517552658, + "loss": 1.3428, + "theoretical_loss": 3.702310062527282, + "tokens_seen": 932118528 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036240722166499504, + "loss": 1.3802, + "theoretical_loss": 3.7022851613395096, + "tokens_seen": 932184064 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036239719157472416, + "loss": 1.4088, + "theoretical_loss": 3.7022602623924623, + "tokens_seen": 932249600 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003623871614844534, + "loss": 1.4365, + "theoretical_loss": 3.702235365685783, + "tokens_seen": 932315136 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003623771313941825, + "loss": 1.2971, + "theoretical_loss": 3.702210471219111, + "tokens_seen": 932380672 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036236710130391176, + "loss": 1.3589, + "theoretical_loss": 3.7021855789920872, + "tokens_seen": 932446208 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036235707121364094, + "loss": 1.3229, + "theoretical_loss": 3.702160689004354, + "tokens_seen": 932511744 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003623470411233701, + "loss": 1.3595, + "theoretical_loss": 3.702135801255552, + "tokens_seen": 932577280 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003623370110330993, + "loss": 1.2945, + "theoretical_loss": 3.7021109157453225, + "tokens_seen": 932642816 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003623269809428285, + "loss": 1.3424, + "theoretical_loss": 3.7020860324733063, + "tokens_seen": 932708352 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036231695085255767, + "loss": 1.3936, + "theoretical_loss": 3.7020611514391457, + "tokens_seen": 932773888 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003623069207622869, + "loss": 1.2429, + "theoretical_loss": 3.7020362726424816, + "tokens_seen": 932839424 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036229689067201603, + "loss": 1.2466, + "theoretical_loss": 3.7020113960829564, + "tokens_seen": 932904960 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036228686058174526, + "loss": 1.3025, + "theoretical_loss": 3.701986521760211, + "tokens_seen": 932970496 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036227683049147444, + "loss": 1.3899, + "theoretical_loss": 3.701961649673888, + "tokens_seen": 933036032 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003622668004012036, + "loss": 1.3974, + "theoretical_loss": 3.701936779823628, + "tokens_seen": 933101568 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003622567703109328, + "loss": 1.3987, + "theoretical_loss": 3.7019119122090736, + "tokens_seen": 933167104 + }, + { + "epoch": 0.28, + "learning_rate": 0.000362246740220662, + "loss": 1.212, + "theoretical_loss": 3.7018870468298672, + "tokens_seen": 933232640 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036223671013039117, + "loss": 1.3052, + "theoretical_loss": 3.701862183685651, + "tokens_seen": 933298176 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003622266800401204, + "loss": 1.3897, + "theoretical_loss": 3.701837322776066, + "tokens_seen": 933363712 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036221664994984953, + "loss": 1.3057, + "theoretical_loss": 3.7018124641007564, + "tokens_seen": 933429248 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036220661985957877, + "loss": 1.3361, + "theoretical_loss": 3.7017876076593623, + "tokens_seen": 933494784 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003621965897693079, + "loss": 1.3155, + "theoretical_loss": 3.701762753451528, + "tokens_seen": 933560320 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036218655967903713, + "loss": 1.3928, + "theoretical_loss": 3.701737901476895, + "tokens_seen": 933625856 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003621765295887663, + "loss": 1.3088, + "theoretical_loss": 3.7017130517351067, + "tokens_seen": 933691392 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003621664994984955, + "loss": 1.2969, + "theoretical_loss": 3.701688204225804, + "tokens_seen": 933756928 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036215646940822467, + "loss": 1.417, + "theoretical_loss": 3.7016633589486325, + "tokens_seen": 933822464 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 314676, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3296477794647217, + "objective/train/theoretical_loss": 3.7016385159032326, + "objective/train/tokens_used": 933888000, + "theoretical_loss": 3.7016385159032326, + "tokens_seen": 933888000 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036214643931795385, + "loss": 1.29, + "theoretical_loss": 3.7016385159032326, + "tokens_seen": 933888000 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036213640922768303, + "loss": 1.2345, + "theoretical_loss": 3.7016136750892485, + "tokens_seen": 933953536 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036212637913741227, + "loss": 1.483, + "theoretical_loss": 3.701588836506323, + "tokens_seen": 934019072 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003621163490471414, + "loss": 1.2198, + "theoretical_loss": 3.7015640001540984, + "tokens_seen": 934084608 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036210631895687063, + "loss": 1.4094, + "theoretical_loss": 3.701539166032219, + "tokens_seen": 934150144 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003620962888665998, + "loss": 1.3945, + "theoretical_loss": 3.7015143341403274, + "tokens_seen": 934215680 + }, + { + "epoch": 0.28, + "learning_rate": 0.000362086258776329, + "loss": 1.3399, + "theoretical_loss": 3.7014895044780673, + "tokens_seen": 934281216 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003620762286860582, + "loss": 1.3888, + "theoretical_loss": 3.701464677045082, + "tokens_seen": 934346752 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036206619859578736, + "loss": 1.4108, + "theoretical_loss": 3.7014398518410148, + "tokens_seen": 934412288 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036205616850551654, + "loss": 1.3396, + "theoretical_loss": 3.7014150288655094, + "tokens_seen": 934477824 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003620461384152458, + "loss": 1.3299, + "theoretical_loss": 3.7013902081182097, + "tokens_seen": 934543360 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003620361083249749, + "loss": 1.2005, + "theoretical_loss": 3.7013653895987595, + "tokens_seen": 934608896 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036202607823470414, + "loss": 1.3316, + "theoretical_loss": 3.701340573306802, + "tokens_seen": 934674432 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036201604814443326, + "loss": 1.4933, + "theoretical_loss": 3.7013157592419814, + "tokens_seen": 934739968 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003620060180541625, + "loss": 1.3932, + "theoretical_loss": 3.701290947403942, + "tokens_seen": 934805504 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003619959879638917, + "loss": 1.3544, + "theoretical_loss": 3.701266137792328, + "tokens_seen": 934871040 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036198595787362086, + "loss": 1.299, + "theoretical_loss": 3.7012413304067833, + "tokens_seen": 934936576 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036197592778335004, + "loss": 1.3716, + "theoretical_loss": 3.701216525246952, + "tokens_seen": 935002112 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003619658976930792, + "loss": 1.3755, + "theoretical_loss": 3.701191722312478, + "tokens_seen": 935067648 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003619558676028084, + "loss": 1.3924, + "theoretical_loss": 3.701166921603007, + "tokens_seen": 935133184 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036194583751253764, + "loss": 1.4004, + "theoretical_loss": 3.7011421231181822, + "tokens_seen": 935198720 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036193580742226677, + "loss": 1.4167, + "theoretical_loss": 3.7011173268576494, + "tokens_seen": 935264256 + }, + { + "epoch": 0.28, + "learning_rate": 0.000361925777331996, + "loss": 1.2463, + "theoretical_loss": 3.701092532821052, + "tokens_seen": 935329792 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003619157472417252, + "loss": 1.4235, + "theoretical_loss": 3.7010677410080355, + "tokens_seen": 935395328 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036190571715145436, + "loss": 1.4181, + "theoretical_loss": 3.7010429514182444, + "tokens_seen": 935460864 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036189568706118354, + "loss": 1.3457, + "theoretical_loss": 3.701018164051324, + "tokens_seen": 935526400 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003618856569709127, + "loss": 1.3308, + "theoretical_loss": 3.7009933789069187, + "tokens_seen": 935591936 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003618756268806419, + "loss": 1.3389, + "theoretical_loss": 3.7009685959846736, + "tokens_seen": 935657472 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036186559679037114, + "loss": 1.4557, + "theoretical_loss": 3.7009438152842344, + "tokens_seen": 935723008 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003618555667001003, + "loss": 1.3093, + "theoretical_loss": 3.7009190368052463, + "tokens_seen": 935788544 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003618455366098295, + "loss": 1.3501, + "theoretical_loss": 3.7008942605473543, + "tokens_seen": 935854080 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003618355065195587, + "loss": 1.3118, + "theoretical_loss": 3.7008694865102036, + "tokens_seen": 935919616 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036182547642928787, + "loss": 1.3087, + "theoretical_loss": 3.7008447146934396, + "tokens_seen": 935985152 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003618154463390171, + "loss": 1.2762, + "theoretical_loss": 3.7008199450967085, + "tokens_seen": 936050688 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036180541624874623, + "loss": 1.3652, + "theoretical_loss": 3.7007951777196553, + "tokens_seen": 936116224 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036179538615847546, + "loss": 1.3927, + "theoretical_loss": 3.7007704125619263, + "tokens_seen": 936181760 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036178535606820464, + "loss": 1.3026, + "theoretical_loss": 3.7007456496231668, + "tokens_seen": 936247296 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003617753259779338, + "loss": 1.3514, + "theoretical_loss": 3.7007208889030228, + "tokens_seen": 936312832 + }, + { + "epoch": 0.28, + "learning_rate": 0.000361765295887663, + "loss": 1.2748, + "theoretical_loss": 3.7006961304011403, + "tokens_seen": 936378368 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003617552657973922, + "loss": 1.3386, + "theoretical_loss": 3.7006713741171655, + "tokens_seen": 936443904 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036174523570712137, + "loss": 1.3631, + "theoretical_loss": 3.700646620050744, + "tokens_seen": 936509440 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003617352056168506, + "loss": 1.3309, + "theoretical_loss": 3.700621868201523, + "tokens_seen": 936574976 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036172517552657973, + "loss": 1.1772, + "theoretical_loss": 3.7005971185691475, + "tokens_seen": 936640512 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036171514543630897, + "loss": 1.3421, + "theoretical_loss": 3.700572371153265, + "tokens_seen": 936706048 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003617051153460381, + "loss": 1.2879, + "theoretical_loss": 3.7005476259535213, + "tokens_seen": 936771584 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036169508525576733, + "loss": 1.2537, + "theoretical_loss": 3.700522882969563, + "tokens_seen": 936837120 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003616850551654965, + "loss": 1.4067, + "theoretical_loss": 3.7004981422010372, + "tokens_seen": 936902656 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003616750250752257, + "loss": 1.3043, + "theoretical_loss": 3.7004734036475897, + "tokens_seen": 936968192 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036166499498495487, + "loss": 1.2736, + "theoretical_loss": 3.7004486673088675, + "tokens_seen": 937033728 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036165496489468405, + "loss": 1.392, + "theoretical_loss": 3.7004239331845183, + "tokens_seen": 937099264 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 315912, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2183436155319214, + "objective/train/theoretical_loss": 3.700399201274188, + "objective/train/tokens_used": 937164800, + "theoretical_loss": 3.700399201274188, + "tokens_seen": 937164800 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036164493480441323, + "loss": 1.3164, + "theoretical_loss": 3.700399201274188, + "tokens_seen": 937164800 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036163490471414247, + "loss": 1.4052, + "theoretical_loss": 3.700374471577524, + "tokens_seen": 937230336 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003616248746238716, + "loss": 1.3312, + "theoretical_loss": 3.700349744094174, + "tokens_seen": 937295872 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036161484453360083, + "loss": 1.271, + "theoretical_loss": 3.7003250188237837, + "tokens_seen": 937361408 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036160481444333, + "loss": 1.3329, + "theoretical_loss": 3.700300295766002, + "tokens_seen": 937426944 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003615947843530592, + "loss": 1.385, + "theoretical_loss": 3.700275574920475, + "tokens_seen": 937492480 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003615847542627884, + "loss": 1.3094, + "theoretical_loss": 3.700250856286851, + "tokens_seen": 937558016 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036157472417251756, + "loss": 1.361, + "theoretical_loss": 3.7002261398647764, + "tokens_seen": 937623552 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036156469408224674, + "loss": 1.3261, + "theoretical_loss": 3.7002014256539004, + "tokens_seen": 937689088 + }, + { + "epoch": 0.28, + "learning_rate": 0.000361554663991976, + "loss": 1.3212, + "theoretical_loss": 3.7001767136538692, + "tokens_seen": 937754624 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003615446339017051, + "loss": 1.3362, + "theoretical_loss": 3.700152003864331, + "tokens_seen": 937820160 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036153460381143434, + "loss": 1.2942, + "theoretical_loss": 3.700127296284934, + "tokens_seen": 937885696 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036152457372116346, + "loss": 1.2848, + "theoretical_loss": 3.700102590915326, + "tokens_seen": 937951232 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003615145436308927, + "loss": 1.3074, + "theoretical_loss": 3.700077887755154, + "tokens_seen": 938016768 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003615045135406219, + "loss": 1.3021, + "theoretical_loss": 3.7000531868040674, + "tokens_seen": 938082304 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036149448345035106, + "loss": 1.4274, + "theoretical_loss": 3.7000284880617134, + "tokens_seen": 938147840 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036148445336008024, + "loss": 1.3162, + "theoretical_loss": 3.700003791527741, + "tokens_seen": 938213376 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003614744232698094, + "loss": 1.3068, + "theoretical_loss": 3.699979097201798, + "tokens_seen": 938278912 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003614643931795386, + "loss": 1.2985, + "theoretical_loss": 3.699954405083533, + "tokens_seen": 938344448 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036145436308926784, + "loss": 1.2515, + "theoretical_loss": 3.699929715172594, + "tokens_seen": 938409984 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036144433299899697, + "loss": 1.3144, + "theoretical_loss": 3.69990502746863, + "tokens_seen": 938475520 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003614343029087262, + "loss": 1.328, + "theoretical_loss": 3.69988034197129, + "tokens_seen": 938541056 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003614242728184554, + "loss": 1.4819, + "theoretical_loss": 3.6998556586802214, + "tokens_seen": 938606592 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036141424272818456, + "loss": 1.3977, + "theoretical_loss": 3.6998309775950746, + "tokens_seen": 938672128 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036140421263791374, + "loss": 1.406, + "theoretical_loss": 3.699806298715497, + "tokens_seen": 938737664 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003613941825476429, + "loss": 1.3702, + "theoretical_loss": 3.6997816220411384, + "tokens_seen": 938803200 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003613841524573721, + "loss": 1.4582, + "theoretical_loss": 3.6997569475716476, + "tokens_seen": 938868736 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036137412236710134, + "loss": 1.3244, + "theoretical_loss": 3.6997322753066744, + "tokens_seen": 938934272 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036136409227683047, + "loss": 1.3317, + "theoretical_loss": 3.699707605245867, + "tokens_seen": 938999808 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003613540621865597, + "loss": 1.3282, + "theoretical_loss": 3.699682937388875, + "tokens_seen": 939065344 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036134403209628883, + "loss": 1.3846, + "theoretical_loss": 3.6996582717353474, + "tokens_seen": 939130880 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036133400200601807, + "loss": 1.4171, + "theoretical_loss": 3.699633608284935, + "tokens_seen": 939196416 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036132397191574725, + "loss": 1.2911, + "theoretical_loss": 3.699608947037285, + "tokens_seen": 939261952 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036131394182547643, + "loss": 1.4034, + "theoretical_loss": 3.699584287992049, + "tokens_seen": 939327488 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003613039117352056, + "loss": 1.3171, + "theoretical_loss": 3.699559631148876, + "tokens_seen": 939393024 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036129388164493484, + "loss": 1.3749, + "theoretical_loss": 3.6995349765074157, + "tokens_seen": 939458560 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036128385155466397, + "loss": 1.3992, + "theoretical_loss": 3.699510324067318, + "tokens_seen": 939524096 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003612738214643932, + "loss": 1.2809, + "theoretical_loss": 3.6994856738282325, + "tokens_seen": 939589632 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036126379137412233, + "loss": 1.346, + "theoretical_loss": 3.6994610257898097, + "tokens_seen": 939655168 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036125376128385157, + "loss": 1.3076, + "theoretical_loss": 3.6994363799516994, + "tokens_seen": 939720704 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036124373119358075, + "loss": 1.3497, + "theoretical_loss": 3.699411736313552, + "tokens_seen": 939786240 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036123370110330993, + "loss": 1.2832, + "theoretical_loss": 3.699387094875017, + "tokens_seen": 939851776 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003612236710130391, + "loss": 1.3191, + "theoretical_loss": 3.6993624556357454, + "tokens_seen": 939917312 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003612136409227683, + "loss": 1.217, + "theoretical_loss": 3.6993378185953874, + "tokens_seen": 939982848 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003612036108324975, + "loss": 1.3955, + "theoretical_loss": 3.6993131837535937, + "tokens_seen": 940048384 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003611935807422267, + "loss": 1.294, + "theoretical_loss": 3.6992885511100146, + "tokens_seen": 940113920 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036118355065195584, + "loss": 1.2947, + "theoretical_loss": 3.6992639206643005, + "tokens_seen": 940179456 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036117352056168507, + "loss": 1.3492, + "theoretical_loss": 3.6992392924161024, + "tokens_seen": 940244992 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003611634904714142, + "loss": 1.3838, + "theoretical_loss": 3.699214666365071, + "tokens_seen": 940310528 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036115346038114343, + "loss": 1.3185, + "theoretical_loss": 3.6991900425108577, + "tokens_seen": 940376064 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 316955, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3372684717178345, + "objective/train/theoretical_loss": 3.6991654208531126, + "objective/train/tokens_used": 940441600, + "theoretical_loss": 3.6991654208531126, + "tokens_seen": 940441600 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003611434302908726, + "loss": 1.2903, + "theoretical_loss": 3.6991654208531126, + "tokens_seen": 940441600 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003611334002006018, + "loss": 1.4098, + "theoretical_loss": 3.6991408013914873, + "tokens_seen": 940507136 + }, + { + "epoch": 0.29, + "learning_rate": 0.000361123370110331, + "loss": 1.4092, + "theoretical_loss": 3.6991161841256326, + "tokens_seen": 940572672 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003611133400200602, + "loss": 1.3786, + "theoretical_loss": 3.6990915690551995, + "tokens_seen": 940638208 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003611033099297894, + "loss": 1.3018, + "theoretical_loss": 3.6990669561798404, + "tokens_seen": 940703744 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003610932798395186, + "loss": 1.3597, + "theoretical_loss": 3.699042345499205, + "tokens_seen": 940769280 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036108324974924776, + "loss": 1.3355, + "theoretical_loss": 3.6990177370129462, + "tokens_seen": 940834816 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036107321965897694, + "loss": 1.4123, + "theoretical_loss": 3.698993130720714, + "tokens_seen": 940900352 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003610631895687062, + "loss": 1.3189, + "theoretical_loss": 3.6989685266221617, + "tokens_seen": 940965888 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003610531594784353, + "loss": 1.333, + "theoretical_loss": 3.69894392471694, + "tokens_seen": 941031424 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036104312938816454, + "loss": 1.3649, + "theoretical_loss": 3.6989193250047006, + "tokens_seen": 941096960 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036103309929789366, + "loss": 1.2849, + "theoretical_loss": 3.6988947274850954, + "tokens_seen": 941162496 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003610230692076229, + "loss": 1.2578, + "theoretical_loss": 3.6988701321577766, + "tokens_seen": 941228032 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003610130391173521, + "loss": 1.3401, + "theoretical_loss": 3.698845539022396, + "tokens_seen": 941293568 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036100300902708126, + "loss": 1.342, + "theoretical_loss": 3.6988209480786054, + "tokens_seen": 941359104 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036099297893681044, + "loss": 1.369, + "theoretical_loss": 3.6987963593260567, + "tokens_seen": 941424640 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003609829488465396, + "loss": 1.254, + "theoretical_loss": 3.6987717727644034, + "tokens_seen": 941490176 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003609729187562688, + "loss": 1.4086, + "theoretical_loss": 3.698747188393297, + "tokens_seen": 941555712 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036096288866599804, + "loss": 1.3436, + "theoretical_loss": 3.6987226062123897, + "tokens_seen": 941621248 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036095285857572717, + "loss": 1.3681, + "theoretical_loss": 3.698698026221334, + "tokens_seen": 941686784 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003609428284854564, + "loss": 1.2904, + "theoretical_loss": 3.698673448419783, + "tokens_seen": 941752320 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003609327983951856, + "loss": 1.3378, + "theoretical_loss": 3.6986488728073885, + "tokens_seen": 941817856 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036092276830491476, + "loss": 1.4053, + "theoretical_loss": 3.6986242993838037, + "tokens_seen": 941883392 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036091273821464394, + "loss": 1.3453, + "theoretical_loss": 3.6985997281486815, + "tokens_seen": 941948928 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003609027081243731, + "loss": 1.3675, + "theoretical_loss": 3.6985751591016744, + "tokens_seen": 942014464 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003608926780341023, + "loss": 1.3823, + "theoretical_loss": 3.6985505922424355, + "tokens_seen": 942080000 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036088264794383154, + "loss": 1.3084, + "theoretical_loss": 3.698526027570618, + "tokens_seen": 942145536 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036087261785356067, + "loss": 1.4311, + "theoretical_loss": 3.698501465085875, + "tokens_seen": 942211072 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003608625877632899, + "loss": 1.3511, + "theoretical_loss": 3.6984769047878587, + "tokens_seen": 942276608 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036085255767301903, + "loss": 1.2849, + "theoretical_loss": 3.6984523466762234, + "tokens_seen": 942342144 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036084252758274827, + "loss": 1.437, + "theoretical_loss": 3.698427790750623, + "tokens_seen": 942407680 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036083249749247745, + "loss": 1.4283, + "theoretical_loss": 3.698403237010709, + "tokens_seen": 942473216 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036082246740220663, + "loss": 1.3124, + "theoretical_loss": 3.6983786854561362, + "tokens_seen": 942538752 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003608124373119358, + "loss": 1.4366, + "theoretical_loss": 3.6983541360865586, + "tokens_seen": 942604288 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036080240722166505, + "loss": 1.3386, + "theoretical_loss": 3.6983295889016286, + "tokens_seen": 942669824 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036079237713139417, + "loss": 1.2962, + "theoretical_loss": 3.6983050439010006, + "tokens_seen": 942735360 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003607823470411234, + "loss": 1.2818, + "theoretical_loss": 3.6982805010843283, + "tokens_seen": 942800896 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036077231695085253, + "loss": 1.3585, + "theoretical_loss": 3.6982559604512657, + "tokens_seen": 942866432 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036076228686058177, + "loss": 1.2945, + "theoretical_loss": 3.6982314220014665, + "tokens_seen": 942931968 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036075225677031095, + "loss": 1.3662, + "theoretical_loss": 3.698206885734585, + "tokens_seen": 942997504 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036074222668004013, + "loss": 1.3662, + "theoretical_loss": 3.6981823516502756, + "tokens_seen": 943063040 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003607321965897693, + "loss": 1.2688, + "theoretical_loss": 3.698157819748192, + "tokens_seen": 943128576 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003607221664994985, + "loss": 1.3522, + "theoretical_loss": 3.6981332900279886, + "tokens_seen": 943194112 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003607121364092277, + "loss": 1.3035, + "theoretical_loss": 3.698108762489319, + "tokens_seen": 943259648 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003607021063189569, + "loss": 1.372, + "theoretical_loss": 3.6980842371318396, + "tokens_seen": 943325184 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036069207622868604, + "loss": 1.3254, + "theoretical_loss": 3.698059713955203, + "tokens_seen": 943390720 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036068204613841527, + "loss": 1.3264, + "theoretical_loss": 3.6980351929590647, + "tokens_seen": 943456256 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003606720160481444, + "loss": 1.3958, + "theoretical_loss": 3.6980106741430796, + "tokens_seen": 943521792 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036066198595787364, + "loss": 1.4602, + "theoretical_loss": 3.697986157506902, + "tokens_seen": 943587328 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003606519558676028, + "loss": 1.3052, + "theoretical_loss": 3.697961643050186, + "tokens_seen": 943652864 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 318213, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3625022172927856, + "objective/train/theoretical_loss": 3.697937130772588, + "objective/train/tokens_used": 943718400, + "theoretical_loss": 3.697937130772588, + "tokens_seen": 943718400 + }, + { + "epoch": 0.29, + "learning_rate": 0.000360641925777332, + "loss": 1.3619, + "theoretical_loss": 3.697937130772588, + "tokens_seen": 943718400 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003606318956870612, + "loss": 1.4355, + "theoretical_loss": 3.697912620673762, + "tokens_seen": 943783936 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003606218655967904, + "loss": 1.3147, + "theoretical_loss": 3.6978881127533634, + "tokens_seen": 943849472 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036061183550651954, + "loss": 1.4448, + "theoretical_loss": 3.6978636070110475, + "tokens_seen": 943915008 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003606018054162488, + "loss": 1.2676, + "theoretical_loss": 3.6978391034464693, + "tokens_seen": 943980544 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003605917753259779, + "loss": 1.2887, + "theoretical_loss": 3.6978146020592835, + "tokens_seen": 944046080 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036058174523570714, + "loss": 1.3622, + "theoretical_loss": 3.697790102849147, + "tokens_seen": 944111616 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003605717151454363, + "loss": 1.351, + "theoretical_loss": 3.6977656058157136, + "tokens_seen": 944177152 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003605616850551655, + "loss": 1.332, + "theoretical_loss": 3.69774111095864, + "tokens_seen": 944242688 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003605516549648947, + "loss": 1.3743, + "theoretical_loss": 3.697716618277582, + "tokens_seen": 944308224 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036054162487462386, + "loss": 1.4211, + "theoretical_loss": 3.6976921277721937, + "tokens_seen": 944373760 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036053159478435304, + "loss": 1.3162, + "theoretical_loss": 3.697667639442132, + "tokens_seen": 944439296 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003605215646940823, + "loss": 1.3029, + "theoretical_loss": 3.6976431532870535, + "tokens_seen": 944504832 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003605115346038114, + "loss": 1.3321, + "theoretical_loss": 3.697618669306612, + "tokens_seen": 944570368 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036050150451354064, + "loss": 1.3234, + "theoretical_loss": 3.697594187500466, + "tokens_seen": 944635904 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036049147442326977, + "loss": 1.2696, + "theoretical_loss": 3.6975697078682694, + "tokens_seen": 944701440 + }, + { + "epoch": 0.29, + "learning_rate": 0.000360481444332999, + "loss": 1.377, + "theoretical_loss": 3.69754523040968, + "tokens_seen": 944766976 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003604714142427282, + "loss": 1.3667, + "theoretical_loss": 3.697520755124353, + "tokens_seen": 944832512 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036046138415245737, + "loss": 1.4482, + "theoretical_loss": 3.6974962820119455, + "tokens_seen": 944898048 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036045135406218655, + "loss": 1.4804, + "theoretical_loss": 3.697471811072113, + "tokens_seen": 944963584 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003604413239719158, + "loss": 1.3836, + "theoretical_loss": 3.697447342304513, + "tokens_seen": 945029120 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003604312938816449, + "loss": 1.3939, + "theoretical_loss": 3.697422875708801, + "tokens_seen": 945094656 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036042126379137414, + "loss": 1.3172, + "theoretical_loss": 3.6973984112846345, + "tokens_seen": 945160192 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036041123370110327, + "loss": 1.3326, + "theoretical_loss": 3.69737394903167, + "tokens_seen": 945225728 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003604012036108325, + "loss": 1.3676, + "theoretical_loss": 3.697349488949564, + "tokens_seen": 945291264 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003603911735205617, + "loss": 1.436, + "theoretical_loss": 3.6973250310379733, + "tokens_seen": 945356800 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036038114343029087, + "loss": 1.4593, + "theoretical_loss": 3.697300575296555, + "tokens_seen": 945422336 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036037111334002005, + "loss": 1.2988, + "theoretical_loss": 3.6972761217249666, + "tokens_seen": 945487872 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036036108324974923, + "loss": 1.4073, + "theoretical_loss": 3.6972516703228644, + "tokens_seen": 945553408 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036035105315947847, + "loss": 1.3669, + "theoretical_loss": 3.697227221089906, + "tokens_seen": 945618944 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036034102306920765, + "loss": 1.4139, + "theoretical_loss": 3.697202774025749, + "tokens_seen": 945684480 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036033099297893683, + "loss": 1.2772, + "theoretical_loss": 3.6971783291300504, + "tokens_seen": 945750016 + }, + { + "epoch": 0.29, + "learning_rate": 0.000360320962888666, + "loss": 1.3093, + "theoretical_loss": 3.6971538864024676, + "tokens_seen": 945815552 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036031093279839525, + "loss": 1.3264, + "theoretical_loss": 3.697129445842658, + "tokens_seen": 945881088 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036030090270812437, + "loss": 1.3583, + "theoretical_loss": 3.697105007450279, + "tokens_seen": 945946624 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003602908726178536, + "loss": 1.295, + "theoretical_loss": 3.697080571224989, + "tokens_seen": 946012160 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036028084252758273, + "loss": 1.3722, + "theoretical_loss": 3.6970561371664443, + "tokens_seen": 946077696 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036027081243731197, + "loss": 1.3569, + "theoretical_loss": 3.6970317052743042, + "tokens_seen": 946143232 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036026078234704115, + "loss": 1.2893, + "theoretical_loss": 3.697007275548226, + "tokens_seen": 946208768 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036025075225677033, + "loss": 1.3649, + "theoretical_loss": 3.6969828479878677, + "tokens_seen": 946274304 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003602407221664995, + "loss": 1.2893, + "theoretical_loss": 3.6969584225928873, + "tokens_seen": 946339840 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003602306920762287, + "loss": 1.2709, + "theoretical_loss": 3.696933999362943, + "tokens_seen": 946405376 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003602206619859579, + "loss": 1.3337, + "theoretical_loss": 3.696909578297692, + "tokens_seen": 946470912 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003602106318956871, + "loss": 1.337, + "theoretical_loss": 3.6968851593967944, + "tokens_seen": 946536448 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036020060180541624, + "loss": 1.3293, + "theoretical_loss": 3.696860742659908, + "tokens_seen": 946601984 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003601905717151455, + "loss": 1.3849, + "theoretical_loss": 3.6968363280866896, + "tokens_seen": 946667520 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003601805416248746, + "loss": 1.4319, + "theoretical_loss": 3.6968119156768, + "tokens_seen": 946733056 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036017051153460384, + "loss": 1.4266, + "theoretical_loss": 3.696787505429896, + "tokens_seen": 946798592 + }, + { + "epoch": 0.29, + "learning_rate": 0.000360160481444333, + "loss": 1.2716, + "theoretical_loss": 3.6967630973456376, + "tokens_seen": 946864128 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003601504513540622, + "loss": 1.3706, + "theoretical_loss": 3.6967386914236826, + "tokens_seen": 946929664 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 319004, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.291743516921997, + "objective/train/theoretical_loss": 3.6967142876636903, + "objective/train/tokens_used": 946995200, + "theoretical_loss": 3.6967142876636903, + "tokens_seen": 946995200 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003601404212637914, + "loss": 1.2996, + "theoretical_loss": 3.6967142876636903, + "tokens_seen": 946995200 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003601303911735206, + "loss": 1.2191, + "theoretical_loss": 3.696689886065319, + "tokens_seen": 947060736 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036012036108324974, + "loss": 1.3452, + "theoretical_loss": 3.6966654866282282, + "tokens_seen": 947126272 + }, + { + "epoch": 0.29, + "learning_rate": 0.000360110330992979, + "loss": 1.3518, + "theoretical_loss": 3.6966410893520774, + "tokens_seen": 947191808 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003601003009027081, + "loss": 1.3674, + "theoretical_loss": 3.696616694236524, + "tokens_seen": 947257344 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036009027081243734, + "loss": 1.3842, + "theoretical_loss": 3.696592301281229, + "tokens_seen": 947322880 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003600802407221665, + "loss": 1.4555, + "theoretical_loss": 3.6965679104858515, + "tokens_seen": 947388416 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003600702106318957, + "loss": 1.3252, + "theoretical_loss": 3.6965435218500495, + "tokens_seen": 947453952 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003600601805416249, + "loss": 1.4079, + "theoretical_loss": 3.6965191353734843, + "tokens_seen": 947519488 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036005015045135406, + "loss": 1.3925, + "theoretical_loss": 3.6964947510558135, + "tokens_seen": 947585024 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036004012036108324, + "loss": 1.3562, + "theoretical_loss": 3.6964703688966978, + "tokens_seen": 947650560 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003600300902708125, + "loss": 1.3991, + "theoretical_loss": 3.696445988895797, + "tokens_seen": 947716096 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003600200601805416, + "loss": 1.3293, + "theoretical_loss": 3.69642161105277, + "tokens_seen": 947781632 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036001003009027084, + "loss": 1.2703, + "theoretical_loss": 3.6963972353672774, + "tokens_seen": 947847168 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035999999999999997, + "loss": 1.3313, + "theoretical_loss": 3.696372861838979, + "tokens_seen": 947912704 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003599899699097292, + "loss": 1.3885, + "theoretical_loss": 3.696348490467534, + "tokens_seen": 947978240 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003599799398194584, + "loss": 1.3628, + "theoretical_loss": 3.6963241212526032, + "tokens_seen": 948043776 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035996990972918757, + "loss": 1.3511, + "theoretical_loss": 3.6962997541938467, + "tokens_seen": 948109312 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035995987963891675, + "loss": 1.3696, + "theoretical_loss": 3.696275389290925, + "tokens_seen": 948174848 + }, + { + "epoch": 0.29, + "learning_rate": 0.000359949849548646, + "loss": 1.413, + "theoretical_loss": 3.6962510265434974, + "tokens_seen": 948240384 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003599398194583751, + "loss": 1.3852, + "theoretical_loss": 3.6962266659512246, + "tokens_seen": 948305920 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035992978936810434, + "loss": 1.3615, + "theoretical_loss": 3.6962023075137673, + "tokens_seen": 948371456 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035991975927783347, + "loss": 1.376, + "theoretical_loss": 3.696177951230786, + "tokens_seen": 948436992 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003599097291875627, + "loss": 1.3698, + "theoretical_loss": 3.6961535971019415, + "tokens_seen": 948502528 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003598996990972919, + "loss": 1.4226, + "theoretical_loss": 3.696129245126894, + "tokens_seen": 948568064 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035988966900702107, + "loss": 1.3434, + "theoretical_loss": 3.6961048953053046, + "tokens_seen": 948633600 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035987963891675025, + "loss": 1.3389, + "theoretical_loss": 3.6960805476368335, + "tokens_seen": 948699136 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035986960882647943, + "loss": 1.2888, + "theoretical_loss": 3.696056202121142, + "tokens_seen": 948764672 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003598595787362086, + "loss": 1.4383, + "theoretical_loss": 3.696031858757891, + "tokens_seen": 948830208 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035984954864593785, + "loss": 1.5667, + "theoretical_loss": 3.696007517546742, + "tokens_seen": 948895744 + }, + { + "epoch": 0.29, + "learning_rate": 0.000359839518555667, + "loss": 1.3395, + "theoretical_loss": 3.6959831784873565, + "tokens_seen": 948961280 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003598294884653962, + "loss": 1.5085, + "theoretical_loss": 3.695958841579394, + "tokens_seen": 949026816 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035981945837512534, + "loss": 1.2631, + "theoretical_loss": 3.6959345068225167, + "tokens_seen": 949092352 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035980942828485457, + "loss": 1.4495, + "theoretical_loss": 3.6959101742163867, + "tokens_seen": 949157888 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035979939819458375, + "loss": 1.3009, + "theoretical_loss": 3.695885843760664, + "tokens_seen": 949223424 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035978936810431293, + "loss": 1.3763, + "theoretical_loss": 3.695861515455011, + "tokens_seen": 949288960 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003597793380140421, + "loss": 1.3739, + "theoretical_loss": 3.6958371892990893, + "tokens_seen": 949354496 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035976930792377135, + "loss": 1.3293, + "theoretical_loss": 3.695812865292561, + "tokens_seen": 949420032 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003597592778335005, + "loss": 1.3092, + "theoretical_loss": 3.695788543435086, + "tokens_seen": 949485568 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003597492477432297, + "loss": 1.3757, + "theoretical_loss": 3.695764223726328, + "tokens_seen": 949551104 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035973921765295884, + "loss": 1.3533, + "theoretical_loss": 3.695739906165948, + "tokens_seen": 949616640 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003597291875626881, + "loss": 1.3682, + "theoretical_loss": 3.6957155907536086, + "tokens_seen": 949682176 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035971915747241726, + "loss": 1.2495, + "theoretical_loss": 3.6956912774889714, + "tokens_seen": 949747712 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035970912738214644, + "loss": 1.3186, + "theoretical_loss": 3.6956669663716983, + "tokens_seen": 949813248 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003596990972918756, + "loss": 1.4976, + "theoretical_loss": 3.695642657401452, + "tokens_seen": 949878784 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003596890672016048, + "loss": 1.4061, + "theoretical_loss": 3.6956183505778943, + "tokens_seen": 949944320 + }, + { + "epoch": 0.29, + "learning_rate": 0.000359679037111334, + "loss": 1.2981, + "theoretical_loss": 3.6955940459006884, + "tokens_seen": 950009856 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003596690070210632, + "loss": 1.4154, + "theoretical_loss": 3.6955697433694956, + "tokens_seen": 950075392 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035965897693079234, + "loss": 1.376, + "theoretical_loss": 3.6955454429839785, + "tokens_seen": 950140928 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003596489468405216, + "loss": 1.3863, + "theoretical_loss": 3.695521144743801, + "tokens_seen": 950206464 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 320228, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3423049449920654, + "objective/train/theoretical_loss": 3.695496848648624, + "objective/train/tokens_used": 950272000, + "theoretical_loss": 3.695496848648624, + "tokens_seen": 950272000 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035963891675025076, + "loss": 1.3507, + "theoretical_loss": 3.695496848648624, + "tokens_seen": 950272000 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035962888665997994, + "loss": 1.3764, + "theoretical_loss": 3.695472554698112, + "tokens_seen": 950337536 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003596188565697091, + "loss": 1.2548, + "theoretical_loss": 3.695448262891926, + "tokens_seen": 950403072 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003596088264794383, + "loss": 1.2871, + "theoretical_loss": 3.69542397322973, + "tokens_seen": 950468608 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035959879638916754, + "loss": 1.3722, + "theoretical_loss": 3.6953996857111875, + "tokens_seen": 950534144 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003595887662988967, + "loss": 1.342, + "theoretical_loss": 3.69537540033596, + "tokens_seen": 950599680 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003595787362086259, + "loss": 1.3018, + "theoretical_loss": 3.6953511171037112, + "tokens_seen": 950665216 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003595687061183551, + "loss": 1.3203, + "theoretical_loss": 3.6953268360141047, + "tokens_seen": 950730752 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035955867602808426, + "loss": 1.4245, + "theoretical_loss": 3.695302557066804, + "tokens_seen": 950796288 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035954864593781344, + "loss": 1.2239, + "theoretical_loss": 3.695278280261472, + "tokens_seen": 950861824 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003595386158475427, + "loss": 1.2791, + "theoretical_loss": 3.6952540055977714, + "tokens_seen": 950927360 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003595285857572718, + "loss": 1.3972, + "theoretical_loss": 3.695229733075367, + "tokens_seen": 950992896 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035951855566700104, + "loss": 1.2998, + "theoretical_loss": 3.695205462693922, + "tokens_seen": 951058432 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035950852557673017, + "loss": 1.3938, + "theoretical_loss": 3.695181194453099, + "tokens_seen": 951123968 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003594984954864594, + "loss": 1.2849, + "theoretical_loss": 3.6951569283525627, + "tokens_seen": 951189504 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003594884653961886, + "loss": 1.2877, + "theoretical_loss": 3.695132664391977, + "tokens_seen": 951255040 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035947843530591777, + "loss": 1.36, + "theoretical_loss": 3.6951084025710053, + "tokens_seen": 951320576 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035946840521564695, + "loss": 1.4218, + "theoretical_loss": 3.695084142889312, + "tokens_seen": 951386112 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003594583751253762, + "loss": 1.3281, + "theoretical_loss": 3.695059885346561, + "tokens_seen": 951451648 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003594483450351053, + "loss": 1.3897, + "theoretical_loss": 3.6950356299424163, + "tokens_seen": 951517184 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035943831494483454, + "loss": 1.2743, + "theoretical_loss": 3.6950113766765416, + "tokens_seen": 951582720 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035942828485456367, + "loss": 1.3624, + "theoretical_loss": 3.694987125548602, + "tokens_seen": 951648256 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003594182547642929, + "loss": 1.4036, + "theoretical_loss": 3.694962876558261, + "tokens_seen": 951713792 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003594082246740221, + "loss": 1.3191, + "theoretical_loss": 3.6949386297051836, + "tokens_seen": 951779328 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035939819458375127, + "loss": 1.445, + "theoretical_loss": 3.6949143849890342, + "tokens_seen": 951844864 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035938816449348045, + "loss": 1.3322, + "theoretical_loss": 3.6948901424094776, + "tokens_seen": 951910400 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035937813440320963, + "loss": 1.387, + "theoretical_loss": 3.6948659019661783, + "tokens_seen": 951975936 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003593681043129388, + "loss": 1.3516, + "theoretical_loss": 3.6948416636588, + "tokens_seen": 952041472 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035935807422266805, + "loss": 1.5193, + "theoretical_loss": 3.694817427487009, + "tokens_seen": 952107008 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003593480441323972, + "loss": 1.5026, + "theoretical_loss": 3.6947931934504687, + "tokens_seen": 952172544 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003593380140421264, + "loss": 1.3609, + "theoretical_loss": 3.6947689615488457, + "tokens_seen": 952238080 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035932798395185554, + "loss": 1.3841, + "theoretical_loss": 3.6947447317818036, + "tokens_seen": 952303616 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035931795386158477, + "loss": 1.2979, + "theoretical_loss": 3.694720504149008, + "tokens_seen": 952369152 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035930792377131395, + "loss": 1.4167, + "theoretical_loss": 3.694696278650124, + "tokens_seen": 952434688 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035929789368104313, + "loss": 1.2523, + "theoretical_loss": 3.6946720552848173, + "tokens_seen": 952500224 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003592878635907723, + "loss": 1.431, + "theoretical_loss": 3.694647834052753, + "tokens_seen": 952565760 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035927783350050155, + "loss": 1.3808, + "theoretical_loss": 3.6946236149535956, + "tokens_seen": 952631296 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003592678034102307, + "loss": 1.358, + "theoretical_loss": 3.6945993979870115, + "tokens_seen": 952696832 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003592577733199599, + "loss": 1.3594, + "theoretical_loss": 3.694575183152666, + "tokens_seen": 952762368 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035924774322968904, + "loss": 1.3515, + "theoretical_loss": 3.6945509704502246, + "tokens_seen": 952827904 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003592377131394183, + "loss": 1.4234, + "theoretical_loss": 3.6945267598793534, + "tokens_seen": 952893440 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035922768304914746, + "loss": 1.393, + "theoretical_loss": 3.694502551439718, + "tokens_seen": 952958976 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035921765295887664, + "loss": 1.2994, + "theoretical_loss": 3.6944783451309835, + "tokens_seen": 953024512 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003592076228686058, + "loss": 1.4339, + "theoretical_loss": 3.694454140952817, + "tokens_seen": 953090048 + }, + { + "epoch": 0.29, + "learning_rate": 0.000359197592778335, + "loss": 1.3161, + "theoretical_loss": 3.694429938904884, + "tokens_seen": 953155584 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003591875626880642, + "loss": 1.4006, + "theoretical_loss": 3.69440573898685, + "tokens_seen": 953221120 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003591775325977934, + "loss": 1.3511, + "theoretical_loss": 3.694381541198382, + "tokens_seen": 953286656 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035916750250752254, + "loss": 1.4087, + "theoretical_loss": 3.694357345539146, + "tokens_seen": 953352192 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003591574724172518, + "loss": 1.3127, + "theoretical_loss": 3.6943331520088076, + "tokens_seen": 953417728 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035914744232698096, + "loss": 1.4097, + "theoretical_loss": 3.694308960607034, + "tokens_seen": 953483264 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 321167, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3546267747879028, + "objective/train/theoretical_loss": 3.694284771333491, + "objective/train/tokens_used": 953548800, + "theoretical_loss": 3.694284771333491, + "tokens_seen": 953548800 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035913741223671014, + "loss": 1.3416, + "theoretical_loss": 3.694284771333491, + "tokens_seen": 953548800 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003591273821464393, + "loss": 1.3487, + "theoretical_loss": 3.6942605841878464, + "tokens_seen": 953614336 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003591173520561685, + "loss": 1.4244, + "theoretical_loss": 3.694236399169765, + "tokens_seen": 953679872 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003591073219658977, + "loss": 1.4943, + "theoretical_loss": 3.6942122162789146, + "tokens_seen": 953745408 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003590972918756269, + "loss": 1.3757, + "theoretical_loss": 3.694188035514962, + "tokens_seen": 953810944 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035908726178535605, + "loss": 1.324, + "theoretical_loss": 3.694163856877573, + "tokens_seen": 953876480 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003590772316950853, + "loss": 1.3163, + "theoretical_loss": 3.6941396803664155, + "tokens_seen": 953942016 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003590672016048144, + "loss": 1.2261, + "theoretical_loss": 3.6941155059811566, + "tokens_seen": 954007552 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035905717151454364, + "loss": 1.281, + "theoretical_loss": 3.6940913337214627, + "tokens_seen": 954073088 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003590471414242728, + "loss": 1.3859, + "theoretical_loss": 3.6940671635870013, + "tokens_seen": 954138624 + }, + { + "epoch": 0.29, + "learning_rate": 0.000359037111334002, + "loss": 1.2307, + "theoretical_loss": 3.6940429955774388, + "tokens_seen": 954204160 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003590270812437312, + "loss": 1.3648, + "theoretical_loss": 3.6940188296924434, + "tokens_seen": 954269696 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035901705115346037, + "loss": 1.3866, + "theoretical_loss": 3.6939946659316822, + "tokens_seen": 954335232 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035900702106318955, + "loss": 1.4427, + "theoretical_loss": 3.693970504294823, + "tokens_seen": 954400768 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003589969909729188, + "loss": 1.3147, + "theoretical_loss": 3.6939463447815317, + "tokens_seen": 954466304 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003589869608826479, + "loss": 1.4057, + "theoretical_loss": 3.693922187391478, + "tokens_seen": 954531840 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035897693079237715, + "loss": 1.4301, + "theoretical_loss": 3.6938980321243284, + "tokens_seen": 954597376 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035896690070210633, + "loss": 1.3069, + "theoretical_loss": 3.6938738789797507, + "tokens_seen": 954662912 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003589568706118355, + "loss": 1.3407, + "theoretical_loss": 3.6938497279574127, + "tokens_seen": 954728448 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003589468405215647, + "loss": 1.3454, + "theoretical_loss": 3.6938255790569823, + "tokens_seen": 954793984 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035893681043129387, + "loss": 1.4055, + "theoretical_loss": 3.6938014322781276, + "tokens_seen": 954859520 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035892678034102305, + "loss": 1.3224, + "theoretical_loss": 3.6937772876205166, + "tokens_seen": 954925056 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003589167502507523, + "loss": 1.3574, + "theoretical_loss": 3.6937531450838175, + "tokens_seen": 954990592 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003589067201604814, + "loss": 1.4576, + "theoretical_loss": 3.6937290046676976, + "tokens_seen": 955056128 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035889669007021065, + "loss": 1.3984, + "theoretical_loss": 3.693704866371826, + "tokens_seen": 955121664 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003588866599799398, + "loss": 1.3243, + "theoretical_loss": 3.6936807301958714, + "tokens_seen": 955187200 + }, + { + "epoch": 0.29, + "learning_rate": 0.000358876629889669, + "loss": 1.3746, + "theoretical_loss": 3.693656596139501, + "tokens_seen": 955252736 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003588665997993982, + "loss": 1.333, + "theoretical_loss": 3.693632464202384, + "tokens_seen": 955318272 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003588565697091274, + "loss": 1.2879, + "theoretical_loss": 3.693608334384189, + "tokens_seen": 955383808 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003588465396188566, + "loss": 1.3697, + "theoretical_loss": 3.6935842066845845, + "tokens_seen": 955449344 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035883650952858574, + "loss": 1.3667, + "theoretical_loss": 3.6935600811032394, + "tokens_seen": 955514880 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035882647943831497, + "loss": 1.4225, + "theoretical_loss": 3.6935359576398215, + "tokens_seen": 955580416 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035881644934804415, + "loss": 1.3087, + "theoretical_loss": 3.6935118362940007, + "tokens_seen": 955645952 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035880641925777333, + "loss": 1.3457, + "theoretical_loss": 3.6934877170654454, + "tokens_seen": 955711488 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003587963891675025, + "loss": 1.4292, + "theoretical_loss": 3.693463599953825, + "tokens_seen": 955777024 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035878635907723175, + "loss": 1.3514, + "theoretical_loss": 3.693439484958808, + "tokens_seen": 955842560 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003587763289869609, + "loss": 1.4847, + "theoretical_loss": 3.693415372080064, + "tokens_seen": 955908096 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003587662988966901, + "loss": 1.4384, + "theoretical_loss": 3.693391261317262, + "tokens_seen": 955973632 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035875626880641924, + "loss": 1.3728, + "theoretical_loss": 3.693367152670071, + "tokens_seen": 956039168 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003587462387161485, + "loss": 1.2445, + "theoretical_loss": 3.6933430461381613, + "tokens_seen": 956104704 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035873620862587766, + "loss": 1.3473, + "theoretical_loss": 3.693318941721201, + "tokens_seen": 956170240 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035872617853560684, + "loss": 1.3487, + "theoretical_loss": 3.693294839418861, + "tokens_seen": 956235776 + }, + { + "epoch": 0.29, + "learning_rate": 0.000358716148445336, + "loss": 1.2703, + "theoretical_loss": 3.69327073923081, + "tokens_seen": 956301312 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003587061183550652, + "loss": 1.3927, + "theoretical_loss": 3.6932466411567173, + "tokens_seen": 956366848 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003586960882647944, + "loss": 1.3113, + "theoretical_loss": 3.693222545196254, + "tokens_seen": 956432384 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003586860581745236, + "loss": 1.4608, + "theoretical_loss": 3.693198451349089, + "tokens_seen": 956497920 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035867602808425274, + "loss": 1.437, + "theoretical_loss": 3.6931743596148916, + "tokens_seen": 956563456 + }, + { + "epoch": 0.29, + "learning_rate": 0.000358665997993982, + "loss": 1.4113, + "theoretical_loss": 3.6931502699933336, + "tokens_seen": 956628992 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035865596790371116, + "loss": 1.3236, + "theoretical_loss": 3.693126182484083, + "tokens_seen": 956694528 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035864593781344034, + "loss": 1.4206, + "theoretical_loss": 3.693102097086811, + "tokens_seen": 956760064 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 322381, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4039580821990967, + "objective/train/theoretical_loss": 3.693078013801187, + "objective/train/tokens_used": 956825600, + "theoretical_loss": 3.693078013801187, + "tokens_seen": 956825600 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003586359077231695, + "loss": 1.4045, + "theoretical_loss": 3.693078013801187, + "tokens_seen": 956825600 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003586258776328987, + "loss": 1.368, + "theoretical_loss": 3.6930539326268828, + "tokens_seen": 956891136 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003586158475426279, + "loss": 1.3704, + "theoretical_loss": 3.6930298535635666, + "tokens_seen": 956956672 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003586058174523571, + "loss": 1.3806, + "theoretical_loss": 3.693005776610911, + "tokens_seen": 957022208 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035859578736208625, + "loss": 1.335, + "theoretical_loss": 3.692981701768585, + "tokens_seen": 957087744 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003585857572718155, + "loss": 1.4409, + "theoretical_loss": 3.6929576290362593, + "tokens_seen": 957153280 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003585757271815446, + "loss": 1.3899, + "theoretical_loss": 3.6929335584136047, + "tokens_seen": 957218816 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035856569709127384, + "loss": 1.3386, + "theoretical_loss": 3.692909489900292, + "tokens_seen": 957284352 + }, + { + "epoch": 0.29, + "learning_rate": 0.000358555667001003, + "loss": 1.3583, + "theoretical_loss": 3.6928854234959925, + "tokens_seen": 957349888 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003585456369107322, + "loss": 1.4491, + "theoretical_loss": 3.692861359200376, + "tokens_seen": 957415424 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003585356068204614, + "loss": 1.4074, + "theoretical_loss": 3.6928372970131136, + "tokens_seen": 957480960 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035852557673019057, + "loss": 1.3516, + "theoretical_loss": 3.692813236933877, + "tokens_seen": 957546496 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035851554663991975, + "loss": 1.3346, + "theoretical_loss": 3.692789178962337, + "tokens_seen": 957612032 + }, + { + "epoch": 0.29, + "learning_rate": 0.000358505516549649, + "loss": 1.3588, + "theoretical_loss": 3.692765123098164, + "tokens_seen": 957677568 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003584954864593781, + "loss": 1.3776, + "theoretical_loss": 3.69274106934103, + "tokens_seen": 957743104 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035848545636910735, + "loss": 1.3245, + "theoretical_loss": 3.6927170176906063, + "tokens_seen": 957808640 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035847542627883653, + "loss": 1.4084, + "theoretical_loss": 3.692692968146564, + "tokens_seen": 957874176 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003584653961885657, + "loss": 1.2836, + "theoretical_loss": 3.692668920708574, + "tokens_seen": 957939712 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003584553660982949, + "loss": 1.3626, + "theoretical_loss": 3.692644875376309, + "tokens_seen": 958005248 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035844533600802407, + "loss": 1.3637, + "theoretical_loss": 3.6926208321494394, + "tokens_seen": 958070784 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035843530591775325, + "loss": 1.3024, + "theoretical_loss": 3.692596791027638, + "tokens_seen": 958136320 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003584252758274825, + "loss": 1.3744, + "theoretical_loss": 3.6925727520105753, + "tokens_seen": 958201856 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003584152457372116, + "loss": 1.2509, + "theoretical_loss": 3.692548715097924, + "tokens_seen": 958267392 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035840521564694085, + "loss": 1.4199, + "theoretical_loss": 3.6925246802893557, + "tokens_seen": 958332928 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035839518555667, + "loss": 1.2871, + "theoretical_loss": 3.6925006475845423, + "tokens_seen": 958398464 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003583851554663992, + "loss": 1.3005, + "theoretical_loss": 3.692476616983156, + "tokens_seen": 958464000 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003583751253761284, + "loss": 1.39, + "theoretical_loss": 3.6924525884848687, + "tokens_seen": 958529536 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003583650952858576, + "loss": 1.3886, + "theoretical_loss": 3.6924285620893524, + "tokens_seen": 958595072 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035835506519558676, + "loss": 1.3619, + "theoretical_loss": 3.69240453779628, + "tokens_seen": 958660608 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035834503510531594, + "loss": 1.3986, + "theoretical_loss": 3.692380515605323, + "tokens_seen": 958726144 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003583350050150451, + "loss": 1.3946, + "theoretical_loss": 3.6923564955161545, + "tokens_seen": 958791680 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035832497492477435, + "loss": 1.4232, + "theoretical_loss": 3.6923324775284465, + "tokens_seen": 958857216 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003583149448345035, + "loss": 1.4157, + "theoretical_loss": 3.692308461641872, + "tokens_seen": 958922752 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003583049147442327, + "loss": 1.3285, + "theoretical_loss": 3.6922844478561028, + "tokens_seen": 958988288 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003582948846539619, + "loss": 1.3556, + "theoretical_loss": 3.6922604361708125, + "tokens_seen": 959053824 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003582848545636911, + "loss": 1.3332, + "theoretical_loss": 3.692236426585673, + "tokens_seen": 959119360 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035827482447342026, + "loss": 1.3191, + "theoretical_loss": 3.6922124191003576, + "tokens_seen": 959184896 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035826479438314944, + "loss": 1.381, + "theoretical_loss": 3.6921884137145393, + "tokens_seen": 959250432 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003582547642928786, + "loss": 1.3956, + "theoretical_loss": 3.692164410427891, + "tokens_seen": 959315968 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035824473420260786, + "loss": 1.2591, + "theoretical_loss": 3.6921404092400856, + "tokens_seen": 959381504 + }, + { + "epoch": 0.29, + "learning_rate": 0.000358234704112337, + "loss": 1.4268, + "theoretical_loss": 3.6921164101507964, + "tokens_seen": 959447040 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003582246740220662, + "loss": 1.3084, + "theoretical_loss": 3.6920924131596964, + "tokens_seen": 959512576 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035821464393179535, + "loss": 1.379, + "theoretical_loss": 3.692068418266459, + "tokens_seen": 959578112 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003582046138415246, + "loss": 1.4937, + "theoretical_loss": 3.6920444254707574, + "tokens_seen": 959643648 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035819458375125376, + "loss": 1.4467, + "theoretical_loss": 3.6920204347722656, + "tokens_seen": 959709184 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035818455366098294, + "loss": 1.4104, + "theoretical_loss": 3.6919964461706565, + "tokens_seen": 959774720 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003581745235707121, + "loss": 1.4459, + "theoretical_loss": 3.691972459665603, + "tokens_seen": 959840256 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035816449348044136, + "loss": 1.3682, + "theoretical_loss": 3.6919484752567806, + "tokens_seen": 959905792 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003581544633901705, + "loss": 1.3802, + "theoretical_loss": 3.691924492943861, + "tokens_seen": 959971328 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003581444332998997, + "loss": 1.4875, + "theoretical_loss": 3.6919005127265194, + "tokens_seen": 960036864 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 323635, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4951250553131104, + "objective/train/theoretical_loss": 3.6918765346044298, + "objective/train/tokens_used": 960102400, + "theoretical_loss": 3.6918765346044298, + "tokens_seen": 960102400 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035813440320962885, + "loss": 1.368, + "theoretical_loss": 3.6918765346044298, + "tokens_seen": 960102400 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003581243731193581, + "loss": 1.3906, + "theoretical_loss": 3.6918525585772644, + "tokens_seen": 960167936 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035811434302908727, + "loss": 1.3225, + "theoretical_loss": 3.691828584644699, + "tokens_seen": 960233472 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035810431293881645, + "loss": 1.2728, + "theoretical_loss": 3.6918046128064064, + "tokens_seen": 960299008 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003580942828485457, + "loss": 1.422, + "theoretical_loss": 3.691780643062062, + "tokens_seen": 960364544 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003580842527582748, + "loss": 1.3581, + "theoretical_loss": 3.6917566754113387, + "tokens_seen": 960430080 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035807422266800404, + "loss": 1.3694, + "theoretical_loss": 3.6917327098539117, + "tokens_seen": 960495616 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003580641925777332, + "loss": 1.3871, + "theoretical_loss": 3.6917087463894553, + "tokens_seen": 960561152 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003580541624874624, + "loss": 1.3814, + "theoretical_loss": 3.6916847850176433, + "tokens_seen": 960626688 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003580441323971916, + "loss": 1.4045, + "theoretical_loss": 3.6916608257381505, + "tokens_seen": 960692224 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035803410230692077, + "loss": 1.3326, + "theoretical_loss": 3.6916368685506518, + "tokens_seen": 960757760 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035802407221664995, + "loss": 1.3537, + "theoretical_loss": 3.691612913454822, + "tokens_seen": 960823296 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003580140421263792, + "loss": 1.3674, + "theoretical_loss": 3.6915889604503356, + "tokens_seen": 960888832 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003580040120361083, + "loss": 1.3351, + "theoretical_loss": 3.6915650095368666, + "tokens_seen": 960954368 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035799398194583755, + "loss": 1.3852, + "theoretical_loss": 3.6915410607140915, + "tokens_seen": 961019904 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035798395185556673, + "loss": 1.3974, + "theoretical_loss": 3.6915171139816834, + "tokens_seen": 961085440 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003579739217652959, + "loss": 1.3485, + "theoretical_loss": 3.691493169339319, + "tokens_seen": 961150976 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003579638916750251, + "loss": 1.3413, + "theoretical_loss": 3.691469226786672, + "tokens_seen": 961216512 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035795386158475427, + "loss": 1.4448, + "theoretical_loss": 3.6914452863234186, + "tokens_seen": 961282048 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035794383149448345, + "loss": 1.4113, + "theoretical_loss": 3.691421347949234, + "tokens_seen": 961347584 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003579338014042127, + "loss": 1.2716, + "theoretical_loss": 3.691397411663792, + "tokens_seen": 961413120 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003579237713139418, + "loss": 1.3471, + "theoretical_loss": 3.6913734774667706, + "tokens_seen": 961478656 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035791374122367105, + "loss": 1.3471, + "theoretical_loss": 3.691349545357843, + "tokens_seen": 961544192 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003579037111334002, + "loss": 1.4204, + "theoretical_loss": 3.6913256153366856, + "tokens_seen": 961609728 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003578936810431294, + "loss": 1.4663, + "theoretical_loss": 3.691301687402974, + "tokens_seen": 961675264 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003578836509528586, + "loss": 1.3766, + "theoretical_loss": 3.6912777615563837, + "tokens_seen": 961740800 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003578736208625878, + "loss": 1.3248, + "theoretical_loss": 3.69125383779659, + "tokens_seen": 961806336 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035786359077231696, + "loss": 1.3538, + "theoretical_loss": 3.69122991612327, + "tokens_seen": 961871872 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035785356068204614, + "loss": 1.3419, + "theoretical_loss": 3.691205996536099, + "tokens_seen": 961937408 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003578435305917753, + "loss": 1.3358, + "theoretical_loss": 3.6911820790347525, + "tokens_seen": 962002944 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035783350050150455, + "loss": 1.3897, + "theoretical_loss": 3.6911581636189066, + "tokens_seen": 962068480 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003578234704112337, + "loss": 1.351, + "theoretical_loss": 3.691134250288238, + "tokens_seen": 962134016 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003578134403209629, + "loss": 1.2821, + "theoretical_loss": 3.6911103390424227, + "tokens_seen": 962199552 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003578034102306921, + "loss": 1.3671, + "theoretical_loss": 3.6910864298811363, + "tokens_seen": 962265088 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003577933801404213, + "loss": 1.4243, + "theoretical_loss": 3.6910625228040557, + "tokens_seen": 962330624 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035778335005015046, + "loss": 1.359, + "theoretical_loss": 3.6910386178108574, + "tokens_seen": 962396160 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035777331995987964, + "loss": 1.3161, + "theoretical_loss": 3.691014714901218, + "tokens_seen": 962461696 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003577632898696088, + "loss": 1.3115, + "theoretical_loss": 3.6909908140748136, + "tokens_seen": 962527232 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035775325977933806, + "loss": 1.4547, + "theoretical_loss": 3.6909669153313205, + "tokens_seen": 962592768 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003577432296890672, + "loss": 1.3927, + "theoretical_loss": 3.690943018670416, + "tokens_seen": 962658304 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003577331995987964, + "loss": 1.4377, + "theoretical_loss": 3.690919124091777, + "tokens_seen": 962723840 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035772316950852555, + "loss": 1.3279, + "theoretical_loss": 3.69089523159508, + "tokens_seen": 962789376 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003577131394182548, + "loss": 1.1957, + "theoretical_loss": 3.690871341180001, + "tokens_seen": 962854912 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035770310932798396, + "loss": 1.3112, + "theoretical_loss": 3.690847452846219, + "tokens_seen": 962920448 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035769307923771314, + "loss": 1.2915, + "theoretical_loss": 3.6908235665934095, + "tokens_seen": 962985984 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003576830491474423, + "loss": 1.3847, + "theoretical_loss": 3.6907996824212503, + "tokens_seen": 963051520 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035767301905717156, + "loss": 1.2866, + "theoretical_loss": 3.690775800329418, + "tokens_seen": 963117056 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003576629889669007, + "loss": 1.2312, + "theoretical_loss": 3.6907519203175907, + "tokens_seen": 963182592 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003576529588766299, + "loss": 1.3384, + "theoretical_loss": 3.6907280423854445, + "tokens_seen": 963248128 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035764292878635905, + "loss": 1.2736, + "theoretical_loss": 3.690704166532658, + "tokens_seen": 963313664 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 324566, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.360540747642517, + "objective/train/theoretical_loss": 3.690680292758908, + "objective/train/tokens_used": 963379200, + "theoretical_loss": 3.690680292758908, + "tokens_seen": 963379200 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003576328986960883, + "loss": 1.3531, + "theoretical_loss": 3.690680292758908, + "tokens_seen": 963379200 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035762286860581747, + "loss": 1.2845, + "theoretical_loss": 3.690656421063873, + "tokens_seen": 963444736 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035761283851554665, + "loss": 1.3745, + "theoretical_loss": 3.690632551447229, + "tokens_seen": 963510272 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035760280842527583, + "loss": 1.4197, + "theoretical_loss": 3.690608683908655, + "tokens_seen": 963575808 + }, + { + "epoch": 0.29, + "learning_rate": 0.000357592778335005, + "loss": 1.241, + "theoretical_loss": 3.6905848184478285, + "tokens_seen": 963641344 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003575827482447342, + "loss": 1.3704, + "theoretical_loss": 3.690560955064427, + "tokens_seen": 963706880 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003575727181544634, + "loss": 1.3786, + "theoretical_loss": 3.690537093758129, + "tokens_seen": 963772416 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035756268806419255, + "loss": 1.3867, + "theoretical_loss": 3.690513234528612, + "tokens_seen": 963837952 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003575526579739218, + "loss": 1.3353, + "theoretical_loss": 3.690489377375554, + "tokens_seen": 963903488 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003575426278836509, + "loss": 1.3238, + "theoretical_loss": 3.6904655222986333, + "tokens_seen": 963969024 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035753259779338015, + "loss": 1.3369, + "theoretical_loss": 3.690441669297528, + "tokens_seen": 964034560 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035752256770310933, + "loss": 1.3876, + "theoretical_loss": 3.6904178183719165, + "tokens_seen": 964100096 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003575125376128385, + "loss": 1.3958, + "theoretical_loss": 3.690393969521477, + "tokens_seen": 964165632 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003575025075225677, + "loss": 1.2975, + "theoretical_loss": 3.6903701227458883, + "tokens_seen": 964231168 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035749247743229693, + "loss": 1.3655, + "theoretical_loss": 3.6903462780448284, + "tokens_seen": 964296704 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035748244734202606, + "loss": 1.3038, + "theoretical_loss": 3.690322435417976, + "tokens_seen": 964362240 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003574724172517553, + "loss": 1.2967, + "theoretical_loss": 3.69029859486501, + "tokens_seen": 964427776 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003574623871614844, + "loss": 1.3484, + "theoretical_loss": 3.690274756385609, + "tokens_seen": 964493312 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035745235707121365, + "loss": 1.4255, + "theoretical_loss": 3.6902509199794507, + "tokens_seen": 964558848 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035744232698094283, + "loss": 1.4757, + "theoretical_loss": 3.6902270856462156, + "tokens_seen": 964624384 + }, + { + "epoch": 0.29, + "learning_rate": 0.000357432296890672, + "loss": 1.3828, + "theoretical_loss": 3.690203253385582, + "tokens_seen": 964689920 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003574222668004012, + "loss": 1.3376, + "theoretical_loss": 3.690179423197229, + "tokens_seen": 964755456 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003574122367101304, + "loss": 1.4223, + "theoretical_loss": 3.690155595080835, + "tokens_seen": 964820992 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035740220661985956, + "loss": 1.4041, + "theoretical_loss": 3.6901317690360793, + "tokens_seen": 964886528 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003573921765295888, + "loss": 1.3786, + "theoretical_loss": 3.6901079450626417, + "tokens_seen": 964952064 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003573821464393179, + "loss": 1.3835, + "theoretical_loss": 3.6900841231602017, + "tokens_seen": 965017600 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035737211634904716, + "loss": 1.3294, + "theoretical_loss": 3.690060303328438, + "tokens_seen": 965083136 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003573620862587763, + "loss": 1.315, + "theoretical_loss": 3.6900364855670293, + "tokens_seen": 965148672 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003573520561685055, + "loss": 1.3979, + "theoretical_loss": 3.6900126698756566, + "tokens_seen": 965214208 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035734202607823475, + "loss": 1.3884, + "theoretical_loss": 3.689988856253998, + "tokens_seen": 965279744 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003573319959879639, + "loss": 1.3842, + "theoretical_loss": 3.6899650447017347, + "tokens_seen": 965345280 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003573219658976931, + "loss": 1.3144, + "theoretical_loss": 3.6899412352185452, + "tokens_seen": 965410816 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003573119358074223, + "loss": 1.2911, + "theoretical_loss": 3.68991742780411, + "tokens_seen": 965476352 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003573019057171515, + "loss": 1.4303, + "theoretical_loss": 3.689893622458108, + "tokens_seen": 965541888 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035729187562688066, + "loss": 1.37, + "theoretical_loss": 3.6898698191802195, + "tokens_seen": 965607424 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035728184553660984, + "loss": 1.2941, + "theoretical_loss": 3.689846017970125, + "tokens_seen": 965672960 + }, + { + "epoch": 0.29, + "learning_rate": 0.000357271815446339, + "loss": 1.3505, + "theoretical_loss": 3.689822218827505, + "tokens_seen": 965738496 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035726178535606826, + "loss": 1.4475, + "theoretical_loss": 3.6897984217520383, + "tokens_seen": 965804032 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003572517552657974, + "loss": 1.3314, + "theoretical_loss": 3.6897746267434055, + "tokens_seen": 965869568 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003572417251755266, + "loss": 1.4037, + "theoretical_loss": 3.6897508338012877, + "tokens_seen": 965935104 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035723169508525575, + "loss": 1.3458, + "theoretical_loss": 3.689727042925364, + "tokens_seen": 966000640 + }, + { + "epoch": 0.29, + "learning_rate": 0.000357221664994985, + "loss": 1.3348, + "theoretical_loss": 3.689703254115315, + "tokens_seen": 966066176 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035721163490471416, + "loss": 1.2543, + "theoretical_loss": 3.689679467370823, + "tokens_seen": 966131712 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035720160481444334, + "loss": 1.3742, + "theoretical_loss": 3.6896556826915656, + "tokens_seen": 966197248 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003571915747241725, + "loss": 1.4376, + "theoretical_loss": 3.689631900077226, + "tokens_seen": 966262784 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035718154463390176, + "loss": 1.3686, + "theoretical_loss": 3.6896081195274837, + "tokens_seen": 966328320 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003571715145436309, + "loss": 1.4258, + "theoretical_loss": 3.6895843410420195, + "tokens_seen": 966393856 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003571614844533601, + "loss": 1.3701, + "theoretical_loss": 3.689560564620514, + "tokens_seen": 966459392 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035715145436308925, + "loss": 1.2658, + "theoretical_loss": 3.689536790262649, + "tokens_seen": 966524928 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003571414242728185, + "loss": 1.3413, + "theoretical_loss": 3.6895130179681046, + "tokens_seen": 966590464 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 325754, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3496108055114746, + "objective/train/theoretical_loss": 3.689489247736563, + "objective/train/tokens_used": 966656000, + "theoretical_loss": 3.689489247736563, + "tokens_seen": 966656000 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035713139418254767, + "loss": 1.3712, + "theoretical_loss": 3.689489247736563, + "tokens_seen": 966656000 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035712136409227685, + "loss": 1.3771, + "theoretical_loss": 3.689465479567703, + "tokens_seen": 966721536 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035711133400200603, + "loss": 1.4215, + "theoretical_loss": 3.6894417134612087, + "tokens_seen": 966787072 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003571013039117352, + "loss": 1.3953, + "theoretical_loss": 3.6894179494167596, + "tokens_seen": 966852608 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003570912738214644, + "loss": 1.2675, + "theoretical_loss": 3.6893941874340372, + "tokens_seen": 966918144 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003570812437311936, + "loss": 1.3592, + "theoretical_loss": 3.6893704275127233, + "tokens_seen": 966983680 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035707121364092275, + "loss": 1.3132, + "theoretical_loss": 3.6893466696524992, + "tokens_seen": 967049216 + }, + { + "epoch": 0.29, + "learning_rate": 0.000357061183550652, + "loss": 1.3394, + "theoretical_loss": 3.689322913853047, + "tokens_seen": 967114752 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003570511534603811, + "loss": 1.3504, + "theoretical_loss": 3.689299160114047, + "tokens_seen": 967180288 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035704112337011035, + "loss": 1.341, + "theoretical_loss": 3.689275408435182, + "tokens_seen": 967245824 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035703109327983953, + "loss": 1.3727, + "theoretical_loss": 3.6892516588161333, + "tokens_seen": 967311360 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003570210631895687, + "loss": 1.3568, + "theoretical_loss": 3.689227911256583, + "tokens_seen": 967376896 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003570110330992979, + "loss": 1.3253, + "theoretical_loss": 3.6892041657562125, + "tokens_seen": 967442432 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035700100300902713, + "loss": 1.275, + "theoretical_loss": 3.689180422314705, + "tokens_seen": 967507968 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035699097291875626, + "loss": 1.2802, + "theoretical_loss": 3.689156680931741, + "tokens_seen": 967573504 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003569809428284855, + "loss": 1.3868, + "theoretical_loss": 3.6891329416070033, + "tokens_seen": 967639040 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003569709127382146, + "loss": 1.3615, + "theoretical_loss": 3.6891092043401743, + "tokens_seen": 967704576 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035696088264794385, + "loss": 1.3666, + "theoretical_loss": 3.6890854691309354, + "tokens_seen": 967770112 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035695085255767303, + "loss": 1.3544, + "theoretical_loss": 3.6890617359789704, + "tokens_seen": 967835648 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003569408224674022, + "loss": 1.3461, + "theoretical_loss": 3.689038004883961, + "tokens_seen": 967901184 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003569307923771314, + "loss": 1.3547, + "theoretical_loss": 3.6890142758455884, + "tokens_seen": 967966720 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003569207622868606, + "loss": 1.3868, + "theoretical_loss": 3.6889905488635373, + "tokens_seen": 968032256 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035691073219658976, + "loss": 1.4309, + "theoretical_loss": 3.6889668239374886, + "tokens_seen": 968097792 + }, + { + "epoch": 0.29, + "learning_rate": 0.000356900702106319, + "loss": 1.3385, + "theoretical_loss": 3.6889431010671254, + "tokens_seen": 968163328 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003568906720160481, + "loss": 1.2476, + "theoretical_loss": 3.6889193802521314, + "tokens_seen": 968228864 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035688064192577736, + "loss": 1.3228, + "theoretical_loss": 3.6888956614921886, + "tokens_seen": 968294400 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003568706118355065, + "loss": 1.3643, + "theoretical_loss": 3.68887194478698, + "tokens_seen": 968359936 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003568605817452357, + "loss": 1.3341, + "theoretical_loss": 3.6888482301361885, + "tokens_seen": 968425472 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003568505516549649, + "loss": 1.4286, + "theoretical_loss": 3.688824517539497, + "tokens_seen": 968491008 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003568405215646941, + "loss": 1.378, + "theoretical_loss": 3.688800806996589, + "tokens_seen": 968556544 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035683049147442326, + "loss": 1.3872, + "theoretical_loss": 3.6887770985071473, + "tokens_seen": 968622080 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003568204613841525, + "loss": 1.4289, + "theoretical_loss": 3.688753392070855, + "tokens_seen": 968687616 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003568104312938816, + "loss": 1.3242, + "theoretical_loss": 3.6887296876873963, + "tokens_seen": 968753152 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035680040120361086, + "loss": 1.3568, + "theoretical_loss": 3.688705985356454, + "tokens_seen": 968818688 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035679037111334, + "loss": 1.5008, + "theoretical_loss": 3.688682285077711, + "tokens_seen": 968884224 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003567803410230692, + "loss": 1.3443, + "theoretical_loss": 3.6886585868508517, + "tokens_seen": 968949760 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003567703109327984, + "loss": 1.4476, + "theoretical_loss": 3.688634890675559, + "tokens_seen": 969015296 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003567602808425276, + "loss": 1.3065, + "theoretical_loss": 3.688611196551517, + "tokens_seen": 969080832 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035675025075225677, + "loss": 1.3559, + "theoretical_loss": 3.68858750447841, + "tokens_seen": 969146368 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035674022066198595, + "loss": 1.4075, + "theoretical_loss": 3.6885638144559207, + "tokens_seen": 969211904 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035673019057171513, + "loss": 1.3729, + "theoretical_loss": 3.6885401264837334, + "tokens_seen": 969277440 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035672016048144436, + "loss": 1.3341, + "theoretical_loss": 3.688516440561532, + "tokens_seen": 969342976 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003567101303911735, + "loss": 1.3509, + "theoretical_loss": 3.6884927566890005, + "tokens_seen": 969408512 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003567001003009027, + "loss": 1.3346, + "theoretical_loss": 3.6884690748658233, + "tokens_seen": 969474048 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035669007021063185, + "loss": 1.3655, + "theoretical_loss": 3.6884453950916845, + "tokens_seen": 969539584 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003566800401203611, + "loss": 1.2271, + "theoretical_loss": 3.6884217173662677, + "tokens_seen": 969605120 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035667001003009027, + "loss": 1.2601, + "theoretical_loss": 3.6883980416892577, + "tokens_seen": 969670656 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035665997993981945, + "loss": 1.3675, + "theoretical_loss": 3.688374368060339, + "tokens_seen": 969736192 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035664994984954863, + "loss": 1.392, + "theoretical_loss": 3.688350696479196, + "tokens_seen": 969801728 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035663991975927787, + "loss": 1.4175, + "theoretical_loss": 3.6883270269455126, + "tokens_seen": 969867264 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 326730, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1563050746917725, + "objective/train/theoretical_loss": 3.688303359458974, + "objective/train/tokens_used": 969932800, + "theoretical_loss": 3.688303359458974, + "tokens_seen": 969932800 + }, + { + "epoch": 0.29, + "learning_rate": 0.000356629889669007, + "loss": 1.242, + "theoretical_loss": 3.688303359458974, + "tokens_seen": 969932800 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035661985957873623, + "loss": 1.3929, + "theoretical_loss": 3.6882796940192644, + "tokens_seen": 969998336 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035660982948846536, + "loss": 1.2746, + "theoretical_loss": 3.6882560306260688, + "tokens_seen": 970063872 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003565997993981946, + "loss": 1.3542, + "theoretical_loss": 3.6882323692790724, + "tokens_seen": 970129408 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003565897693079238, + "loss": 1.3412, + "theoretical_loss": 3.688208709977959, + "tokens_seen": 970194944 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035657973921765295, + "loss": 1.3714, + "theoretical_loss": 3.6881850527224147, + "tokens_seen": 970260480 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003565697091273822, + "loss": 1.3226, + "theoretical_loss": 3.6881613975121232, + "tokens_seen": 970326016 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003565596790371113, + "loss": 1.273, + "theoretical_loss": 3.6881377443467707, + "tokens_seen": 970391552 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035654964894684055, + "loss": 1.3376, + "theoretical_loss": 3.6881140932260417, + "tokens_seen": 970457088 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035653961885656973, + "loss": 1.3576, + "theoretical_loss": 3.688090444149622, + "tokens_seen": 970522624 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003565295887662989, + "loss": 1.3814, + "theoretical_loss": 3.6880667971171963, + "tokens_seen": 970588160 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003565195586760281, + "loss": 1.3688, + "theoretical_loss": 3.68804315212845, + "tokens_seen": 970653696 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035650952858575733, + "loss": 1.3673, + "theoretical_loss": 3.6880195091830688, + "tokens_seen": 970719232 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035649949849548646, + "loss": 1.4188, + "theoretical_loss": 3.687995868280738, + "tokens_seen": 970784768 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003564894684052157, + "loss": 1.4073, + "theoretical_loss": 3.687972229421143, + "tokens_seen": 970850304 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003564794383149448, + "loss": 1.4073, + "theoretical_loss": 3.6879485926039695, + "tokens_seen": 970915840 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035646940822467405, + "loss": 1.275, + "theoretical_loss": 3.687924957828903, + "tokens_seen": 970981376 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035645937813440323, + "loss": 1.3262, + "theoretical_loss": 3.6879013250956305, + "tokens_seen": 971046912 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003564493480441324, + "loss": 1.3787, + "theoretical_loss": 3.687877694403836, + "tokens_seen": 971112448 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003564393179538616, + "loss": 1.3399, + "theoretical_loss": 3.687854065753206, + "tokens_seen": 971177984 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003564292878635908, + "loss": 1.2527, + "theoretical_loss": 3.6878304391434273, + "tokens_seen": 971243520 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035641925777331996, + "loss": 1.2114, + "theoretical_loss": 3.687806814574185, + "tokens_seen": 971309056 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003564092276830492, + "loss": 1.3516, + "theoretical_loss": 3.687783192045166, + "tokens_seen": 971374592 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003563991975927783, + "loss": 1.2677, + "theoretical_loss": 3.6877595715560556, + "tokens_seen": 971440128 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035638916750250756, + "loss": 1.4548, + "theoretical_loss": 3.6877359531065412, + "tokens_seen": 971505664 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003563791374122367, + "loss": 1.299, + "theoretical_loss": 3.6877123366963076, + "tokens_seen": 971571200 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003563691073219659, + "loss": 1.3183, + "theoretical_loss": 3.6876887223250425, + "tokens_seen": 971636736 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003563590772316951, + "loss": 1.3795, + "theoretical_loss": 3.687665109992431, + "tokens_seen": 971702272 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003563490471414243, + "loss": 1.362, + "theoretical_loss": 3.6876414996981612, + "tokens_seen": 971767808 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035633901705115346, + "loss": 1.2272, + "theoretical_loss": 3.687617891441919, + "tokens_seen": 971833344 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003563289869608827, + "loss": 1.4396, + "theoretical_loss": 3.6875942852233905, + "tokens_seen": 971898880 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003563189568706118, + "loss": 1.4213, + "theoretical_loss": 3.6875706810422635, + "tokens_seen": 971964416 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035630892678034106, + "loss": 1.3115, + "theoretical_loss": 3.687547078898224, + "tokens_seen": 972029952 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003562988966900702, + "loss": 1.3775, + "theoretical_loss": 3.6875234787909585, + "tokens_seen": 972095488 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003562888665997994, + "loss": 1.3028, + "theoretical_loss": 3.6874998807201553, + "tokens_seen": 972161024 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003562788365095286, + "loss": 1.3647, + "theoretical_loss": 3.6874762846855003, + "tokens_seen": 972226560 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003562688064192578, + "loss": 1.3757, + "theoretical_loss": 3.687452690686681, + "tokens_seen": 972292096 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035625877632898697, + "loss": 1.3953, + "theoretical_loss": 3.687429098723385, + "tokens_seen": 972357632 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035624874623871615, + "loss": 1.3755, + "theoretical_loss": 3.687405508795298, + "tokens_seen": 972423168 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035623871614844533, + "loss": 1.3243, + "theoretical_loss": 3.6873819209021086, + "tokens_seen": 972488704 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035622868605817456, + "loss": 1.4234, + "theoretical_loss": 3.687358335043504, + "tokens_seen": 972554240 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003562186559679037, + "loss": 1.4921, + "theoretical_loss": 3.687334751219171, + "tokens_seen": 972619776 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003562086258776329, + "loss": 1.3802, + "theoretical_loss": 3.6873111694287983, + "tokens_seen": 972685312 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035619859578736205, + "loss": 1.3111, + "theoretical_loss": 3.6872875896720725, + "tokens_seen": 972750848 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003561885656970913, + "loss": 1.2921, + "theoretical_loss": 3.687264011948681, + "tokens_seen": 972816384 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035617853560682047, + "loss": 1.2101, + "theoretical_loss": 3.687240436258312, + "tokens_seen": 972881920 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035616850551654965, + "loss": 1.3063, + "theoretical_loss": 3.6872168626006534, + "tokens_seen": 972947456 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035615847542627883, + "loss": 1.2209, + "theoretical_loss": 3.687193290975393, + "tokens_seen": 973012992 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035614844533600807, + "loss": 1.2932, + "theoretical_loss": 3.6871697213822188, + "tokens_seen": 973078528 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003561384152457372, + "loss": 1.3261, + "theoretical_loss": 3.6871461538208177, + "tokens_seen": 973144064 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 327980, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2756046056747437, + "objective/train/theoretical_loss": 3.6871225882908787, + "objective/train/tokens_used": 973209600, + "theoretical_loss": 3.6871225882908787, + "tokens_seen": 973209600 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035612838515546643, + "loss": 1.3557, + "theoretical_loss": 3.6871225882908787, + "tokens_seen": 973209600 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035611835506519556, + "loss": 1.3906, + "theoretical_loss": 3.68709902479209, + "tokens_seen": 973275136 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003561083249749248, + "loss": 1.3584, + "theoretical_loss": 3.687075463324139, + "tokens_seen": 973340672 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035609829488465397, + "loss": 1.3611, + "theoretical_loss": 3.6870519038867156, + "tokens_seen": 973406208 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035608826479438315, + "loss": 1.3683, + "theoretical_loss": 3.687028346479506, + "tokens_seen": 973471744 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035607823470411233, + "loss": 1.3649, + "theoretical_loss": 3.6870047911022, + "tokens_seen": 973537280 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003560682046138415, + "loss": 1.3641, + "theoretical_loss": 3.6869812377544857, + "tokens_seen": 973602816 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003560581745235707, + "loss": 1.2288, + "theoretical_loss": 3.6869576864360516, + "tokens_seen": 973668352 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035604814443329993, + "loss": 1.3866, + "theoretical_loss": 3.6869341371465865, + "tokens_seen": 973733888 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035603811434302906, + "loss": 1.4645, + "theoretical_loss": 3.686910589885778, + "tokens_seen": 973799424 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003560280842527583, + "loss": 1.3206, + "theoretical_loss": 3.686887044653316, + "tokens_seen": 973864960 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003560180541624874, + "loss": 1.2762, + "theoretical_loss": 3.6868635014488893, + "tokens_seen": 973930496 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035600802407221666, + "loss": 1.3485, + "theoretical_loss": 3.686839960272186, + "tokens_seen": 973996032 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035599799398194584, + "loss": 1.4124, + "theoretical_loss": 3.686816421122896, + "tokens_seen": 974061568 + }, + { + "epoch": 0.3, + "learning_rate": 0.000355987963891675, + "loss": 1.392, + "theoretical_loss": 3.686792884000708, + "tokens_seen": 974127104 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003559779338014042, + "loss": 1.3725, + "theoretical_loss": 3.68676934890531, + "tokens_seen": 974192640 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035596790371113344, + "loss": 1.3546, + "theoretical_loss": 3.6867458158363924, + "tokens_seen": 974258176 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035595787362086256, + "loss": 1.3567, + "theoretical_loss": 3.6867222847936443, + "tokens_seen": 974323712 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003559478435305918, + "loss": 1.3929, + "theoretical_loss": 3.686698755776754, + "tokens_seen": 974389248 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003559378134403209, + "loss": 1.3062, + "theoretical_loss": 3.686675228785412, + "tokens_seen": 974454784 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035592778335005016, + "loss": 1.2266, + "theoretical_loss": 3.686651703819307, + "tokens_seen": 974520320 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035591775325977934, + "loss": 1.3586, + "theoretical_loss": 3.686628180878129, + "tokens_seen": 974585856 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003559077231695085, + "loss": 1.4261, + "theoretical_loss": 3.686604659961567, + "tokens_seen": 974651392 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003558976930792377, + "loss": 1.3152, + "theoretical_loss": 3.6865811410693117, + "tokens_seen": 974716928 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003558876629889669, + "loss": 1.3754, + "theoretical_loss": 3.6865576242010514, + "tokens_seen": 974782464 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035587763289869607, + "loss": 1.3408, + "theoretical_loss": 3.6865341093564767, + "tokens_seen": 974848000 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003558676028084253, + "loss": 1.3482, + "theoretical_loss": 3.6865105965352765, + "tokens_seen": 974913536 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035585757271815443, + "loss": 1.344, + "theoretical_loss": 3.6864870857371423, + "tokens_seen": 974979072 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035584754262788366, + "loss": 1.2844, + "theoretical_loss": 3.6864635769617626, + "tokens_seen": 975044608 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003558375125376129, + "loss": 1.3426, + "theoretical_loss": 3.6864400702088282, + "tokens_seen": 975110144 + }, + { + "epoch": 0.3, + "learning_rate": 0.000355827482447342, + "loss": 1.3736, + "theoretical_loss": 3.686416565478029, + "tokens_seen": 975175680 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035581745235707126, + "loss": 1.4662, + "theoretical_loss": 3.6863930627690555, + "tokens_seen": 975241216 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003558074222668004, + "loss": 1.3397, + "theoretical_loss": 3.6863695620815973, + "tokens_seen": 975306752 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003557973921765296, + "loss": 1.4026, + "theoretical_loss": 3.686346063415345, + "tokens_seen": 975372288 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003557873620862588, + "loss": 1.3794, + "theoretical_loss": 3.686322566769989, + "tokens_seen": 975437824 + }, + { + "epoch": 0.3, + "learning_rate": 0.000355777331995988, + "loss": 1.3235, + "theoretical_loss": 3.6862990721452196, + "tokens_seen": 975503360 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035576730190571717, + "loss": 1.3541, + "theoretical_loss": 3.6862755795407276, + "tokens_seen": 975568896 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035575727181544635, + "loss": 1.3965, + "theoretical_loss": 3.686252088956203, + "tokens_seen": 975634432 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035574724172517553, + "loss": 1.3053, + "theoretical_loss": 3.6862286003913374, + "tokens_seen": 975699968 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035573721163490476, + "loss": 1.3479, + "theoretical_loss": 3.686205113845821, + "tokens_seen": 975765504 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003557271815446339, + "loss": 1.3597, + "theoretical_loss": 3.686181629319344, + "tokens_seen": 975831040 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003557171514543631, + "loss": 1.4053, + "theoretical_loss": 3.6861581468115983, + "tokens_seen": 975896576 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035570712136409225, + "loss": 1.4111, + "theoretical_loss": 3.6861346663222743, + "tokens_seen": 975962112 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003556970912738215, + "loss": 1.3989, + "theoretical_loss": 3.686111187851063, + "tokens_seen": 976027648 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035568706118355067, + "loss": 1.3999, + "theoretical_loss": 3.686087711397655, + "tokens_seen": 976093184 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035567703109327985, + "loss": 1.3007, + "theoretical_loss": 3.6860642369617427, + "tokens_seen": 976158720 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035566700100300903, + "loss": 1.4202, + "theoretical_loss": 3.686040764543016, + "tokens_seen": 976224256 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035565697091273827, + "loss": 1.3093, + "theoretical_loss": 3.686017294141167, + "tokens_seen": 976289792 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003556469408224674, + "loss": 1.4096, + "theoretical_loss": 3.6859938257558866, + "tokens_seen": 976355328 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035563691073219663, + "loss": 1.3741, + "theoretical_loss": 3.6859703593868662, + "tokens_seen": 976420864 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 329258, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2802512645721436, + "objective/train/theoretical_loss": 3.6859468950337977, + "objective/train/tokens_used": 976486400, + "theoretical_loss": 3.6859468950337977, + "tokens_seen": 976486400 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035562688064192576, + "loss": 1.3178, + "theoretical_loss": 3.6859468950337977, + "tokens_seen": 976486400 + }, + { + "epoch": 0.3, + "learning_rate": 0.000355616850551655, + "loss": 1.397, + "theoretical_loss": 3.685923432696372, + "tokens_seen": 976551936 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035560682046138417, + "loss": 1.379, + "theoretical_loss": 3.6858999723742807, + "tokens_seen": 976617472 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035559679037111335, + "loss": 1.3788, + "theoretical_loss": 3.685876514067216, + "tokens_seen": 976683008 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035558676028084253, + "loss": 1.3462, + "theoretical_loss": 3.685853057774869, + "tokens_seen": 976748544 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003555767301905717, + "loss": 1.3664, + "theoretical_loss": 3.6858296034969324, + "tokens_seen": 976814080 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003555667001003009, + "loss": 1.3435, + "theoretical_loss": 3.6858061512330975, + "tokens_seen": 976879616 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035555667001003013, + "loss": 1.4937, + "theoretical_loss": 3.685782700983056, + "tokens_seen": 976945152 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035554663991975926, + "loss": 1.3745, + "theoretical_loss": 3.6857592527465006, + "tokens_seen": 977010688 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003555366098294885, + "loss": 1.3276, + "theoretical_loss": 3.6857358065231227, + "tokens_seen": 977076224 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003555265797392176, + "loss": 1.3525, + "theoretical_loss": 3.685712362312615, + "tokens_seen": 977141760 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035551654964894686, + "loss": 1.3792, + "theoretical_loss": 3.6856889201146688, + "tokens_seen": 977207296 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035550651955867604, + "loss": 1.3596, + "theoretical_loss": 3.6856654799289776, + "tokens_seen": 977272832 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003554964894684052, + "loss": 1.3155, + "theoretical_loss": 3.6856420417552336, + "tokens_seen": 977338368 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003554864593781344, + "loss": 1.2972, + "theoretical_loss": 3.685618605593128, + "tokens_seen": 977403904 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035547642928786364, + "loss": 1.4116, + "theoretical_loss": 3.6855951714423543, + "tokens_seen": 977469440 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035546639919759276, + "loss": 1.3644, + "theoretical_loss": 3.6855717393026044, + "tokens_seen": 977534976 + }, + { + "epoch": 0.3, + "learning_rate": 0.000355456369107322, + "loss": 1.3872, + "theoretical_loss": 3.685548309173572, + "tokens_seen": 977600512 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003554463390170511, + "loss": 1.2396, + "theoretical_loss": 3.6855248810549486, + "tokens_seen": 977666048 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035543630892678036, + "loss": 1.4225, + "theoretical_loss": 3.6855014549464276, + "tokens_seen": 977731584 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035542627883650954, + "loss": 1.4108, + "theoretical_loss": 3.685478030847702, + "tokens_seen": 977797120 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003554162487462387, + "loss": 1.2984, + "theoretical_loss": 3.685454608758464, + "tokens_seen": 977862656 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003554062186559679, + "loss": 1.2765, + "theoretical_loss": 3.6854311886784066, + "tokens_seen": 977928192 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003553961885656971, + "loss": 1.3413, + "theoretical_loss": 3.685407770607224, + "tokens_seen": 977993728 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035538615847542627, + "loss": 1.3935, + "theoretical_loss": 3.685384354544608, + "tokens_seen": 978059264 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003553761283851555, + "loss": 1.4346, + "theoretical_loss": 3.685360940490252, + "tokens_seen": 978124800 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035536609829488463, + "loss": 1.3822, + "theoretical_loss": 3.685337528443849, + "tokens_seen": 978190336 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035535606820461386, + "loss": 1.3277, + "theoretical_loss": 3.6853141184050937, + "tokens_seen": 978255872 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035534603811434304, + "loss": 1.4049, + "theoretical_loss": 3.6852907103736774, + "tokens_seen": 978321408 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003553360080240722, + "loss": 1.3471, + "theoretical_loss": 3.6852673043492956, + "tokens_seen": 978386944 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003553259779338014, + "loss": 1.3279, + "theoretical_loss": 3.68524390033164, + "tokens_seen": 978452480 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003553159478435306, + "loss": 1.4722, + "theoretical_loss": 3.6852204983204055, + "tokens_seen": 978518016 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035530591775325977, + "loss": 1.2561, + "theoretical_loss": 3.6851970983152844, + "tokens_seen": 978583552 + }, + { + "epoch": 0.3, + "learning_rate": 0.000355295887662989, + "loss": 1.4164, + "theoretical_loss": 3.685173700315972, + "tokens_seen": 978649088 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035528585757271813, + "loss": 1.3818, + "theoretical_loss": 3.6851503043221605, + "tokens_seen": 978714624 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035527582748244737, + "loss": 1.3441, + "theoretical_loss": 3.685126910333545, + "tokens_seen": 978780160 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003552657973921765, + "loss": 1.3538, + "theoretical_loss": 3.6851035183498184, + "tokens_seen": 978845696 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035525576730190573, + "loss": 1.3543, + "theoretical_loss": 3.6850801283706756, + "tokens_seen": 978911232 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003552457372116349, + "loss": 1.3878, + "theoretical_loss": 3.68505674039581, + "tokens_seen": 978976768 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003552357071213641, + "loss": 1.4304, + "theoretical_loss": 3.6850333544249154, + "tokens_seen": 979042304 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035522567703109327, + "loss": 1.3581, + "theoretical_loss": 3.6850099704576866, + "tokens_seen": 979107840 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035521564694082245, + "loss": 1.3334, + "theoretical_loss": 3.684986588493818, + "tokens_seen": 979173376 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035520561685055163, + "loss": 1.2831, + "theoretical_loss": 3.684963208533003, + "tokens_seen": 979238912 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035519558676028087, + "loss": 1.3839, + "theoretical_loss": 3.684939830574937, + "tokens_seen": 979304448 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035518555667001, + "loss": 1.3302, + "theoretical_loss": 3.6849164546193136, + "tokens_seen": 979369984 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035517552657973923, + "loss": 1.448, + "theoretical_loss": 3.6848930806658275, + "tokens_seen": 979435520 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003551654964894684, + "loss": 1.4205, + "theoretical_loss": 3.684869708714174, + "tokens_seen": 979501056 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003551554663991976, + "loss": 1.4186, + "theoretical_loss": 3.684846338764047, + "tokens_seen": 979566592 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003551454363089268, + "loss": 1.3564, + "theoretical_loss": 3.6848229708151408, + "tokens_seen": 979632128 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035513540621865596, + "loss": 1.2856, + "theoretical_loss": 3.684799604867151, + "tokens_seen": 979697664 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 330180, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3741490840911865, + "objective/train/theoretical_loss": 3.6847762409197724, + "objective/train/tokens_used": 979763200, + "theoretical_loss": 3.6847762409197724, + "tokens_seen": 979763200 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035512537612838514, + "loss": 1.3321, + "theoretical_loss": 3.6847762409197724, + "tokens_seen": 979763200 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035511534603811437, + "loss": 1.3918, + "theoretical_loss": 3.6847528789726995, + "tokens_seen": 979828736 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003551053159478435, + "loss": 1.4096, + "theoretical_loss": 3.6847295190256273, + "tokens_seen": 979894272 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035509528585757273, + "loss": 1.3337, + "theoretical_loss": 3.684706161078251, + "tokens_seen": 979959808 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003550852557673019, + "loss": 1.32, + "theoretical_loss": 3.6846828051302656, + "tokens_seen": 980025344 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003550752256770311, + "loss": 1.285, + "theoretical_loss": 3.6846594511813664, + "tokens_seen": 980090880 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035506519558676033, + "loss": 1.3675, + "theoretical_loss": 3.6846360992312484, + "tokens_seen": 980156416 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035505516549648946, + "loss": 1.2833, + "theoretical_loss": 3.6846127492796077, + "tokens_seen": 980221952 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003550451354062187, + "loss": 1.3609, + "theoretical_loss": 3.6845894013261384, + "tokens_seen": 980287488 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003550351053159478, + "loss": 1.4251, + "theoretical_loss": 3.684566055370537, + "tokens_seen": 980353024 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035502507522567706, + "loss": 1.3768, + "theoretical_loss": 3.6845427114124982, + "tokens_seen": 980418560 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035501504513540624, + "loss": 1.4012, + "theoretical_loss": 3.684519369451718, + "tokens_seen": 980484096 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003550050150451354, + "loss": 1.3763, + "theoretical_loss": 3.6844960294878923, + "tokens_seen": 980549632 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003549949849548646, + "loss": 1.4921, + "theoretical_loss": 3.684472691520716, + "tokens_seen": 980615168 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035498495486459384, + "loss": 1.3002, + "theoretical_loss": 3.684449355549886, + "tokens_seen": 980680704 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035497492477432296, + "loss": 1.3402, + "theoretical_loss": 3.6844260215750975, + "tokens_seen": 980746240 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003549648946840522, + "loss": 1.2039, + "theoretical_loss": 3.684402689596046, + "tokens_seen": 980811776 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003549548645937813, + "loss": 1.3547, + "theoretical_loss": 3.6843793596124277, + "tokens_seen": 980877312 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035494483450351056, + "loss": 1.3609, + "theoretical_loss": 3.684356031623939, + "tokens_seen": 980942848 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035493480441323974, + "loss": 1.4408, + "theoretical_loss": 3.6843327056302764, + "tokens_seen": 981008384 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003549247743229689, + "loss": 1.3555, + "theoretical_loss": 3.6843093816311354, + "tokens_seen": 981073920 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003549147442326981, + "loss": 1.4446, + "theoretical_loss": 3.6842860596262117, + "tokens_seen": 981139456 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003549047141424273, + "loss": 1.1717, + "theoretical_loss": 3.6842627396152023, + "tokens_seen": 981204992 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035489468405215647, + "loss": 1.3068, + "theoretical_loss": 3.684239421597804, + "tokens_seen": 981270528 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003548846539618857, + "loss": 1.3731, + "theoretical_loss": 3.6842161055737126, + "tokens_seen": 981336064 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035487462387161483, + "loss": 1.2717, + "theoretical_loss": 3.6841927915426247, + "tokens_seen": 981401600 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035486459378134406, + "loss": 1.3577, + "theoretical_loss": 3.684169479504237, + "tokens_seen": 981467136 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035485456369107324, + "loss": 1.3212, + "theoretical_loss": 3.684146169458246, + "tokens_seen": 981532672 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003548445336008024, + "loss": 1.3681, + "theoretical_loss": 3.6841228614043486, + "tokens_seen": 981598208 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003548345035105316, + "loss": 1.4066, + "theoretical_loss": 3.6840995553422413, + "tokens_seen": 981663744 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003548244734202608, + "loss": 1.3691, + "theoretical_loss": 3.6840762512716214, + "tokens_seen": 981729280 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035481444332998997, + "loss": 1.4248, + "theoretical_loss": 3.6840529491921847, + "tokens_seen": 981794816 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003548044132397192, + "loss": 1.2355, + "theoretical_loss": 3.68402964910363, + "tokens_seen": 981860352 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035479438314944833, + "loss": 1.4035, + "theoretical_loss": 3.6840063510056527, + "tokens_seen": 981925888 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035478435305917757, + "loss": 1.3078, + "theoretical_loss": 3.6839830548979506, + "tokens_seen": 981991424 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003547743229689067, + "loss": 1.244, + "theoretical_loss": 3.683959760780221, + "tokens_seen": 982056960 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035476429287863593, + "loss": 1.3968, + "theoretical_loss": 3.68393646865216, + "tokens_seen": 982122496 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003547542627883651, + "loss": 1.4208, + "theoretical_loss": 3.6839131785134667, + "tokens_seen": 982188032 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003547442326980943, + "loss": 1.3929, + "theoretical_loss": 3.683889890363837, + "tokens_seen": 982253568 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035473420260782347, + "loss": 1.3481, + "theoretical_loss": 3.6838666042029695, + "tokens_seen": 982319104 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035472417251755265, + "loss": 1.2629, + "theoretical_loss": 3.6838433200305607, + "tokens_seen": 982384640 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035471414242728183, + "loss": 1.2547, + "theoretical_loss": 3.683820037846308, + "tokens_seen": 982450176 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035470411233701107, + "loss": 1.3722, + "theoretical_loss": 3.6837967576499104, + "tokens_seen": 982515712 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003546940822467402, + "loss": 1.3239, + "theoretical_loss": 3.6837734794410646, + "tokens_seen": 982581248 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035468405215646943, + "loss": 1.3353, + "theoretical_loss": 3.6837502032194687, + "tokens_seen": 982646784 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003546740220661986, + "loss": 1.427, + "theoretical_loss": 3.68372692898482, + "tokens_seen": 982712320 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003546639919759278, + "loss": 1.3203, + "theoretical_loss": 3.6837036567368173, + "tokens_seen": 982777856 + }, + { + "epoch": 0.3, + "learning_rate": 0.000354653961885657, + "loss": 1.3475, + "theoretical_loss": 3.6836803864751575, + "tokens_seen": 982843392 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035464393179538616, + "loss": 1.41, + "theoretical_loss": 3.6836571181995392, + "tokens_seen": 982908928 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035463390170511534, + "loss": 1.4298, + "theoretical_loss": 3.6836338519096605, + "tokens_seen": 982974464 + }, + { + "debugging/length": 618.875, + "debugging/num_segments": 8, + "debugging/score": 0.06526899563759822, + "debugging/score_std": 0.02239594677938035, + "epoch": 0.3, + "objective/train/docs_used": 331388, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4433459043502808, + "objective/train/theoretical_loss": 3.68361058760522, + "objective/train/tokens_used": 983040000, + "theoretical_loss": 3.68361058760522, + "tokens_seen": 983040000 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035462387161484457, + "loss": 1.4411, + "theoretical_loss": 3.68361058760522, + "tokens_seen": 983040000 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003546138415245737, + "loss": 1.4117, + "theoretical_loss": 3.683587325285915, + "tokens_seen": 983105536 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035460381143430293, + "loss": 1.214, + "theoretical_loss": 3.6835640649514443, + "tokens_seen": 983171072 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035459378134403206, + "loss": 1.2805, + "theoretical_loss": 3.6835408066015063, + "tokens_seen": 983236608 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003545837512537613, + "loss": 1.5248, + "theoretical_loss": 3.6835175502357993, + "tokens_seen": 983302144 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003545737211634905, + "loss": 1.3171, + "theoretical_loss": 3.6834942958540218, + "tokens_seen": 983367680 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035456369107321966, + "loss": 1.2872, + "theoretical_loss": 3.683471043455872, + "tokens_seen": 983433216 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035455366098294884, + "loss": 1.3474, + "theoretical_loss": 3.6834477930410494, + "tokens_seen": 983498752 + }, + { + "epoch": 0.3, + "learning_rate": 0.000354543630892678, + "loss": 1.327, + "theoretical_loss": 3.683424544609252, + "tokens_seen": 983564288 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003545336008024072, + "loss": 1.3205, + "theoretical_loss": 3.6834012981601787, + "tokens_seen": 983629824 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035452357071213644, + "loss": 1.3564, + "theoretical_loss": 3.683378053693528, + "tokens_seen": 983695360 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035451354062186556, + "loss": 1.3971, + "theoretical_loss": 3.683354811209, + "tokens_seen": 983760896 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003545035105315948, + "loss": 1.3379, + "theoretical_loss": 3.6833315707062924, + "tokens_seen": 983826432 + }, + { + "epoch": 0.3, + "learning_rate": 0.000354493480441324, + "loss": 1.3129, + "theoretical_loss": 3.6833083321851046, + "tokens_seen": 983891968 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035448345035105316, + "loss": 1.3457, + "theoretical_loss": 3.6832850956451355, + "tokens_seen": 983957504 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035447342026078234, + "loss": 1.4257, + "theoretical_loss": 3.6832618610860846, + "tokens_seen": 984023040 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003544633901705115, + "loss": 1.4722, + "theoretical_loss": 3.6832386285076506, + "tokens_seen": 984088576 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003544533600802407, + "loss": 1.4629, + "theoretical_loss": 3.6832153979095335, + "tokens_seen": 984154112 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035444332998996994, + "loss": 1.4057, + "theoretical_loss": 3.6831921692914324, + "tokens_seen": 984219648 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035443329989969907, + "loss": 1.3027, + "theoretical_loss": 3.6831689426530465, + "tokens_seen": 984285184 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003544232698094283, + "loss": 1.3639, + "theoretical_loss": 3.683145717994075, + "tokens_seen": 984350720 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035441323971915743, + "loss": 1.3463, + "theoretical_loss": 3.683122495314218, + "tokens_seen": 984416256 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035440320962888667, + "loss": 1.2965, + "theoretical_loss": 3.683099274613175, + "tokens_seen": 984481792 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035439317953861585, + "loss": 1.2676, + "theoretical_loss": 3.6830760558906457, + "tokens_seen": 984547328 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035438314944834503, + "loss": 1.3313, + "theoretical_loss": 3.6830528391463293, + "tokens_seen": 984612864 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003543731193580742, + "loss": 1.353, + "theoretical_loss": 3.6830296243799263, + "tokens_seen": 984678400 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035436308926780344, + "loss": 1.444, + "theoretical_loss": 3.6830064115911365, + "tokens_seen": 984743936 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003543530591775326, + "loss": 1.3121, + "theoretical_loss": 3.6829832007796597, + "tokens_seen": 984809472 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003543430290872618, + "loss": 1.338, + "theoretical_loss": 3.6829599919451956, + "tokens_seen": 984875008 + }, + { + "epoch": 0.3, + "learning_rate": 0.000354332998996991, + "loss": 1.2664, + "theoretical_loss": 3.682936785087444, + "tokens_seen": 984940544 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035432296890672017, + "loss": 1.3317, + "theoretical_loss": 3.6829135802061064, + "tokens_seen": 985006080 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003543129388164494, + "loss": 1.3318, + "theoretical_loss": 3.6828903773008816, + "tokens_seen": 985071616 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035430290872617853, + "loss": 1.2688, + "theoretical_loss": 3.6828671763714707, + "tokens_seen": 985137152 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035429287863590777, + "loss": 1.3893, + "theoretical_loss": 3.6828439774175736, + "tokens_seen": 985202688 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003542828485456369, + "loss": 1.3618, + "theoretical_loss": 3.682820780438891, + "tokens_seen": 985268224 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035427281845536613, + "loss": 1.376, + "theoretical_loss": 3.6827975854351225, + "tokens_seen": 985333760 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003542627883650953, + "loss": 1.2824, + "theoretical_loss": 3.68277439240597, + "tokens_seen": 985399296 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003542527582748245, + "loss": 1.3278, + "theoretical_loss": 3.682751201351133, + "tokens_seen": 985464832 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035424272818455367, + "loss": 1.3823, + "theoretical_loss": 3.6827280122703128, + "tokens_seen": 985530368 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035423269809428285, + "loss": 1.3312, + "theoretical_loss": 3.68270482516321, + "tokens_seen": 985595904 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035422266800401203, + "loss": 1.1891, + "theoretical_loss": 3.682681640029525, + "tokens_seen": 985661440 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035421263791374127, + "loss": 1.3131, + "theoretical_loss": 3.6826584568689587, + "tokens_seen": 985726976 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003542026078234704, + "loss": 1.3382, + "theoretical_loss": 3.6826352756812124, + "tokens_seen": 985792512 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035419257773319963, + "loss": 1.4248, + "theoretical_loss": 3.6826120964659874, + "tokens_seen": 985858048 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003541825476429288, + "loss": 1.378, + "theoretical_loss": 3.682588919222984, + "tokens_seen": 985923584 + }, + { + "epoch": 0.3, + "learning_rate": 0.000354172517552658, + "loss": 1.3391, + "theoretical_loss": 3.682565743951903, + "tokens_seen": 985989120 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003541624874623872, + "loss": 1.3399, + "theoretical_loss": 3.682542570652447, + "tokens_seen": 986054656 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035415245737211636, + "loss": 1.3963, + "theoretical_loss": 3.6825193993243155, + "tokens_seen": 986120192 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035414242728184554, + "loss": 1.4145, + "theoretical_loss": 3.682496229967211, + "tokens_seen": 986185728 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035413239719157477, + "loss": 1.3032, + "theoretical_loss": 3.6824730625808346, + "tokens_seen": 986251264 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 332386, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.371630311012268, + "objective/train/theoretical_loss": 3.6824498971648882, + "objective/train/tokens_used": 986316800, + "theoretical_loss": 3.6824498971648882, + "tokens_seen": 986316800 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003541223671013039, + "loss": 1.4124, + "theoretical_loss": 3.6824498971648882, + "tokens_seen": 986316800 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035411233701103314, + "loss": 1.2835, + "theoretical_loss": 3.682426733719072, + "tokens_seen": 986382336 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035410230692076226, + "loss": 1.4022, + "theoretical_loss": 3.6824035722430892, + "tokens_seen": 986447872 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003540922768304915, + "loss": 1.3404, + "theoretical_loss": 3.6823804127366397, + "tokens_seen": 986513408 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003540822467402207, + "loss": 1.4415, + "theoretical_loss": 3.6823572551994266, + "tokens_seen": 986578944 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035407221664994986, + "loss": 1.2672, + "theoretical_loss": 3.6823340996311513, + "tokens_seen": 986644480 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035406218655967904, + "loss": 1.3226, + "theoretical_loss": 3.682310946031515, + "tokens_seen": 986710016 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003540521564694082, + "loss": 1.2805, + "theoretical_loss": 3.6822877944002204, + "tokens_seen": 986775552 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003540421263791374, + "loss": 1.3005, + "theoretical_loss": 3.682264644736969, + "tokens_seen": 986841088 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035403209628886664, + "loss": 1.356, + "theoretical_loss": 3.6822414970414634, + "tokens_seen": 986906624 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035402206619859577, + "loss": 1.325, + "theoretical_loss": 3.682218351313405, + "tokens_seen": 986972160 + }, + { + "epoch": 0.3, + "learning_rate": 0.000354012036108325, + "loss": 1.2774, + "theoretical_loss": 3.6821952075524957, + "tokens_seen": 987037696 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003540020060180542, + "loss": 1.3324, + "theoretical_loss": 3.6821720657584387, + "tokens_seen": 987103232 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035399197592778336, + "loss": 1.4525, + "theoretical_loss": 3.6821489259309357, + "tokens_seen": 987168768 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035398194583751254, + "loss": 1.3394, + "theoretical_loss": 3.682125788069689, + "tokens_seen": 987234304 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003539719157472417, + "loss": 1.3206, + "theoretical_loss": 3.682102652174401, + "tokens_seen": 987299840 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003539618856569709, + "loss": 1.3218, + "theoretical_loss": 3.682079518244775, + "tokens_seen": 987365376 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035395185556670014, + "loss": 1.4383, + "theoretical_loss": 3.6820563862805127, + "tokens_seen": 987430912 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035394182547642927, + "loss": 1.4735, + "theoretical_loss": 3.6820332562813167, + "tokens_seen": 987496448 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003539317953861585, + "loss": 1.4052, + "theoretical_loss": 3.6820101282468896, + "tokens_seen": 987561984 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035392176529588763, + "loss": 1.2977, + "theoretical_loss": 3.6819870021769345, + "tokens_seen": 987627520 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035391173520561687, + "loss": 1.3777, + "theoretical_loss": 3.6819638780711546, + "tokens_seen": 987693056 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035390170511534605, + "loss": 1.368, + "theoretical_loss": 3.6819407559292516, + "tokens_seen": 987758592 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035389167502507523, + "loss": 1.3833, + "theoretical_loss": 3.6819176357509296, + "tokens_seen": 987824128 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003538816449348044, + "loss": 1.3358, + "theoretical_loss": 3.681894517535891, + "tokens_seen": 987889664 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035387161484453364, + "loss": 1.3212, + "theoretical_loss": 3.6818714012838383, + "tokens_seen": 987955200 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035386158475426277, + "loss": 1.2661, + "theoretical_loss": 3.6818482869944757, + "tokens_seen": 988020736 + }, + { + "epoch": 0.3, + "learning_rate": 0.000353851554663992, + "loss": 1.2813, + "theoretical_loss": 3.681825174667506, + "tokens_seen": 988086272 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035384152457372113, + "loss": 1.3038, + "theoretical_loss": 3.6818020643026323, + "tokens_seen": 988151808 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035383149448345037, + "loss": 1.4871, + "theoretical_loss": 3.681778955899558, + "tokens_seen": 988217344 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035382146439317955, + "loss": 1.4268, + "theoretical_loss": 3.6817558494579865, + "tokens_seen": 988282880 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035381143430290873, + "loss": 1.3919, + "theoretical_loss": 3.681732744977621, + "tokens_seen": 988348416 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003538014042126379, + "loss": 1.3471, + "theoretical_loss": 3.681709642458166, + "tokens_seen": 988413952 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003537913741223671, + "loss": 1.4082, + "theoretical_loss": 3.6816865418993237, + "tokens_seen": 988479488 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003537813440320963, + "loss": 1.3436, + "theoretical_loss": 3.681663443300798, + "tokens_seen": 988545024 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003537713139418255, + "loss": 1.46, + "theoretical_loss": 3.6816403466622942, + "tokens_seen": 988610560 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035376128385155464, + "loss": 1.3154, + "theoretical_loss": 3.681617251983514, + "tokens_seen": 988676096 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035375125376128387, + "loss": 1.2796, + "theoretical_loss": 3.681594159264162, + "tokens_seen": 988741632 + }, + { + "epoch": 0.3, + "learning_rate": 0.000353741223671013, + "loss": 1.3049, + "theoretical_loss": 3.681571068503942, + "tokens_seen": 988807168 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035373119358074223, + "loss": 1.3317, + "theoretical_loss": 3.6815479797025588, + "tokens_seen": 988872704 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003537211634904714, + "loss": 1.3882, + "theoretical_loss": 3.6815248928597155, + "tokens_seen": 988938240 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003537111334002006, + "loss": 1.3696, + "theoretical_loss": 3.681501807975116, + "tokens_seen": 989003776 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003537011033099298, + "loss": 1.3241, + "theoretical_loss": 3.681478725048465, + "tokens_seen": 989069312 + }, + { + "epoch": 0.3, + "learning_rate": 0.000353691073219659, + "loss": 1.3127, + "theoretical_loss": 3.6814556440794672, + "tokens_seen": 989134848 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035368104312938814, + "loss": 1.4832, + "theoretical_loss": 3.681432565067826, + "tokens_seen": 989200384 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003536710130391174, + "loss": 1.4048, + "theoretical_loss": 3.681409488013246, + "tokens_seen": 989265920 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003536609829488465, + "loss": 1.3559, + "theoretical_loss": 3.6813864129154314, + "tokens_seen": 989331456 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035365095285857574, + "loss": 1.3952, + "theoretical_loss": 3.6813633397740873, + "tokens_seen": 989396992 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003536409227683049, + "loss": 1.3456, + "theoretical_loss": 3.681340268588918, + "tokens_seen": 989462528 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003536308926780341, + "loss": 1.265, + "theoretical_loss": 3.6813171993596274, + "tokens_seen": 989528064 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 333591, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3872365951538086, + "objective/train/theoretical_loss": 3.681294132085921, + "objective/train/tokens_used": 989593600, + "theoretical_loss": 3.681294132085921, + "tokens_seen": 989593600 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003536208625877633, + "loss": 1.2808, + "theoretical_loss": 3.681294132085921, + "tokens_seen": 989593600 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035361083249749246, + "loss": 1.3313, + "theoretical_loss": 3.6812710667675033, + "tokens_seen": 989659136 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003536008024072217, + "loss": 1.3335, + "theoretical_loss": 3.6812480034040793, + "tokens_seen": 989724672 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003535907723169509, + "loss": 1.3764, + "theoretical_loss": 3.681224941995353, + "tokens_seen": 989790208 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035358074222668006, + "loss": 1.4805, + "theoretical_loss": 3.681201882541031, + "tokens_seen": 989855744 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035357071213640924, + "loss": 1.3784, + "theoretical_loss": 3.681178825040816, + "tokens_seen": 989921280 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003535606820461384, + "loss": 1.409, + "theoretical_loss": 3.6811557694944153, + "tokens_seen": 989986816 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003535506519558676, + "loss": 1.3206, + "theoretical_loss": 3.6811327159015326, + "tokens_seen": 990052352 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035354062186559684, + "loss": 1.2662, + "theoretical_loss": 3.6811096642618732, + "tokens_seen": 990117888 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035353059177532597, + "loss": 1.2774, + "theoretical_loss": 3.6810866145751433, + "tokens_seen": 990183424 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003535205616850552, + "loss": 1.2763, + "theoretical_loss": 3.6810635668410474, + "tokens_seen": 990248960 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003535105315947844, + "loss": 1.3098, + "theoretical_loss": 3.6810405210592907, + "tokens_seen": 990314496 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035350050150451356, + "loss": 1.3832, + "theoretical_loss": 3.681017477229579, + "tokens_seen": 990380032 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035349047141424274, + "loss": 1.2876, + "theoretical_loss": 3.680994435351618, + "tokens_seen": 990445568 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003534804413239719, + "loss": 1.3724, + "theoretical_loss": 3.6809713954251126, + "tokens_seen": 990511104 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003534704112337011, + "loss": 1.3373, + "theoretical_loss": 3.6809483574497692, + "tokens_seen": 990576640 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035346038114343034, + "loss": 1.3247, + "theoretical_loss": 3.6809253214252933, + "tokens_seen": 990642176 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035345035105315947, + "loss": 1.3356, + "theoretical_loss": 3.6809022873513904, + "tokens_seen": 990707712 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003534403209628887, + "loss": 1.2874, + "theoretical_loss": 3.680879255227766, + "tokens_seen": 990773248 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035343029087261783, + "loss": 1.2956, + "theoretical_loss": 3.680856225054127, + "tokens_seen": 990838784 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035342026078234707, + "loss": 1.3462, + "theoretical_loss": 3.680833196830178, + "tokens_seen": 990904320 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035341023069207625, + "loss": 1.2985, + "theoretical_loss": 3.6808101705556266, + "tokens_seen": 990969856 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035340020060180543, + "loss": 1.366, + "theoretical_loss": 3.6807871462301778, + "tokens_seen": 991035392 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003533901705115346, + "loss": 1.3248, + "theoretical_loss": 3.6807641238535376, + "tokens_seen": 991100928 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035338014042126384, + "loss": 1.2948, + "theoretical_loss": 3.6807411034254125, + "tokens_seen": 991166464 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035337011033099297, + "loss": 1.39, + "theoretical_loss": 3.6807180849455094, + "tokens_seen": 991232000 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003533600802407222, + "loss": 1.3069, + "theoretical_loss": 3.680695068413534, + "tokens_seen": 991297536 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035335005015045133, + "loss": 1.3765, + "theoretical_loss": 3.680672053829192, + "tokens_seen": 991363072 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035334002006018057, + "loss": 1.3121, + "theoretical_loss": 3.6806490411921917, + "tokens_seen": 991428608 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035332998996990975, + "loss": 1.4375, + "theoretical_loss": 3.6806260305022382, + "tokens_seen": 991494144 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035331995987963893, + "loss": 1.3884, + "theoretical_loss": 3.680603021759038, + "tokens_seen": 991559680 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003533099297893681, + "loss": 1.4081, + "theoretical_loss": 3.680580014962299, + "tokens_seen": 991625216 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003532998996990973, + "loss": 1.3246, + "theoretical_loss": 3.680557010111726, + "tokens_seen": 991690752 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003532898696088265, + "loss": 1.3177, + "theoretical_loss": 3.6805340072070276, + "tokens_seen": 991756288 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003532798395185557, + "loss": 1.3446, + "theoretical_loss": 3.68051100624791, + "tokens_seen": 991821824 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035326980942828484, + "loss": 1.2545, + "theoretical_loss": 3.6804880072340795, + "tokens_seen": 991887360 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035325977933801407, + "loss": 1.2413, + "theoretical_loss": 3.680465010165244, + "tokens_seen": 991952896 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003532497492477432, + "loss": 1.4276, + "theoretical_loss": 3.68044201504111, + "tokens_seen": 992018432 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035323971915747243, + "loss": 1.2928, + "theoretical_loss": 3.680419021861385, + "tokens_seen": 992083968 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003532296890672016, + "loss": 1.2669, + "theoretical_loss": 3.6803960306257752, + "tokens_seen": 992149504 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003532196589769308, + "loss": 1.3315, + "theoretical_loss": 3.680373041333989, + "tokens_seen": 992215040 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035320962888666, + "loss": 1.4012, + "theoretical_loss": 3.680350053985733, + "tokens_seen": 992280576 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003531995987963892, + "loss": 1.2832, + "theoretical_loss": 3.6803270685807146, + "tokens_seen": 992346112 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035318956870611834, + "loss": 1.2792, + "theoretical_loss": 3.680304085118641, + "tokens_seen": 992411648 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003531795386158476, + "loss": 1.2522, + "theoretical_loss": 3.6802811035992207, + "tokens_seen": 992477184 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003531695085255767, + "loss": 1.3179, + "theoretical_loss": 3.68025812402216, + "tokens_seen": 992542720 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035315947843530594, + "loss": 1.3281, + "theoretical_loss": 3.680235146387167, + "tokens_seen": 992608256 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003531494483450351, + "loss": 1.4062, + "theoretical_loss": 3.6802121706939497, + "tokens_seen": 992673792 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003531394182547643, + "loss": 1.3753, + "theoretical_loss": 3.680189196942216, + "tokens_seen": 992739328 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003531293881644935, + "loss": 1.2886, + "theoretical_loss": 3.6801662251316722, + "tokens_seen": 992804864 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 334407, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3840703964233398, + "objective/train/theoretical_loss": 3.6801432552620277, + "objective/train/tokens_used": 992870400, + "theoretical_loss": 3.6801432552620277, + "tokens_seen": 992870400 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035311935807422266, + "loss": 1.4384, + "theoretical_loss": 3.6801432552620277, + "tokens_seen": 992870400 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035310932798395184, + "loss": 1.3292, + "theoretical_loss": 3.68012028733299, + "tokens_seen": 992935936 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003530992978936811, + "loss": 1.368, + "theoretical_loss": 3.6800973213442667, + "tokens_seen": 993001472 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003530892678034102, + "loss": 1.4022, + "theoretical_loss": 3.680074357295566, + "tokens_seen": 993067008 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035307923771313944, + "loss": 1.2779, + "theoretical_loss": 3.680051395186596, + "tokens_seen": 993132544 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035306920762286857, + "loss": 1.3105, + "theoretical_loss": 3.680028435017065, + "tokens_seen": 993198080 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003530591775325978, + "loss": 1.3868, + "theoretical_loss": 3.6800054767866817, + "tokens_seen": 993263616 + }, + { + "epoch": 0.3, + "learning_rate": 0.000353049147442327, + "loss": 1.3158, + "theoretical_loss": 3.6799825204951535, + "tokens_seen": 993329152 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035303911735205617, + "loss": 1.2272, + "theoretical_loss": 3.6799595661421893, + "tokens_seen": 993394688 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035302908726178535, + "loss": 1.3068, + "theoretical_loss": 3.679936613727498, + "tokens_seen": 993460224 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003530190571715146, + "loss": 1.3904, + "theoretical_loss": 3.6799136632507867, + "tokens_seen": 993525760 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003530090270812437, + "loss": 1.4273, + "theoretical_loss": 3.6798907147117648, + "tokens_seen": 993591296 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035299899699097294, + "loss": 1.2958, + "theoretical_loss": 3.6798677681101406, + "tokens_seen": 993656832 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035298896690070207, + "loss": 1.2229, + "theoretical_loss": 3.6798448234456234, + "tokens_seen": 993722368 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003529789368104313, + "loss": 1.3533, + "theoretical_loss": 3.6798218807179213, + "tokens_seen": 993787904 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003529689067201605, + "loss": 1.2743, + "theoretical_loss": 3.6797989399267435, + "tokens_seen": 993853440 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035295887662988967, + "loss": 1.4044, + "theoretical_loss": 3.679776001071799, + "tokens_seen": 993918976 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035294884653961885, + "loss": 1.3654, + "theoretical_loss": 3.679753064152796, + "tokens_seen": 993984512 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035293881644934803, + "loss": 1.272, + "theoretical_loss": 3.679730129169444, + "tokens_seen": 994050048 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003529287863590772, + "loss": 1.3614, + "theoretical_loss": 3.679707196121452, + "tokens_seen": 994115584 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035291875626880645, + "loss": 1.2635, + "theoretical_loss": 3.6796842650085293, + "tokens_seen": 994181120 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003529087261785356, + "loss": 1.2829, + "theoretical_loss": 3.6796613358303842, + "tokens_seen": 994246656 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003528986960882648, + "loss": 1.4285, + "theoretical_loss": 3.6796384085867277, + "tokens_seen": 994312192 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035288866599799394, + "loss": 1.2203, + "theoretical_loss": 3.679615483277267, + "tokens_seen": 994377728 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035287863590772317, + "loss": 1.4365, + "theoretical_loss": 3.679592559901713, + "tokens_seen": 994443264 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035286860581745235, + "loss": 1.3687, + "theoretical_loss": 3.6795696384597747, + "tokens_seen": 994508800 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035285857572718153, + "loss": 1.2984, + "theoretical_loss": 3.679546718951161, + "tokens_seen": 994574336 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035284854563691077, + "loss": 1.3803, + "theoretical_loss": 3.6795238013755824, + "tokens_seen": 994639872 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035283851554663995, + "loss": 1.3211, + "theoretical_loss": 3.6795008857327476, + "tokens_seen": 994705408 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035282848545636913, + "loss": 1.2745, + "theoretical_loss": 3.6794779720223674, + "tokens_seen": 994770944 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003528184553660983, + "loss": 1.3206, + "theoretical_loss": 3.6794550602441505, + "tokens_seen": 994836480 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003528084252758275, + "loss": 1.3205, + "theoretical_loss": 3.6794321503978074, + "tokens_seen": 994902016 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003527983951855567, + "loss": 1.3063, + "theoretical_loss": 3.679409242483047, + "tokens_seen": 994967552 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003527883650952859, + "loss": 1.2917, + "theoretical_loss": 3.6793863364995802, + "tokens_seen": 995033088 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035277833500501504, + "loss": 1.3307, + "theoretical_loss": 3.6793634324471167, + "tokens_seen": 995098624 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035276830491474427, + "loss": 1.4016, + "theoretical_loss": 3.6793405303253675, + "tokens_seen": 995164160 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003527582748244734, + "loss": 1.2381, + "theoretical_loss": 3.6793176301340407, + "tokens_seen": 995229696 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035274824473420263, + "loss": 1.4249, + "theoretical_loss": 3.6792947318728473, + "tokens_seen": 995295232 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003527382146439318, + "loss": 1.2755, + "theoretical_loss": 3.679271835541499, + "tokens_seen": 995360768 + }, + { + "epoch": 0.3, + "learning_rate": 0.000352728184553661, + "loss": 1.2301, + "theoretical_loss": 3.6792489411397042, + "tokens_seen": 995426304 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003527181544633902, + "loss": 1.3111, + "theoretical_loss": 3.6792260486671737, + "tokens_seen": 995491840 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003527081243731194, + "loss": 1.2731, + "theoretical_loss": 3.6792031581236184, + "tokens_seen": 995557376 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035269809428284854, + "loss": 1.3545, + "theoretical_loss": 3.6791802695087483, + "tokens_seen": 995622912 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003526880641925778, + "loss": 1.2916, + "theoretical_loss": 3.6791573828222752, + "tokens_seen": 995688448 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003526780341023069, + "loss": 1.2891, + "theoretical_loss": 3.679134498063908, + "tokens_seen": 995753984 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035266800401203614, + "loss": 1.4129, + "theoretical_loss": 3.6791116152333583, + "tokens_seen": 995819520 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003526579739217653, + "loss": 1.3726, + "theoretical_loss": 3.6790887343303367, + "tokens_seen": 995885056 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003526479438314945, + "loss": 1.3446, + "theoretical_loss": 3.679065855354554, + "tokens_seen": 995950592 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003526379137412237, + "loss": 1.2845, + "theoretical_loss": 3.6790429783057212, + "tokens_seen": 996016128 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035262788365095286, + "loss": 1.1404, + "theoretical_loss": 3.679020103183549, + "tokens_seen": 996081664 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 335492, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.391661286354065, + "objective/train/theoretical_loss": 3.6789972299877487, + "objective/train/tokens_used": 996147200, + "theoretical_loss": 3.6789972299877487, + "tokens_seen": 996147200 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035261785356068204, + "loss": 1.4086, + "theoretical_loss": 3.6789972299877487, + "tokens_seen": 996147200 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003526078234704113, + "loss": 1.2514, + "theoretical_loss": 3.678974358718031, + "tokens_seen": 996212736 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003525977933801404, + "loss": 1.2884, + "theoretical_loss": 3.678951489374107, + "tokens_seen": 996278272 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035258776328986964, + "loss": 1.4484, + "theoretical_loss": 3.6789286219556887, + "tokens_seen": 996343808 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035257773319959877, + "loss": 1.2527, + "theoretical_loss": 3.678905756462486, + "tokens_seen": 996409344 + }, + { + "epoch": 0.3, + "learning_rate": 0.000352567703109328, + "loss": 1.2698, + "theoretical_loss": 3.6788828928942108, + "tokens_seen": 996474880 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003525576730190572, + "loss": 1.3032, + "theoretical_loss": 3.6788600312505753, + "tokens_seen": 996540416 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035254764292878637, + "loss": 1.3252, + "theoretical_loss": 3.6788371715312898, + "tokens_seen": 996605952 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035253761283851555, + "loss": 1.3467, + "theoretical_loss": 3.6788143137360665, + "tokens_seen": 996671488 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003525275827482448, + "loss": 1.2059, + "theoretical_loss": 3.678791457864616, + "tokens_seen": 996737024 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003525175526579739, + "loss": 1.2323, + "theoretical_loss": 3.678768603916651, + "tokens_seen": 996802560 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035250752256770314, + "loss": 1.4276, + "theoretical_loss": 3.678745751891883, + "tokens_seen": 996868096 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035249749247743227, + "loss": 1.343, + "theoretical_loss": 3.678722901790023, + "tokens_seen": 996933632 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003524874623871615, + "loss": 1.3482, + "theoretical_loss": 3.6787000536107834, + "tokens_seen": 996999168 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003524774322968907, + "loss": 1.3061, + "theoretical_loss": 3.6786772073538763, + "tokens_seen": 997064704 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035246740220661987, + "loss": 1.3669, + "theoretical_loss": 3.678654363019013, + "tokens_seen": 997130240 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035245737211634905, + "loss": 1.2571, + "theoretical_loss": 3.6786315206059057, + "tokens_seen": 997195776 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035244734202607823, + "loss": 1.2578, + "theoretical_loss": 3.678608680114267, + "tokens_seen": 997261312 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003524373119358074, + "loss": 1.337, + "theoretical_loss": 3.678585841543808, + "tokens_seen": 997326848 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035242728184553665, + "loss": 1.2575, + "theoretical_loss": 3.678563004894242, + "tokens_seen": 997392384 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003524172517552658, + "loss": 1.3532, + "theoretical_loss": 3.67854017016528, + "tokens_seen": 997457920 + }, + { + "epoch": 0.3, + "learning_rate": 0.000352407221664995, + "loss": 1.4212, + "theoretical_loss": 3.6785173373566353, + "tokens_seen": 997523456 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035239719157472414, + "loss": 1.3104, + "theoretical_loss": 3.6784945064680197, + "tokens_seen": 997588992 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035238716148445337, + "loss": 1.3098, + "theoretical_loss": 3.6784716774991457, + "tokens_seen": 997654528 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035237713139418255, + "loss": 1.365, + "theoretical_loss": 3.6784488504497266, + "tokens_seen": 997720064 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035236710130391173, + "loss": 1.3252, + "theoretical_loss": 3.6784260253194736, + "tokens_seen": 997785600 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003523570712136409, + "loss": 1.3017, + "theoretical_loss": 3.6784032021081003, + "tokens_seen": 997851136 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035234704112337015, + "loss": 1.3296, + "theoretical_loss": 3.6783803808153195, + "tokens_seen": 997916672 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003523370110330993, + "loss": 1.3197, + "theoretical_loss": 3.678357561440843, + "tokens_seen": 997982208 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003523269809428285, + "loss": 1.3783, + "theoretical_loss": 3.678334743984384, + "tokens_seen": 998047744 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035231695085255764, + "loss": 1.4, + "theoretical_loss": 3.6783119284456554, + "tokens_seen": 998113280 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003523069207622869, + "loss": 1.339, + "theoretical_loss": 3.6782891148243704, + "tokens_seen": 998178816 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035229689067201606, + "loss": 1.3153, + "theoretical_loss": 3.6782663031202416, + "tokens_seen": 998244352 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035228686058174524, + "loss": 1.4739, + "theoretical_loss": 3.6782434933329826, + "tokens_seen": 998309888 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003522768304914744, + "loss": 1.2967, + "theoretical_loss": 3.678220685462305, + "tokens_seen": 998375424 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003522668004012036, + "loss": 1.2267, + "theoretical_loss": 3.678197879507924, + "tokens_seen": 998440960 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003522567703109328, + "loss": 1.369, + "theoretical_loss": 3.678175075469552, + "tokens_seen": 998506496 + }, + { + "epoch": 0.3, + "learning_rate": 0.000352246740220662, + "loss": 1.3212, + "theoretical_loss": 3.678152273346902, + "tokens_seen": 998572032 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035223671013039114, + "loss": 1.4596, + "theoretical_loss": 3.6781294731396876, + "tokens_seen": 998637568 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003522266800401204, + "loss": 1.2842, + "theoretical_loss": 3.678106674847622, + "tokens_seen": 998703104 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035221664994984956, + "loss": 1.2797, + "theoretical_loss": 3.678083878470418, + "tokens_seen": 998768640 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035220661985957874, + "loss": 1.2955, + "theoretical_loss": 3.6780610840077914, + "tokens_seen": 998834176 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003521965897693079, + "loss": 1.287, + "theoretical_loss": 3.6780382914594534, + "tokens_seen": 998899712 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003521865596790371, + "loss": 1.3543, + "theoretical_loss": 3.678015500825119, + "tokens_seen": 998965248 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003521765295887663, + "loss": 1.2581, + "theoretical_loss": 3.6779927121045013, + "tokens_seen": 999030784 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003521664994984955, + "loss": 1.3383, + "theoretical_loss": 3.6779699252973144, + "tokens_seen": 999096320 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035215646940822465, + "loss": 1.2639, + "theoretical_loss": 3.6779471404032718, + "tokens_seen": 999161856 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003521464393179539, + "loss": 1.364, + "theoretical_loss": 3.6779243574220883, + "tokens_seen": 999227392 + }, + { + "epoch": 0.3, + "learning_rate": 0.000352136409227683, + "loss": 1.4121, + "theoretical_loss": 3.677901576353477, + "tokens_seen": 999292928 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035212637913741224, + "loss": 1.1978, + "theoretical_loss": 3.6778787971971516, + "tokens_seen": 999358464 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 336443, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.417507290840149, + "objective/train/theoretical_loss": 3.677856019952827, + "objective/train/tokens_used": 999424000, + "theoretical_loss": 3.677856019952827, + "tokens_seen": 999424000 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003521163490471414, + "loss": 1.4148, + "theoretical_loss": 3.677856019952827, + "tokens_seen": 999424000 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003521063189568706, + "loss": 1.378, + "theoretical_loss": 3.677833244620218, + "tokens_seen": 999489536 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035209628886659984, + "loss": 1.3606, + "theoretical_loss": 3.677810471199037, + "tokens_seen": 999555072 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035208625877632897, + "loss": 1.3846, + "theoretical_loss": 3.6777876996889995, + "tokens_seen": 999620608 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003520762286860582, + "loss": 1.4234, + "theoretical_loss": 3.677764930089819, + "tokens_seen": 999686144 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003520661985957874, + "loss": 1.269, + "theoretical_loss": 3.677742162401211, + "tokens_seen": 999751680 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035205616850551657, + "loss": 1.3265, + "theoretical_loss": 3.6777193966228894, + "tokens_seen": 999817216 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035204613841524575, + "loss": 1.257, + "theoretical_loss": 3.677696632754569, + "tokens_seen": 999882752 + }, + { + "epoch": 0.3, + "learning_rate": 0.000352036108324975, + "loss": 1.3612, + "theoretical_loss": 3.6776738707959633, + "tokens_seen": 999948288 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003520260782347041, + "loss": 1.3685, + "theoretical_loss": 3.677651110746788, + "tokens_seen": 1000013824 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035201604814443334, + "loss": 1.3499, + "theoretical_loss": 3.6776283526067575, + "tokens_seen": 1000079360 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035200601805416247, + "loss": 1.3457, + "theoretical_loss": 3.677605596375587, + "tokens_seen": 1000144896 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003519959879638917, + "loss": 1.3826, + "theoretical_loss": 3.6775828420529906, + "tokens_seen": 1000210432 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003519859578736209, + "loss": 1.352, + "theoretical_loss": 3.6775600896386837, + "tokens_seen": 1000275968 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035197592778335007, + "loss": 1.3673, + "theoretical_loss": 3.677537339132381, + "tokens_seen": 1000341504 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035196589769307925, + "loss": 1.1997, + "theoretical_loss": 3.6775145905337974, + "tokens_seen": 1000407040 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035195586760280843, + "loss": 1.3683, + "theoretical_loss": 3.6774918438426485, + "tokens_seen": 1000472576 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003519458375125376, + "loss": 1.3365, + "theoretical_loss": 3.6774690990586487, + "tokens_seen": 1000538112 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035193580742226685, + "loss": 1.3673, + "theoretical_loss": 3.677446356181514, + "tokens_seen": 1000603648 + }, + { + "epoch": 0.3, + "learning_rate": 0.000351925777331996, + "loss": 1.2621, + "theoretical_loss": 3.6774236152109587, + "tokens_seen": 1000669184 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003519157472417252, + "loss": 1.3288, + "theoretical_loss": 3.6774008761466987, + "tokens_seen": 1000734720 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035190571715145434, + "loss": 1.4147, + "theoretical_loss": 3.67737813898845, + "tokens_seen": 1000800256 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035189568706118357, + "loss": 1.3969, + "theoretical_loss": 3.677355403735927, + "tokens_seen": 1000865792 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035188565697091275, + "loss": 1.3479, + "theoretical_loss": 3.6773326703888456, + "tokens_seen": 1000931328 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035187562688064193, + "loss": 1.3826, + "theoretical_loss": 3.6773099389469213, + "tokens_seen": 1000996864 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003518655967903711, + "loss": 1.1899, + "theoretical_loss": 3.6772872094098696, + "tokens_seen": 1001062400 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035185556670010035, + "loss": 1.3742, + "theoretical_loss": 3.6772644817774065, + "tokens_seen": 1001127936 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003518455366098295, + "loss": 1.2797, + "theoretical_loss": 3.6772417560492476, + "tokens_seen": 1001193472 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003518355065195587, + "loss": 1.3149, + "theoretical_loss": 3.677219032225109, + "tokens_seen": 1001259008 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035182547642928784, + "loss": 1.2786, + "theoretical_loss": 3.677196310304706, + "tokens_seen": 1001324544 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003518154463390171, + "loss": 1.3259, + "theoretical_loss": 3.6771735902877545, + "tokens_seen": 1001390080 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035180541624874626, + "loss": 1.299, + "theoretical_loss": 3.6771508721739714, + "tokens_seen": 1001455616 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035179538615847544, + "loss": 1.3119, + "theoretical_loss": 3.6771281559630715, + "tokens_seen": 1001521152 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003517853560682046, + "loss": 1.3589, + "theoretical_loss": 3.677105441654772, + "tokens_seen": 1001586688 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003517753259779338, + "loss": 1.2637, + "theoretical_loss": 3.6770827292487884, + "tokens_seen": 1001652224 + }, + { + "epoch": 0.3, + "learning_rate": 0.000351765295887663, + "loss": 1.2973, + "theoretical_loss": 3.677060018744837, + "tokens_seen": 1001717760 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003517552657973922, + "loss": 1.4125, + "theoretical_loss": 3.6770373101426346, + "tokens_seen": 1001783296 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035174523570712134, + "loss": 1.2663, + "theoretical_loss": 3.677014603441897, + "tokens_seen": 1001848832 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003517352056168506, + "loss": 1.3813, + "theoretical_loss": 3.6769918986423407, + "tokens_seen": 1001914368 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035172517552657976, + "loss": 1.2826, + "theoretical_loss": 3.6769691957436823, + "tokens_seen": 1001979904 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035171514543630894, + "loss": 1.3049, + "theoretical_loss": 3.6769464947456383, + "tokens_seen": 1002045440 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003517051153460381, + "loss": 1.27, + "theoretical_loss": 3.6769237956479257, + "tokens_seen": 1002110976 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003516950852557673, + "loss": 1.3971, + "theoretical_loss": 3.6769010984502604, + "tokens_seen": 1002176512 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003516850551654965, + "loss": 1.3434, + "theoretical_loss": 3.67687840315236, + "tokens_seen": 1002242048 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003516750250752257, + "loss": 1.2199, + "theoretical_loss": 3.67685570975394, + "tokens_seen": 1002307584 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035166499498495485, + "loss": 1.3446, + "theoretical_loss": 3.676833018254719, + "tokens_seen": 1002373120 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003516549648946841, + "loss": 1.2233, + "theoretical_loss": 3.676810328654412, + "tokens_seen": 1002438656 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003516449348044132, + "loss": 1.3916, + "theoretical_loss": 3.676787640952737, + "tokens_seen": 1002504192 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035163490471414244, + "loss": 1.2764, + "theoretical_loss": 3.6767649551494115, + "tokens_seen": 1002569728 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003516248746238716, + "loss": 1.2803, + "theoretical_loss": 3.6767422712441515, + "tokens_seen": 1002635264 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 337618, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3122673034667969, + "objective/train/theoretical_loss": 3.676719589236675, + "objective/train/tokens_used": 1002700800, + "theoretical_loss": 3.676719589236675, + "tokens_seen": 1002700800 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003516148445336008, + "loss": 1.3413, + "theoretical_loss": 3.676719589236675, + "tokens_seen": 1002700800 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035160481444333, + "loss": 1.3236, + "theoretical_loss": 3.676696909126698, + "tokens_seen": 1002766336 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035159478435305917, + "loss": 1.3582, + "theoretical_loss": 3.6766742309139397, + "tokens_seen": 1002831872 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035158475426278835, + "loss": 1.1092, + "theoretical_loss": 3.6766515545981155, + "tokens_seen": 1002897408 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003515747241725176, + "loss": 1.349, + "theoretical_loss": 3.676628880178944, + "tokens_seen": 1002962944 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003515646940822467, + "loss": 1.3796, + "theoretical_loss": 3.676606207656142, + "tokens_seen": 1003028480 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035155466399197595, + "loss": 1.2401, + "theoretical_loss": 3.6765835370294275, + "tokens_seen": 1003094016 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035154463390170513, + "loss": 1.3638, + "theoretical_loss": 3.676560868298518, + "tokens_seen": 1003159552 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003515346038114343, + "loss": 1.3591, + "theoretical_loss": 3.6765382014631305, + "tokens_seen": 1003225088 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003515245737211635, + "loss": 1.3937, + "theoretical_loss": 3.6765155365229836, + "tokens_seen": 1003290624 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035151454363089267, + "loss": 1.2209, + "theoretical_loss": 3.6764928734777946, + "tokens_seen": 1003356160 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035150451354062185, + "loss": 1.3526, + "theoretical_loss": 3.676470212327281, + "tokens_seen": 1003421696 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003514944834503511, + "loss": 1.3536, + "theoretical_loss": 3.6764475530711613, + "tokens_seen": 1003487232 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003514844533600802, + "loss": 1.3191, + "theoretical_loss": 3.676424895709153, + "tokens_seen": 1003552768 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035147442326980945, + "loss": 1.3668, + "theoretical_loss": 3.676402240240974, + "tokens_seen": 1003618304 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003514643931795386, + "loss": 1.3279, + "theoretical_loss": 3.676379586666343, + "tokens_seen": 1003683840 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003514543630892678, + "loss": 1.3681, + "theoretical_loss": 3.6763569349849776, + "tokens_seen": 1003749376 + }, + { + "epoch": 0.3, + "learning_rate": 0.000351444332998997, + "loss": 1.3145, + "theoretical_loss": 3.676334285196596, + "tokens_seen": 1003814912 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003514343029087262, + "loss": 1.2396, + "theoretical_loss": 3.676311637300916, + "tokens_seen": 1003880448 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035142427281845536, + "loss": 1.3353, + "theoretical_loss": 3.676288991297657, + "tokens_seen": 1003945984 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035141424272818454, + "loss": 1.374, + "theoretical_loss": 3.6762663471865364, + "tokens_seen": 1004011520 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003514042126379137, + "loss": 1.2749, + "theoretical_loss": 3.676243704967273, + "tokens_seen": 1004077056 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035139418254764295, + "loss": 1.2657, + "theoretical_loss": 3.676221064639585, + "tokens_seen": 1004142592 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003513841524573721, + "loss": 1.3257, + "theoretical_loss": 3.6761984262031917, + "tokens_seen": 1004208128 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003513741223671013, + "loss": 1.3055, + "theoretical_loss": 3.676175789657811, + "tokens_seen": 1004273664 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003513640922768305, + "loss": 1.3637, + "theoretical_loss": 3.676153155003161, + "tokens_seen": 1004339200 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003513540621865597, + "loss": 1.3202, + "theoretical_loss": 3.676130522238962, + "tokens_seen": 1004404736 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003513440320962889, + "loss": 1.216, + "theoretical_loss": 3.6761078913649317, + "tokens_seen": 1004470272 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035133400200601804, + "loss": 1.4056, + "theoretical_loss": 3.6760852623807887, + "tokens_seen": 1004535808 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003513239719157473, + "loss": 1.266, + "theoretical_loss": 3.676062635286253, + "tokens_seen": 1004601344 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035131394182547646, + "loss": 1.4002, + "theoretical_loss": 3.676040010081042, + "tokens_seen": 1004666880 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035130391173520564, + "loss": 1.3793, + "theoretical_loss": 3.6760173867648764, + "tokens_seen": 1004732416 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003512938816449348, + "loss": 1.366, + "theoretical_loss": 3.6759947653374745, + "tokens_seen": 1004797952 + }, + { + "epoch": 0.3, + "learning_rate": 0.000351283851554664, + "loss": 1.3524, + "theoretical_loss": 3.6759721457985552, + "tokens_seen": 1004863488 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003512738214643932, + "loss": 1.3667, + "theoretical_loss": 3.6759495281478376, + "tokens_seen": 1004929024 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003512637913741224, + "loss": 1.2785, + "theoretical_loss": 3.6759269123850413, + "tokens_seen": 1004994560 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035125376128385154, + "loss": 1.2902, + "theoretical_loss": 3.6759042985098858, + "tokens_seen": 1005060096 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003512437311935808, + "loss": 1.2068, + "theoretical_loss": 3.6758816865220902, + "tokens_seen": 1005125632 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035123370110330996, + "loss": 1.3674, + "theoretical_loss": 3.675859076421374, + "tokens_seen": 1005191168 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035122367101303914, + "loss": 1.3884, + "theoretical_loss": 3.6758364682074567, + "tokens_seen": 1005256704 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003512136409227683, + "loss": 1.3828, + "theoretical_loss": 3.6758138618800578, + "tokens_seen": 1005322240 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003512036108324975, + "loss": 1.4102, + "theoretical_loss": 3.675791257438897, + "tokens_seen": 1005387776 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003511935807422267, + "loss": 1.415, + "theoretical_loss": 3.675768654883694, + "tokens_seen": 1005453312 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003511835506519559, + "loss": 1.418, + "theoretical_loss": 3.6757460542141684, + "tokens_seen": 1005518848 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035117352056168505, + "loss": 1.2591, + "theoretical_loss": 3.67572345543004, + "tokens_seen": 1005584384 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003511634904714143, + "loss": 1.419, + "theoretical_loss": 3.675700858531029, + "tokens_seen": 1005649920 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003511534603811434, + "loss": 1.3875, + "theoretical_loss": 3.675678263516855, + "tokens_seen": 1005715456 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035114343029087264, + "loss": 1.4026, + "theoretical_loss": 3.675655670387238, + "tokens_seen": 1005780992 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003511334002006018, + "loss": 1.509, + "theoretical_loss": 3.675633079141898, + "tokens_seen": 1005846528 + }, + { + "epoch": 0.3, + "learning_rate": 0.000351123370110331, + "loss": 1.2319, + "theoretical_loss": 3.6756104897805546, + "tokens_seen": 1005912064 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 338457, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3422868251800537, + "objective/train/theoretical_loss": 3.6755879023029294, + "objective/train/tokens_used": 1005977600, + "theoretical_loss": 3.6755879023029294, + "tokens_seen": 1005977600 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003511133400200602, + "loss": 1.3425, + "theoretical_loss": 3.6755879023029294, + "tokens_seen": 1005977600 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035110330992978937, + "loss": 1.3039, + "theoretical_loss": 3.675565316708741, + "tokens_seen": 1006043136 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035109327983951855, + "loss": 1.3385, + "theoretical_loss": 3.6755427329977106, + "tokens_seen": 1006108672 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003510832497492478, + "loss": 1.2948, + "theoretical_loss": 3.6755201511695588, + "tokens_seen": 1006174208 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003510732196589769, + "loss": 1.3504, + "theoretical_loss": 3.675497571224005, + "tokens_seen": 1006239744 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035106318956870615, + "loss": 1.2183, + "theoretical_loss": 3.675474993160771, + "tokens_seen": 1006305280 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035105315947843533, + "loss": 1.2744, + "theoretical_loss": 3.6754524169795753, + "tokens_seen": 1006370816 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003510431293881645, + "loss": 1.237, + "theoretical_loss": 3.67542984268014, + "tokens_seen": 1006436352 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003510330992978937, + "loss": 1.2999, + "theoretical_loss": 3.675407270262186, + "tokens_seen": 1006501888 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035102306920762287, + "loss": 1.269, + "theoretical_loss": 3.675384699725433, + "tokens_seen": 1006567424 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035101303911735205, + "loss": 1.2489, + "theoretical_loss": 3.675362131069602, + "tokens_seen": 1006632960 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003510030090270813, + "loss": 1.4236, + "theoretical_loss": 3.675339564294414, + "tokens_seen": 1006698496 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003509929789368104, + "loss": 1.3158, + "theoretical_loss": 3.6753169993995902, + "tokens_seen": 1006764032 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035098294884653965, + "loss": 1.2797, + "theoretical_loss": 3.675294436384851, + "tokens_seen": 1006829568 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003509729187562688, + "loss": 1.3249, + "theoretical_loss": 3.6752718752499174, + "tokens_seen": 1006895104 + }, + { + "epoch": 0.31, + "learning_rate": 0.000350962888665998, + "loss": 1.3731, + "theoretical_loss": 3.6752493159945105, + "tokens_seen": 1006960640 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003509528585757272, + "loss": 1.2905, + "theoretical_loss": 3.675226758618352, + "tokens_seen": 1007026176 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003509428284854564, + "loss": 1.295, + "theoretical_loss": 3.6752042031211625, + "tokens_seen": 1007091712 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035093279839518556, + "loss": 1.2806, + "theoretical_loss": 3.6751816495026626, + "tokens_seen": 1007157248 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035092276830491474, + "loss": 1.371, + "theoretical_loss": 3.6751590977625748, + "tokens_seen": 1007222784 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003509127382146439, + "loss": 1.3069, + "theoretical_loss": 3.67513654790062, + "tokens_seen": 1007288320 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035090270812437315, + "loss": 1.3406, + "theoretical_loss": 3.67511399991652, + "tokens_seen": 1007353856 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003508926780341023, + "loss": 1.3321, + "theoretical_loss": 3.675091453809995, + "tokens_seen": 1007419392 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003508826479438315, + "loss": 1.322, + "theoretical_loss": 3.6750689095807676, + "tokens_seen": 1007484928 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003508726178535607, + "loss": 1.3235, + "theoretical_loss": 3.6750463672285596, + "tokens_seen": 1007550464 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003508625877632899, + "loss": 1.3288, + "theoretical_loss": 3.6750238267530917, + "tokens_seen": 1007616000 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035085255767301906, + "loss": 1.3269, + "theoretical_loss": 3.6750012881540863, + "tokens_seen": 1007681536 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035084252758274824, + "loss": 1.3416, + "theoretical_loss": 3.674978751431265, + "tokens_seen": 1007747072 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003508324974924774, + "loss": 1.3928, + "theoretical_loss": 3.674956216584349, + "tokens_seen": 1007812608 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035082246740220666, + "loss": 1.3546, + "theoretical_loss": 3.6749336836130615, + "tokens_seen": 1007878144 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003508124373119358, + "loss": 1.2478, + "theoretical_loss": 3.674911152517123, + "tokens_seen": 1007943680 + }, + { + "epoch": 0.31, + "learning_rate": 0.000350802407221665, + "loss": 1.3337, + "theoretical_loss": 3.674888623296257, + "tokens_seen": 1008009216 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035079237713139415, + "loss": 1.3466, + "theoretical_loss": 3.6748660959501844, + "tokens_seen": 1008074752 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003507823470411234, + "loss": 1.2724, + "theoretical_loss": 3.674843570478627, + "tokens_seen": 1008140288 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035077231695085256, + "loss": 1.2757, + "theoretical_loss": 3.6748210468813087, + "tokens_seen": 1008205824 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035076228686058174, + "loss": 1.3134, + "theoretical_loss": 3.6747985251579505, + "tokens_seen": 1008271360 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003507522567703109, + "loss": 1.2179, + "theoretical_loss": 3.6747760053082743, + "tokens_seen": 1008336896 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035074222668004016, + "loss": 1.2628, + "theoretical_loss": 3.674753487332004, + "tokens_seen": 1008402432 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003507321965897693, + "loss": 1.268, + "theoretical_loss": 3.6747309712288603, + "tokens_seen": 1008467968 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003507221664994985, + "loss": 1.3957, + "theoretical_loss": 3.674708456998567, + "tokens_seen": 1008533504 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035071213640922765, + "loss": 1.2994, + "theoretical_loss": 3.6746859446408457, + "tokens_seen": 1008599040 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003507021063189569, + "loss": 1.4004, + "theoretical_loss": 3.674663434155419, + "tokens_seen": 1008664576 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035069207622868607, + "loss": 1.2359, + "theoretical_loss": 3.6746409255420103, + "tokens_seen": 1008730112 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035068204613841525, + "loss": 1.3198, + "theoretical_loss": 3.674618418800342, + "tokens_seen": 1008795648 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035067201604814443, + "loss": 1.3955, + "theoretical_loss": 3.674595913930137, + "tokens_seen": 1008861184 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003506619859578736, + "loss": 1.2829, + "theoretical_loss": 3.6745734109311177, + "tokens_seen": 1008926720 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003506519558676028, + "loss": 1.2744, + "theoretical_loss": 3.674550909803007, + "tokens_seen": 1008992256 + }, + { + "epoch": 0.31, + "learning_rate": 0.000350641925777332, + "loss": 1.3857, + "theoretical_loss": 3.6745284105455287, + "tokens_seen": 1009057792 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035063189568706115, + "loss": 1.42, + "theoretical_loss": 3.674505913158405, + "tokens_seen": 1009123328 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003506218655967904, + "loss": 1.3192, + "theoretical_loss": 3.6744834176413588, + "tokens_seen": 1009188864 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 339647, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.181186318397522, + "objective/train/theoretical_loss": 3.6744609239941135, + "objective/train/tokens_used": 1009254400, + "theoretical_loss": 3.6744609239941135, + "tokens_seen": 1009254400 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003506118355065195, + "loss": 1.2406, + "theoretical_loss": 3.6744609239941135, + "tokens_seen": 1009254400 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035060180541624875, + "loss": 1.2614, + "theoretical_loss": 3.6744384322163928, + "tokens_seen": 1009319936 + }, + { + "epoch": 0.31, + "learning_rate": 0.000350591775325978, + "loss": 1.3004, + "theoretical_loss": 3.674415942307919, + "tokens_seen": 1009385472 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003505817452357071, + "loss": 1.4059, + "theoretical_loss": 3.674393454268416, + "tokens_seen": 1009451008 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035057171514543635, + "loss": 1.1765, + "theoretical_loss": 3.674370968097607, + "tokens_seen": 1009516544 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035056168505516553, + "loss": 1.3341, + "theoretical_loss": 3.6743484837952165, + "tokens_seen": 1009582080 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003505516549648947, + "loss": 1.3884, + "theoretical_loss": 3.674326001360966, + "tokens_seen": 1009647616 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003505416248746239, + "loss": 1.4026, + "theoretical_loss": 3.67430352079458, + "tokens_seen": 1009713152 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035053159478435307, + "loss": 1.3778, + "theoretical_loss": 3.6742810420957825, + "tokens_seen": 1009778688 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035052156469408225, + "loss": 1.2958, + "theoretical_loss": 3.6742585652642967, + "tokens_seen": 1009844224 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003505115346038115, + "loss": 1.4768, + "theoretical_loss": 3.6742360902998454, + "tokens_seen": 1009909760 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003505015045135406, + "loss": 1.4341, + "theoretical_loss": 3.6742136172021542, + "tokens_seen": 1009975296 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035049147442326985, + "loss": 1.308, + "theoretical_loss": 3.6741911459709464, + "tokens_seen": 1010040832 + }, + { + "epoch": 0.31, + "learning_rate": 0.000350481444332999, + "loss": 1.3596, + "theoretical_loss": 3.674168676605945, + "tokens_seen": 1010106368 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003504714142427282, + "loss": 1.2648, + "theoretical_loss": 3.674146209106875, + "tokens_seen": 1010171904 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003504613841524574, + "loss": 1.325, + "theoretical_loss": 3.6741237434734595, + "tokens_seen": 1010237440 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003504513540621866, + "loss": 1.3351, + "theoretical_loss": 3.6741012797054227, + "tokens_seen": 1010302976 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035044132397191576, + "loss": 1.3319, + "theoretical_loss": 3.6740788178024895, + "tokens_seen": 1010368512 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035043129388164494, + "loss": 1.3066, + "theoretical_loss": 3.6740563577643837, + "tokens_seen": 1010434048 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003504212637913741, + "loss": 1.2789, + "theoretical_loss": 3.674033899590829, + "tokens_seen": 1010499584 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035041123370110335, + "loss": 1.3946, + "theoretical_loss": 3.6740114432815503, + "tokens_seen": 1010565120 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003504012036108325, + "loss": 1.3163, + "theoretical_loss": 3.6739889888362716, + "tokens_seen": 1010630656 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003503911735205617, + "loss": 1.3465, + "theoretical_loss": 3.6739665362547176, + "tokens_seen": 1010696192 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003503811434302909, + "loss": 1.2647, + "theoretical_loss": 3.673944085536613, + "tokens_seen": 1010761728 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003503711133400201, + "loss": 1.3335, + "theoretical_loss": 3.6739216366816816, + "tokens_seen": 1010827264 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035036108324974926, + "loss": 1.3319, + "theoretical_loss": 3.6738991896896485, + "tokens_seen": 1010892800 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035035105315947844, + "loss": 1.2836, + "theoretical_loss": 3.6738767445602383, + "tokens_seen": 1010958336 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003503410230692076, + "loss": 1.3558, + "theoretical_loss": 3.6738543012931757, + "tokens_seen": 1011023872 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035033099297893686, + "loss": 1.3467, + "theoretical_loss": 3.6738318598881854, + "tokens_seen": 1011089408 + }, + { + "epoch": 0.31, + "learning_rate": 0.000350320962888666, + "loss": 1.315, + "theoretical_loss": 3.673809420344993, + "tokens_seen": 1011154944 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003503109327983952, + "loss": 1.171, + "theoretical_loss": 3.6737869826633216, + "tokens_seen": 1011220480 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035030090270812435, + "loss": 1.3631, + "theoretical_loss": 3.6737645468428974, + "tokens_seen": 1011286016 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003502908726178536, + "loss": 1.444, + "theoretical_loss": 3.6737421128834455, + "tokens_seen": 1011351552 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035028084252758276, + "loss": 1.3443, + "theoretical_loss": 3.673719680784691, + "tokens_seen": 1011417088 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035027081243731194, + "loss": 1.3953, + "theoretical_loss": 3.673697250546358, + "tokens_seen": 1011482624 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003502607823470411, + "loss": 1.3136, + "theoretical_loss": 3.6736748221681728, + "tokens_seen": 1011548160 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035025075225677036, + "loss": 1.3935, + "theoretical_loss": 3.67365239564986, + "tokens_seen": 1011613696 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003502407221664995, + "loss": 1.2749, + "theoretical_loss": 3.6736299709911453, + "tokens_seen": 1011679232 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003502306920762287, + "loss": 1.263, + "theoretical_loss": 3.673607548191754, + "tokens_seen": 1011744768 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035022066198595785, + "loss": 1.2802, + "theoretical_loss": 3.6735851272514113, + "tokens_seen": 1011810304 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003502106318956871, + "loss": 1.3917, + "theoretical_loss": 3.6735627081698423, + "tokens_seen": 1011875840 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035020060180541627, + "loss": 1.2203, + "theoretical_loss": 3.6735402909467734, + "tokens_seen": 1011941376 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035019057171514545, + "loss": 1.4, + "theoretical_loss": 3.6735178755819295, + "tokens_seen": 1012006912 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035018054162487463, + "loss": 1.2622, + "theoretical_loss": 3.673495462075037, + "tokens_seen": 1012072448 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003501705115346038, + "loss": 1.4464, + "theoretical_loss": 3.6734730504258204, + "tokens_seen": 1012137984 + }, + { + "epoch": 0.31, + "learning_rate": 0.000350160481444333, + "loss": 1.2787, + "theoretical_loss": 3.6734506406340066, + "tokens_seen": 1012203520 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003501504513540622, + "loss": 1.3452, + "theoretical_loss": 3.673428232699321, + "tokens_seen": 1012269056 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035014042126379135, + "loss": 1.3615, + "theoretical_loss": 3.6734058266214893, + "tokens_seen": 1012334592 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003501303911735206, + "loss": 1.312, + "theoretical_loss": 3.6733834224002377, + "tokens_seen": 1012400128 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003501203610832497, + "loss": 1.3216, + "theoretical_loss": 3.6733610200352924, + "tokens_seen": 1012465664 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 340830, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3100988864898682, + "objective/train/theoretical_loss": 3.673338619526379, + "objective/train/tokens_used": 1012531200, + "theoretical_loss": 3.673338619526379, + "tokens_seen": 1012531200 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035011033099297895, + "loss": 1.3217, + "theoretical_loss": 3.673338619526379, + "tokens_seen": 1012531200 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035010030090270813, + "loss": 1.3284, + "theoretical_loss": 3.6733162208732235, + "tokens_seen": 1012596736 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003500902708124373, + "loss": 1.3975, + "theoretical_loss": 3.6732938240755524, + "tokens_seen": 1012662272 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003500802407221665, + "loss": 1.3154, + "theoretical_loss": 3.673271429133092, + "tokens_seen": 1012727808 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035007021063189573, + "loss": 1.408, + "theoretical_loss": 3.6732490360455685, + "tokens_seen": 1012793344 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035006018054162486, + "loss": 1.2486, + "theoretical_loss": 3.6732266448127078, + "tokens_seen": 1012858880 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003500501504513541, + "loss": 1.2436, + "theoretical_loss": 3.6732042554342375, + "tokens_seen": 1012924416 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003500401203610832, + "loss": 1.2657, + "theoretical_loss": 3.673181867909882, + "tokens_seen": 1012989952 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035003009027081245, + "loss": 1.3585, + "theoretical_loss": 3.6731594822393703, + "tokens_seen": 1013055488 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035002006018054163, + "loss": 1.2986, + "theoretical_loss": 3.673137098422427, + "tokens_seen": 1013121024 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003500100300902708, + "loss": 1.2751, + "theoretical_loss": 3.6731147164587803, + "tokens_seen": 1013186560 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035, + "loss": 1.4088, + "theoretical_loss": 3.6730923363481556, + "tokens_seen": 1013252096 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003499899699097292, + "loss": 1.3415, + "theoretical_loss": 3.67306995809028, + "tokens_seen": 1013317632 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034997993981945836, + "loss": 1.3286, + "theoretical_loss": 3.6730475816848807, + "tokens_seen": 1013383168 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003499699097291876, + "loss": 1.3921, + "theoretical_loss": 3.6730252071316842, + "tokens_seen": 1013448704 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003499598796389167, + "loss": 1.3771, + "theoretical_loss": 3.6730028344304175, + "tokens_seen": 1013514240 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034994984954864596, + "loss": 1.3151, + "theoretical_loss": 3.6729804635808074, + "tokens_seen": 1013579776 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003499398194583751, + "loss": 1.2468, + "theoretical_loss": 3.6729580945825813, + "tokens_seen": 1013645312 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003499297893681043, + "loss": 1.2539, + "theoretical_loss": 3.6729357274354664, + "tokens_seen": 1013710848 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003499197592778335, + "loss": 1.351, + "theoretical_loss": 3.672913362139189, + "tokens_seen": 1013776384 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003499097291875627, + "loss": 1.2654, + "theoretical_loss": 3.6728909986934775, + "tokens_seen": 1013841920 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034989969909729186, + "loss": 1.2397, + "theoretical_loss": 3.6728686370980586, + "tokens_seen": 1013907456 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003498896690070211, + "loss": 1.3063, + "theoretical_loss": 3.6728462773526593, + "tokens_seen": 1013972992 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003498796389167502, + "loss": 1.2903, + "theoretical_loss": 3.6728239194570076, + "tokens_seen": 1014038528 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034986960882647946, + "loss": 1.2889, + "theoretical_loss": 3.67280156341083, + "tokens_seen": 1014104064 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003498595787362086, + "loss": 1.2878, + "theoretical_loss": 3.6727792092138554, + "tokens_seen": 1014169600 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003498495486459378, + "loss": 1.3798, + "theoretical_loss": 3.6727568568658104, + "tokens_seen": 1014235136 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034983951855566706, + "loss": 1.3598, + "theoretical_loss": 3.6727345063664227, + "tokens_seen": 1014300672 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003498294884653962, + "loss": 1.4141, + "theoretical_loss": 3.67271215771542, + "tokens_seen": 1014366208 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003498194583751254, + "loss": 1.3247, + "theoretical_loss": 3.67268981091253, + "tokens_seen": 1014431744 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034980942828485455, + "loss": 1.3916, + "theoretical_loss": 3.672667465957481, + "tokens_seen": 1014497280 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003497993981945838, + "loss": 1.3012, + "theoretical_loss": 3.6726451228500006, + "tokens_seen": 1014562816 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034978936810431296, + "loss": 1.4287, + "theoretical_loss": 3.6726227815898165, + "tokens_seen": 1014628352 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034977933801404214, + "loss": 1.2804, + "theoretical_loss": 3.6726004421766567, + "tokens_seen": 1014693888 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003497693079237713, + "loss": 1.408, + "theoretical_loss": 3.672578104610249, + "tokens_seen": 1014759424 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034975927783350056, + "loss": 1.3351, + "theoretical_loss": 3.672555768890322, + "tokens_seen": 1014824960 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003497492477432297, + "loss": 1.3566, + "theoretical_loss": 3.6725334350166037, + "tokens_seen": 1014890496 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003497392176529589, + "loss": 1.3815, + "theoretical_loss": 3.672511102988822, + "tokens_seen": 1014956032 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034972918756268805, + "loss": 1.4018, + "theoretical_loss": 3.6724887728067053, + "tokens_seen": 1015021568 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003497191574724173, + "loss": 1.2988, + "theoretical_loss": 3.672466444469982, + "tokens_seen": 1015087104 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034970912738214647, + "loss": 1.1927, + "theoretical_loss": 3.6724441179783804, + "tokens_seen": 1015152640 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034969909729187565, + "loss": 1.2544, + "theoretical_loss": 3.672421793331629, + "tokens_seen": 1015218176 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034968906720160483, + "loss": 1.314, + "theoretical_loss": 3.672399470529456, + "tokens_seen": 1015283712 + }, + { + "epoch": 0.31, + "learning_rate": 0.000349679037111334, + "loss": 1.403, + "theoretical_loss": 3.6723771495715907, + "tokens_seen": 1015349248 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003496690070210632, + "loss": 1.376, + "theoretical_loss": 3.6723548304577607, + "tokens_seen": 1015414784 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003496589769307924, + "loss": 1.235, + "theoretical_loss": 3.672332513187695, + "tokens_seen": 1015480320 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034964894684052155, + "loss": 1.2209, + "theoretical_loss": 3.6723101977611226, + "tokens_seen": 1015545856 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003496389167502508, + "loss": 1.297, + "theoretical_loss": 3.672287884177772, + "tokens_seen": 1015611392 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003496288866599799, + "loss": 1.31, + "theoretical_loss": 3.6722655724373725, + "tokens_seen": 1015676928 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034961885656970915, + "loss": 1.2377, + "theoretical_loss": 3.672243262539652, + "tokens_seen": 1015742464 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 341726, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.429366111755371, + "objective/train/theoretical_loss": 3.6722209544843407, + "objective/train/tokens_used": 1015808000, + "theoretical_loss": 3.6722209544843407, + "tokens_seen": 1015808000 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034960882647943833, + "loss": 1.362, + "theoretical_loss": 3.6722209544843407, + "tokens_seen": 1015808000 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003495987963891675, + "loss": 1.2198, + "theoretical_loss": 3.6721986482711664, + "tokens_seen": 1015873536 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003495887662988967, + "loss": 1.3563, + "theoretical_loss": 3.6721763438998587, + "tokens_seen": 1015939072 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034957873620862593, + "loss": 1.3976, + "theoretical_loss": 3.672154041370147, + "tokens_seen": 1016004608 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034956870611835506, + "loss": 1.2845, + "theoretical_loss": 3.6721317406817597, + "tokens_seen": 1016070144 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003495586760280843, + "loss": 1.2814, + "theoretical_loss": 3.6721094418344276, + "tokens_seen": 1016135680 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003495486459378134, + "loss": 1.2007, + "theoretical_loss": 3.672087144827878, + "tokens_seen": 1016201216 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034953861584754265, + "loss": 1.3222, + "theoretical_loss": 3.672064849661841, + "tokens_seen": 1016266752 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034952858575727183, + "loss": 1.2613, + "theoretical_loss": 3.672042556336047, + "tokens_seen": 1016332288 + }, + { + "epoch": 0.31, + "learning_rate": 0.000349518555667001, + "loss": 1.2989, + "theoretical_loss": 3.6720202648502234, + "tokens_seen": 1016397824 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003495085255767302, + "loss": 1.4566, + "theoretical_loss": 3.6719979752041016, + "tokens_seen": 1016463360 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003494984954864594, + "loss": 1.2364, + "theoretical_loss": 3.6719756873974103, + "tokens_seen": 1016528896 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034948846539618856, + "loss": 1.386, + "theoretical_loss": 3.6719534014298794, + "tokens_seen": 1016594432 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003494784353059178, + "loss": 1.2996, + "theoretical_loss": 3.671931117301239, + "tokens_seen": 1016659968 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003494684052156469, + "loss": 1.2927, + "theoretical_loss": 3.6719088350112177, + "tokens_seen": 1016725504 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034945837512537616, + "loss": 1.3747, + "theoretical_loss": 3.6718865545595456, + "tokens_seen": 1016791040 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003494483450351053, + "loss": 1.3102, + "theoretical_loss": 3.6718642759459534, + "tokens_seen": 1016856576 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003494383149448345, + "loss": 1.3575, + "theoretical_loss": 3.6718419991701703, + "tokens_seen": 1016922112 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003494282848545637, + "loss": 1.3825, + "theoretical_loss": 3.671819724231927, + "tokens_seen": 1016987648 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003494182547642929, + "loss": 1.3851, + "theoretical_loss": 3.671797451130953, + "tokens_seen": 1017053184 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034940822467402206, + "loss": 1.1994, + "theoretical_loss": 3.6717751798669775, + "tokens_seen": 1017118720 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003493981945837513, + "loss": 1.3298, + "theoretical_loss": 3.671752910439732, + "tokens_seen": 1017184256 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003493881644934804, + "loss": 1.2602, + "theoretical_loss": 3.6717306428489467, + "tokens_seen": 1017249792 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034937813440320966, + "loss": 1.3346, + "theoretical_loss": 3.671708377094351, + "tokens_seen": 1017315328 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003493681043129388, + "loss": 1.3743, + "theoretical_loss": 3.6716861131756753, + "tokens_seen": 1017380864 + }, + { + "epoch": 0.31, + "learning_rate": 0.000349358074222668, + "loss": 1.37, + "theoretical_loss": 3.671663851092651, + "tokens_seen": 1017446400 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003493480441323972, + "loss": 1.3799, + "theoretical_loss": 3.6716415908450073, + "tokens_seen": 1017511936 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003493380140421264, + "loss": 1.3418, + "theoretical_loss": 3.671619332432475, + "tokens_seen": 1017577472 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034932798395185557, + "loss": 1.4203, + "theoretical_loss": 3.671597075854785, + "tokens_seen": 1017643008 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034931795386158475, + "loss": 1.2408, + "theoretical_loss": 3.671574821111668, + "tokens_seen": 1017708544 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034930792377131393, + "loss": 1.2116, + "theoretical_loss": 3.6715525682028547, + "tokens_seen": 1017774080 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034929789368104316, + "loss": 1.3723, + "theoretical_loss": 3.6715303171280746, + "tokens_seen": 1017839616 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003492878635907723, + "loss": 1.3619, + "theoretical_loss": 3.6715080678870597, + "tokens_seen": 1017905152 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003492778335005015, + "loss": 1.3611, + "theoretical_loss": 3.671485820479541, + "tokens_seen": 1017970688 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034926780341023065, + "loss": 1.2206, + "theoretical_loss": 3.671463574905248, + "tokens_seen": 1018036224 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003492577733199599, + "loss": 1.2828, + "theoretical_loss": 3.671441331163913, + "tokens_seen": 1018101760 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034924774322968907, + "loss": 1.2802, + "theoretical_loss": 3.6714190892552665, + "tokens_seen": 1018167296 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034923771313941825, + "loss": 1.3401, + "theoretical_loss": 3.6713968491790396, + "tokens_seen": 1018232832 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034922768304914743, + "loss": 1.3204, + "theoretical_loss": 3.671374610934963, + "tokens_seen": 1018298368 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034921765295887667, + "loss": 1.2988, + "theoretical_loss": 3.671352374522769, + "tokens_seen": 1018363904 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003492076228686058, + "loss": 1.3, + "theoretical_loss": 3.671330139942188, + "tokens_seen": 1018429440 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034919759277833503, + "loss": 1.3511, + "theoretical_loss": 3.6713079071929506, + "tokens_seen": 1018494976 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034918756268806416, + "loss": 1.2575, + "theoretical_loss": 3.671285676274789, + "tokens_seen": 1018560512 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003491775325977934, + "loss": 1.3069, + "theoretical_loss": 3.671263447187435, + "tokens_seen": 1018626048 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034916750250752257, + "loss": 1.4329, + "theoretical_loss": 3.6712412199306192, + "tokens_seen": 1018691584 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034915747241725175, + "loss": 1.3703, + "theoretical_loss": 3.6712189945040734, + "tokens_seen": 1018757120 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034914744232698093, + "loss": 1.3566, + "theoretical_loss": 3.6711967709075295, + "tokens_seen": 1018822656 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003491374122367101, + "loss": 1.4013, + "theoretical_loss": 3.6711745491407184, + "tokens_seen": 1018888192 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003491273821464393, + "loss": 1.3548, + "theoretical_loss": 3.6711523292033723, + "tokens_seen": 1018953728 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034911735205616853, + "loss": 1.2421, + "theoretical_loss": 3.671130111095223, + "tokens_seen": 1019019264 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 342779, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3703640699386597, + "objective/train/theoretical_loss": 3.671107894816002, + "objective/train/tokens_used": 1019084800, + "theoretical_loss": 3.671107894816002, + "tokens_seen": 1019084800 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034910732196589766, + "loss": 1.352, + "theoretical_loss": 3.671107894816002, + "tokens_seen": 1019084800 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003490972918756269, + "loss": 1.2149, + "theoretical_loss": 3.671085680365441, + "tokens_seen": 1019150336 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034908726178535613, + "loss": 1.3694, + "theoretical_loss": 3.6710634677432727, + "tokens_seen": 1019215872 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034907723169508526, + "loss": 1.3056, + "theoretical_loss": 3.6710412569492283, + "tokens_seen": 1019281408 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003490672016048145, + "loss": 1.2998, + "theoretical_loss": 3.67101904798304, + "tokens_seen": 1019346944 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003490571715145436, + "loss": 1.1932, + "theoretical_loss": 3.67099684084444, + "tokens_seen": 1019412480 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034904714142427285, + "loss": 1.4432, + "theoretical_loss": 3.6709746355331605, + "tokens_seen": 1019478016 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034903711133400203, + "loss": 1.1912, + "theoretical_loss": 3.6709524320489333, + "tokens_seen": 1019543552 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003490270812437312, + "loss": 1.3332, + "theoretical_loss": 3.6709302303914906, + "tokens_seen": 1019609088 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003490170511534604, + "loss": 1.3005, + "theoretical_loss": 3.6709080305605655, + "tokens_seen": 1019674624 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003490070210631896, + "loss": 1.2747, + "theoretical_loss": 3.6708858325558893, + "tokens_seen": 1019740160 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034899699097291876, + "loss": 1.3314, + "theoretical_loss": 3.670863636377195, + "tokens_seen": 1019805696 + }, + { + "epoch": 0.31, + "learning_rate": 0.000348986960882648, + "loss": 1.313, + "theoretical_loss": 3.6708414420242157, + "tokens_seen": 1019871232 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003489769307923771, + "loss": 1.1734, + "theoretical_loss": 3.6708192494966827, + "tokens_seen": 1019936768 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034896690070210636, + "loss": 1.2724, + "theoretical_loss": 3.670797058794329, + "tokens_seen": 1020002304 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003489568706118355, + "loss": 1.4116, + "theoretical_loss": 3.6707748699168876, + "tokens_seen": 1020067840 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003489468405215647, + "loss": 1.3957, + "theoretical_loss": 3.670752682864091, + "tokens_seen": 1020133376 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003489368104312939, + "loss": 1.2903, + "theoretical_loss": 3.670730497635671, + "tokens_seen": 1020198912 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003489267803410231, + "loss": 1.3892, + "theoretical_loss": 3.670708314231362, + "tokens_seen": 1020264448 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034891675025075226, + "loss": 1.1731, + "theoretical_loss": 3.6706861326508964, + "tokens_seen": 1020329984 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003489067201604815, + "loss": 1.3001, + "theoretical_loss": 3.6706639528940066, + "tokens_seen": 1020395520 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003488966900702106, + "loss": 1.3263, + "theoretical_loss": 3.6706417749604263, + "tokens_seen": 1020461056 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034888665997993986, + "loss": 1.3148, + "theoretical_loss": 3.6706195988498873, + "tokens_seen": 1020526592 + }, + { + "epoch": 0.31, + "learning_rate": 0.000348876629889669, + "loss": 1.3127, + "theoretical_loss": 3.670597424562124, + "tokens_seen": 1020592128 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003488665997993982, + "loss": 1.267, + "theoretical_loss": 3.670575252096869, + "tokens_seen": 1020657664 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003488565697091274, + "loss": 1.3169, + "theoretical_loss": 3.670553081453855, + "tokens_seen": 1020723200 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003488465396188566, + "loss": 1.3391, + "theoretical_loss": 3.6705309126328163, + "tokens_seen": 1020788736 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034883650952858577, + "loss": 1.3356, + "theoretical_loss": 3.6705087456334855, + "tokens_seen": 1020854272 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034882647943831495, + "loss": 1.3125, + "theoretical_loss": 3.6704865804555964, + "tokens_seen": 1020919808 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034881644934804413, + "loss": 1.2674, + "theoretical_loss": 3.6704644170988816, + "tokens_seen": 1020985344 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034880641925777336, + "loss": 1.2232, + "theoretical_loss": 3.6704422555630756, + "tokens_seen": 1021050880 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003487963891675025, + "loss": 1.3623, + "theoretical_loss": 3.6704200958479114, + "tokens_seen": 1021116416 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003487863590772317, + "loss": 1.3107, + "theoretical_loss": 3.6703979379531226, + "tokens_seen": 1021181952 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034877632898696085, + "loss": 1.2799, + "theoretical_loss": 3.6703757818784433, + "tokens_seen": 1021247488 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003487662988966901, + "loss": 1.3005, + "theoretical_loss": 3.6703536276236064, + "tokens_seen": 1021313024 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034875626880641927, + "loss": 1.3437, + "theoretical_loss": 3.6703314751883465, + "tokens_seen": 1021378560 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034874623871614845, + "loss": 1.4002, + "theoretical_loss": 3.670309324572396, + "tokens_seen": 1021444096 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034873620862587763, + "loss": 1.2611, + "theoretical_loss": 3.670287175775491, + "tokens_seen": 1021509632 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034872617853560687, + "loss": 1.2815, + "theoretical_loss": 3.6702650287973637, + "tokens_seen": 1021575168 + }, + { + "epoch": 0.31, + "learning_rate": 0.000348716148445336, + "loss": 1.3342, + "theoretical_loss": 3.6702428836377488, + "tokens_seen": 1021640704 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034870611835506523, + "loss": 1.2594, + "theoretical_loss": 3.67022074029638, + "tokens_seen": 1021706240 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034869608826479436, + "loss": 1.3242, + "theoretical_loss": 3.6701985987729913, + "tokens_seen": 1021771776 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003486860581745236, + "loss": 1.2844, + "theoretical_loss": 3.670176459067317, + "tokens_seen": 1021837312 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034867602808425277, + "loss": 1.3955, + "theoretical_loss": 3.6701543211790923, + "tokens_seen": 1021902848 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034866599799398195, + "loss": 1.2112, + "theoretical_loss": 3.6701321851080495, + "tokens_seen": 1021968384 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034865596790371113, + "loss": 1.428, + "theoretical_loss": 3.670110050853925, + "tokens_seen": 1022033920 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003486459378134403, + "loss": 1.3772, + "theoretical_loss": 3.670087918416451, + "tokens_seen": 1022099456 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003486359077231695, + "loss": 1.345, + "theoretical_loss": 3.670065787795364, + "tokens_seen": 1022164992 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034862587763289873, + "loss": 1.3787, + "theoretical_loss": 3.670043658990397, + "tokens_seen": 1022230528 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034861584754262786, + "loss": 1.2689, + "theoretical_loss": 3.670021532001286, + "tokens_seen": 1022296064 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 343640, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3846840858459473, + "objective/train/theoretical_loss": 3.6699994068277637, + "objective/train/tokens_used": 1022361600, + "theoretical_loss": 3.6699994068277637, + "tokens_seen": 1022361600 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003486058174523571, + "loss": 1.4014, + "theoretical_loss": 3.6699994068277637, + "tokens_seen": 1022361600 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003485957873620862, + "loss": 1.3847, + "theoretical_loss": 3.6699772834695663, + "tokens_seen": 1022427136 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034858575727181546, + "loss": 1.254, + "theoretical_loss": 3.6699551619264277, + "tokens_seen": 1022492672 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034857572718154464, + "loss": 1.328, + "theoretical_loss": 3.6699330421980836, + "tokens_seen": 1022558208 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003485656970912738, + "loss": 1.3533, + "theoretical_loss": 3.6699109242842676, + "tokens_seen": 1022623744 + }, + { + "epoch": 0.31, + "learning_rate": 0.000348555667001003, + "loss": 1.3927, + "theoretical_loss": 3.669888808184715, + "tokens_seen": 1022689280 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034854563691073223, + "loss": 1.3077, + "theoretical_loss": 3.6698666938991615, + "tokens_seen": 1022754816 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034853560682046136, + "loss": 1.4259, + "theoretical_loss": 3.6698445814273413, + "tokens_seen": 1022820352 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003485255767301906, + "loss": 1.3361, + "theoretical_loss": 3.669822470768989, + "tokens_seen": 1022885888 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003485155466399197, + "loss": 1.3696, + "theoretical_loss": 3.669800361923841, + "tokens_seen": 1022951424 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034850551654964896, + "loss": 1.3523, + "theoretical_loss": 3.669778254891632, + "tokens_seen": 1023016960 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034849548645937814, + "loss": 1.414, + "theoretical_loss": 3.6697561496720965, + "tokens_seen": 1023082496 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003484854563691073, + "loss": 1.4198, + "theoretical_loss": 3.6697340462649706, + "tokens_seen": 1023148032 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003484754262788365, + "loss": 1.3498, + "theoretical_loss": 3.669711944669989, + "tokens_seen": 1023213568 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003484653961885657, + "loss": 1.2997, + "theoretical_loss": 3.669689844886887, + "tokens_seen": 1023279104 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034845536609829486, + "loss": 1.3633, + "theoretical_loss": 3.669667746915401, + "tokens_seen": 1023344640 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003484453360080241, + "loss": 1.3502, + "theoretical_loss": 3.6696456507552657, + "tokens_seen": 1023410176 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003484353059177532, + "loss": 1.3549, + "theoretical_loss": 3.6696235564062167, + "tokens_seen": 1023475712 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034842527582748246, + "loss": 1.3007, + "theoretical_loss": 3.6696014638679904, + "tokens_seen": 1023541248 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034841524573721164, + "loss": 1.2977, + "theoretical_loss": 3.669579373140321, + "tokens_seen": 1023606784 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003484052156469408, + "loss": 1.3189, + "theoretical_loss": 3.669557284222946, + "tokens_seen": 1023672320 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034839518555667, + "loss": 1.3357, + "theoretical_loss": 3.669535197115599, + "tokens_seen": 1023737856 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003483851554663992, + "loss": 1.2904, + "theoretical_loss": 3.6695131118180173, + "tokens_seen": 1023803392 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034837512537612837, + "loss": 1.3272, + "theoretical_loss": 3.6694910283299373, + "tokens_seen": 1023868928 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003483650952858576, + "loss": 1.3716, + "theoretical_loss": 3.6694689466510937, + "tokens_seen": 1023934464 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034835506519558673, + "loss": 1.2912, + "theoretical_loss": 3.6694468667812226, + "tokens_seen": 1024000000 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034834503510531597, + "loss": 1.3119, + "theoretical_loss": 3.66942478872006, + "tokens_seen": 1024065536 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034833500501504515, + "loss": 1.1769, + "theoretical_loss": 3.669402712467343, + "tokens_seen": 1024131072 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034832497492477433, + "loss": 1.3114, + "theoretical_loss": 3.6693806380228073, + "tokens_seen": 1024196608 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034831494483450356, + "loss": 1.3241, + "theoretical_loss": 3.6693585653861884, + "tokens_seen": 1024262144 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003483049147442327, + "loss": 1.2577, + "theoretical_loss": 3.669336494557223, + "tokens_seen": 1024327680 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003482948846539619, + "loss": 1.4034, + "theoretical_loss": 3.6693144255356485, + "tokens_seen": 1024393216 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034828485456369105, + "loss": 1.2737, + "theoretical_loss": 3.6692923583211994, + "tokens_seen": 1024458752 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003482748244734203, + "loss": 1.2535, + "theoretical_loss": 3.6692702929136134, + "tokens_seen": 1024524288 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034826479438314947, + "loss": 1.3099, + "theoretical_loss": 3.6692482293126263, + "tokens_seen": 1024589824 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034825476429287865, + "loss": 1.4363, + "theoretical_loss": 3.669226167517975, + "tokens_seen": 1024655360 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034824473420260783, + "loss": 1.283, + "theoretical_loss": 3.6692041075293966, + "tokens_seen": 1024720896 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034823470411233707, + "loss": 1.5273, + "theoretical_loss": 3.6691820493466265, + "tokens_seen": 1024786432 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003482246740220662, + "loss": 1.2659, + "theoretical_loss": 3.6691599929694023, + "tokens_seen": 1024851968 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034821464393179543, + "loss": 1.3702, + "theoretical_loss": 3.6691379383974607, + "tokens_seen": 1024917504 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034820461384152456, + "loss": 1.2649, + "theoretical_loss": 3.6691158856305384, + "tokens_seen": 1024983040 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003481945837512538, + "loss": 1.279, + "theoretical_loss": 3.6690938346683724, + "tokens_seen": 1025048576 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034818455366098297, + "loss": 1.3158, + "theoretical_loss": 3.6690717855106993, + "tokens_seen": 1025114112 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034817452357071215, + "loss": 1.2979, + "theoretical_loss": 3.6690497381572564, + "tokens_seen": 1025179648 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034816449348044133, + "loss": 1.3071, + "theoretical_loss": 3.6690276926077807, + "tokens_seen": 1025245184 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003481544633901705, + "loss": 1.2846, + "theoretical_loss": 3.669005648862009, + "tokens_seen": 1025310720 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003481444332998997, + "loss": 1.3723, + "theoretical_loss": 3.668983606919679, + "tokens_seen": 1025376256 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034813440320962893, + "loss": 1.3768, + "theoretical_loss": 3.668961566780527, + "tokens_seen": 1025441792 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034812437311935806, + "loss": 1.3707, + "theoretical_loss": 3.6689395284442914, + "tokens_seen": 1025507328 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003481143430290873, + "loss": 1.2801, + "theoretical_loss": 3.668917491910709, + "tokens_seen": 1025572864 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 344743, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4053757190704346, + "objective/train/theoretical_loss": 3.6688954571795165, + "objective/train/tokens_used": 1025638400, + "theoretical_loss": 3.6688954571795165, + "tokens_seen": 1025638400 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003481043129388164, + "loss": 1.3604, + "theoretical_loss": 3.6688954571795165, + "tokens_seen": 1025638400 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034809428284854566, + "loss": 1.3095, + "theoretical_loss": 3.6688734242504526, + "tokens_seen": 1025703936 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034808425275827484, + "loss": 1.2461, + "theoretical_loss": 3.668851393123254, + "tokens_seen": 1025769472 + }, + { + "epoch": 0.31, + "learning_rate": 0.000348074222668004, + "loss": 1.358, + "theoretical_loss": 3.6688293637976583, + "tokens_seen": 1025835008 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003480641925777332, + "loss": 1.3556, + "theoretical_loss": 3.668807336273403, + "tokens_seen": 1025900544 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034805416248746243, + "loss": 1.4805, + "theoretical_loss": 3.668785310550226, + "tokens_seen": 1025966080 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034804413239719156, + "loss": 1.3115, + "theoretical_loss": 3.6687632866278657, + "tokens_seen": 1026031616 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003480341023069208, + "loss": 1.3004, + "theoretical_loss": 3.668741264506058, + "tokens_seen": 1026097152 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003480240722166499, + "loss": 1.3068, + "theoretical_loss": 3.668719244184542, + "tokens_seen": 1026162688 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034801404212637916, + "loss": 1.4093, + "theoretical_loss": 3.668697225663056, + "tokens_seen": 1026228224 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034800401203610834, + "loss": 1.3162, + "theoretical_loss": 3.6686752089413366, + "tokens_seen": 1026293760 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003479939819458375, + "loss": 1.3418, + "theoretical_loss": 3.6686531940191225, + "tokens_seen": 1026359296 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003479839518555667, + "loss": 1.365, + "theoretical_loss": 3.668631180896152, + "tokens_seen": 1026424832 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003479739217652959, + "loss": 1.373, + "theoretical_loss": 3.6686091695721625, + "tokens_seen": 1026490368 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034796389167502506, + "loss": 1.2249, + "theoretical_loss": 3.6685871600468927, + "tokens_seen": 1026555904 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003479538615847543, + "loss": 1.3581, + "theoretical_loss": 3.6685651523200806, + "tokens_seen": 1026621440 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034794383149448343, + "loss": 1.3329, + "theoretical_loss": 3.6685431463914644, + "tokens_seen": 1026686976 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034793380140421266, + "loss": 1.3539, + "theoretical_loss": 3.668521142260783, + "tokens_seen": 1026752512 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034792377131394184, + "loss": 1.3591, + "theoretical_loss": 3.668499139927773, + "tokens_seen": 1026818048 + }, + { + "epoch": 0.31, + "learning_rate": 0.000347913741223671, + "loss": 1.3186, + "theoretical_loss": 3.668477139392175, + "tokens_seen": 1026883584 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003479037111334002, + "loss": 1.3559, + "theoretical_loss": 3.668455140653726, + "tokens_seen": 1026949120 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003478936810431294, + "loss": 1.3561, + "theoretical_loss": 3.668433143712165, + "tokens_seen": 1027014656 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034788365095285857, + "loss": 1.3554, + "theoretical_loss": 3.6684111485672313, + "tokens_seen": 1027080192 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003478736208625878, + "loss": 1.4045, + "theoretical_loss": 3.6683891552186623, + "tokens_seen": 1027145728 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034786359077231693, + "loss": 1.3621, + "theoretical_loss": 3.668367163666197, + "tokens_seen": 1027211264 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034785356068204617, + "loss": 1.312, + "theoretical_loss": 3.6683451739095747, + "tokens_seen": 1027276800 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003478435305917753, + "loss": 1.3776, + "theoretical_loss": 3.668323185948534, + "tokens_seen": 1027342336 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034783350050150453, + "loss": 1.3043, + "theoretical_loss": 3.668301199782813, + "tokens_seen": 1027407872 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003478234704112337, + "loss": 1.3181, + "theoretical_loss": 3.6682792154121517, + "tokens_seen": 1027473408 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003478134403209629, + "loss": 1.3245, + "theoretical_loss": 3.6682572328362886, + "tokens_seen": 1027538944 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034780341023069207, + "loss": 1.3325, + "theoretical_loss": 3.6682352520549624, + "tokens_seen": 1027604480 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034779338014042125, + "loss": 1.3568, + "theoretical_loss": 3.668213273067913, + "tokens_seen": 1027670016 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034778335005015043, + "loss": 1.1911, + "theoretical_loss": 3.668191295874878, + "tokens_seen": 1027735552 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034777331995987967, + "loss": 1.3667, + "theoretical_loss": 3.6681693204755987, + "tokens_seen": 1027801088 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003477632898696088, + "loss": 1.2668, + "theoretical_loss": 3.6681473468698123, + "tokens_seen": 1027866624 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034775325977933803, + "loss": 1.289, + "theoretical_loss": 3.6681253750572593, + "tokens_seen": 1027932160 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003477432296890672, + "loss": 1.349, + "theoretical_loss": 3.668103405037679, + "tokens_seen": 1027997696 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003477331995987964, + "loss": 1.3225, + "theoretical_loss": 3.66808143681081, + "tokens_seen": 1028063232 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003477231695085256, + "loss": 1.3056, + "theoretical_loss": 3.668059470376393, + "tokens_seen": 1028128768 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034771313941825476, + "loss": 1.2929, + "theoretical_loss": 3.6680375057341665, + "tokens_seen": 1028194304 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034770310932798394, + "loss": 1.3314, + "theoretical_loss": 3.66801554288387, + "tokens_seen": 1028259840 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034769307923771317, + "loss": 1.398, + "theoretical_loss": 3.6679935818252436, + "tokens_seen": 1028325376 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003476830491474423, + "loss": 1.4309, + "theoretical_loss": 3.667971622558027, + "tokens_seen": 1028390912 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034767301905717153, + "loss": 1.2899, + "theoretical_loss": 3.6679496650819594, + "tokens_seen": 1028456448 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034766298896690066, + "loss": 1.2794, + "theoretical_loss": 3.6679277093967815, + "tokens_seen": 1028521984 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003476529588766299, + "loss": 1.3074, + "theoretical_loss": 3.6679057555022325, + "tokens_seen": 1028587520 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003476429287863591, + "loss": 1.3971, + "theoretical_loss": 3.667883803398052, + "tokens_seen": 1028653056 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034763289869608826, + "loss": 1.2999, + "theoretical_loss": 3.6678618530839806, + "tokens_seen": 1028718592 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034762286860581744, + "loss": 1.2748, + "theoretical_loss": 3.6678399045597576, + "tokens_seen": 1028784128 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003476128385155466, + "loss": 1.3753, + "theoretical_loss": 3.6678179578251235, + "tokens_seen": 1028849664 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 345784, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4136476516723633, + "objective/train/theoretical_loss": 3.667796012879819, + "objective/train/tokens_used": 1028915200, + "theoretical_loss": 3.667796012879819, + "tokens_seen": 1028915200 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003476028084252758, + "loss": 1.3497, + "theoretical_loss": 3.667796012879819, + "tokens_seen": 1028915200 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034759277833500504, + "loss": 1.3627, + "theoretical_loss": 3.667774069723583, + "tokens_seen": 1028980736 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003475827482447342, + "loss": 1.1845, + "theoretical_loss": 3.667752128356156, + "tokens_seen": 1029046272 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003475727181544634, + "loss": 1.2995, + "theoretical_loss": 3.6677301887772793, + "tokens_seen": 1029111808 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034756268806419263, + "loss": 1.3407, + "theoretical_loss": 3.667708250986692, + "tokens_seen": 1029177344 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034755265797392176, + "loss": 1.4273, + "theoretical_loss": 3.6676863149841354, + "tokens_seen": 1029242880 + }, + { + "epoch": 0.31, + "learning_rate": 0.000347542627883651, + "loss": 1.4416, + "theoretical_loss": 3.6676643807693496, + "tokens_seen": 1029308416 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003475325977933801, + "loss": 1.3666, + "theoretical_loss": 3.6676424483420744, + "tokens_seen": 1029373952 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034752256770310936, + "loss": 1.3371, + "theoretical_loss": 3.6676205177020513, + "tokens_seen": 1029439488 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034751253761283854, + "loss": 1.2521, + "theoretical_loss": 3.6675985888490206, + "tokens_seen": 1029505024 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003475025075225677, + "loss": 1.3468, + "theoretical_loss": 3.667576661782723, + "tokens_seen": 1029570560 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003474924774322969, + "loss": 1.4378, + "theoretical_loss": 3.6675547365028986, + "tokens_seen": 1029636096 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003474824473420261, + "loss": 1.2502, + "theoretical_loss": 3.6675328130092892, + "tokens_seen": 1029701632 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034747241725175527, + "loss": 1.3201, + "theoretical_loss": 3.6675108913016348, + "tokens_seen": 1029767168 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003474623871614845, + "loss": 1.3442, + "theoretical_loss": 3.6674889713796763, + "tokens_seen": 1029832704 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034745235707121363, + "loss": 1.2304, + "theoretical_loss": 3.667467053243155, + "tokens_seen": 1029898240 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034744232698094286, + "loss": 1.3315, + "theoretical_loss": 3.667445136891812, + "tokens_seen": 1029963776 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034743229689067204, + "loss": 1.3976, + "theoretical_loss": 3.6674232223253878, + "tokens_seen": 1030029312 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003474222668004012, + "loss": 1.2826, + "theoretical_loss": 3.6674013095436235, + "tokens_seen": 1030094848 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003474122367101304, + "loss": 1.3241, + "theoretical_loss": 3.6673793985462604, + "tokens_seen": 1030160384 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003474022066198596, + "loss": 1.3772, + "theoretical_loss": 3.6673574893330403, + "tokens_seen": 1030225920 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034739217652958877, + "loss": 1.4076, + "theoretical_loss": 3.6673355819037035, + "tokens_seen": 1030291456 + }, + { + "epoch": 0.31, + "learning_rate": 0.000347382146439318, + "loss": 1.3639, + "theoretical_loss": 3.6673136762579914, + "tokens_seen": 1030356992 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034737211634904713, + "loss": 1.4595, + "theoretical_loss": 3.6672917723956457, + "tokens_seen": 1030422528 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034736208625877637, + "loss": 1.2202, + "theoretical_loss": 3.6672698703164084, + "tokens_seen": 1030488064 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003473520561685055, + "loss": 1.3298, + "theoretical_loss": 3.6672479700200196, + "tokens_seen": 1030553600 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034734202607823473, + "loss": 1.2696, + "theoretical_loss": 3.6672260715062217, + "tokens_seen": 1030619136 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003473319959879639, + "loss": 1.4061, + "theoretical_loss": 3.667204174774756, + "tokens_seen": 1030684672 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003473219658976931, + "loss": 1.3593, + "theoretical_loss": 3.6671822798253637, + "tokens_seen": 1030750208 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034731193580742227, + "loss": 1.3432, + "theoretical_loss": 3.6671603866577875, + "tokens_seen": 1030815744 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034730190571715145, + "loss": 1.2753, + "theoretical_loss": 3.6671384952717685, + "tokens_seen": 1030881280 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034729187562688063, + "loss": 1.32, + "theoretical_loss": 3.6671166056670486, + "tokens_seen": 1030946816 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034728184553660987, + "loss": 1.3315, + "theoretical_loss": 3.6670947178433693, + "tokens_seen": 1031012352 + }, + { + "epoch": 0.31, + "learning_rate": 0.000347271815446339, + "loss": 1.3421, + "theoretical_loss": 3.667072831800473, + "tokens_seen": 1031077888 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034726178535606823, + "loss": 1.3448, + "theoretical_loss": 3.6670509475381015, + "tokens_seen": 1031143424 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003472517552657974, + "loss": 1.421, + "theoretical_loss": 3.6670290650559965, + "tokens_seen": 1031208960 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003472417251755266, + "loss": 1.2095, + "theoretical_loss": 3.6670071843539005, + "tokens_seen": 1031274496 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003472316950852558, + "loss": 1.2906, + "theoretical_loss": 3.666985305431555, + "tokens_seen": 1031340032 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034722166499498496, + "loss": 1.337, + "theoretical_loss": 3.666963428288703, + "tokens_seen": 1031405568 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034721163490471414, + "loss": 1.3209, + "theoretical_loss": 3.666941552925086, + "tokens_seen": 1031471104 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034720160481444337, + "loss": 1.3384, + "theoretical_loss": 3.6669196793404466, + "tokens_seen": 1031536640 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003471915747241725, + "loss": 1.3126, + "theoretical_loss": 3.666897807534527, + "tokens_seen": 1031602176 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034718154463390173, + "loss": 1.3757, + "theoretical_loss": 3.6668759375070694, + "tokens_seen": 1031667712 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034717151454363086, + "loss": 1.2543, + "theoretical_loss": 3.6668540692578167, + "tokens_seen": 1031733248 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003471614844533601, + "loss": 1.2691, + "theoretical_loss": 3.6668322027865106, + "tokens_seen": 1031798784 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003471514543630893, + "loss": 1.3573, + "theoretical_loss": 3.666810338092895, + "tokens_seen": 1031864320 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034714142427281846, + "loss": 1.3294, + "theoretical_loss": 3.666788475176711, + "tokens_seen": 1031929856 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034713139418254764, + "loss": 1.3217, + "theoretical_loss": 3.6667666140377015, + "tokens_seen": 1031995392 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003471213640922768, + "loss": 1.2915, + "theoretical_loss": 3.6667447546756105, + "tokens_seen": 1032060928 + }, + { + "epoch": 0.31, + "learning_rate": 0.000347111334002006, + "loss": 1.3213, + "theoretical_loss": 3.666722897090179, + "tokens_seen": 1032126464 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 346591, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.268754243850708, + "objective/train/theoretical_loss": 3.6667010412811507, + "objective/train/tokens_used": 1032192000, + "theoretical_loss": 3.6667010412811507, + "tokens_seen": 1032192000 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034710130391173524, + "loss": 1.2619, + "theoretical_loss": 3.6667010412811507, + "tokens_seen": 1032192000 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034709127382146436, + "loss": 1.2697, + "theoretical_loss": 3.6666791872482687, + "tokens_seen": 1032257536 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003470812437311936, + "loss": 1.2999, + "theoretical_loss": 3.6666573349912754, + "tokens_seen": 1032323072 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003470712136409228, + "loss": 1.3436, + "theoretical_loss": 3.6666354845099143, + "tokens_seen": 1032388608 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034706118355065196, + "loss": 1.316, + "theoretical_loss": 3.6666136358039276, + "tokens_seen": 1032454144 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034705115346038114, + "loss": 1.2996, + "theoretical_loss": 3.666591788873059, + "tokens_seen": 1032519680 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003470411233701103, + "loss": 1.3722, + "theoretical_loss": 3.666569943717051, + "tokens_seen": 1032585216 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003470310932798395, + "loss": 1.2819, + "theoretical_loss": 3.666548100335648, + "tokens_seen": 1032650752 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034702106318956874, + "loss": 1.277, + "theoretical_loss": 3.6665262587285916, + "tokens_seen": 1032716288 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034701103309929787, + "loss": 1.3384, + "theoretical_loss": 3.6665044188956264, + "tokens_seen": 1032781824 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003470010030090271, + "loss": 1.3583, + "theoretical_loss": 3.6664825808364956, + "tokens_seen": 1032847360 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034699097291875623, + "loss": 1.3738, + "theoretical_loss": 3.6664607445509416, + "tokens_seen": 1032912896 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034698094282848547, + "loss": 1.2924, + "theoretical_loss": 3.6664389100387087, + "tokens_seen": 1032978432 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034697091273821465, + "loss": 1.4634, + "theoretical_loss": 3.66641707729954, + "tokens_seen": 1033043968 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034696088264794383, + "loss": 1.2886, + "theoretical_loss": 3.6663952463331797, + "tokens_seen": 1033109504 + }, + { + "epoch": 0.31, + "learning_rate": 0.000346950852557673, + "loss": 1.2344, + "theoretical_loss": 3.6663734171393707, + "tokens_seen": 1033175040 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034694082246740224, + "loss": 1.3822, + "theoretical_loss": 3.666351589717857, + "tokens_seen": 1033240576 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034693079237713137, + "loss": 1.4795, + "theoretical_loss": 3.666329764068382, + "tokens_seen": 1033306112 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003469207622868606, + "loss": 1.3377, + "theoretical_loss": 3.6663079401906895, + "tokens_seen": 1033371648 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034691073219658973, + "loss": 1.3474, + "theoretical_loss": 3.666286118084524, + "tokens_seen": 1033437184 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034690070210631897, + "loss": 1.4373, + "theoretical_loss": 3.6662642977496285, + "tokens_seen": 1033502720 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034689067201604815, + "loss": 1.3706, + "theoretical_loss": 3.6662424791857475, + "tokens_seen": 1033568256 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034688064192577733, + "loss": 1.3529, + "theoretical_loss": 3.666220662392625, + "tokens_seen": 1033633792 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003468706118355065, + "loss": 1.2275, + "theoretical_loss": 3.6661988473700045, + "tokens_seen": 1033699328 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003468605817452357, + "loss": 1.3444, + "theoretical_loss": 3.66617703411763, + "tokens_seen": 1033764864 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003468505516549649, + "loss": 1.3818, + "theoretical_loss": 3.6661552226352465, + "tokens_seen": 1033830400 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003468405215646941, + "loss": 1.4046, + "theoretical_loss": 3.6661334129225978, + "tokens_seen": 1033895936 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003468304914744233, + "loss": 1.2376, + "theoretical_loss": 3.6661116049794273, + "tokens_seen": 1033961472 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034682046138415247, + "loss": 1.4121, + "theoretical_loss": 3.666089798805481, + "tokens_seen": 1034027008 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034681043129388165, + "loss": 1.399, + "theoretical_loss": 3.6660679944005015, + "tokens_seen": 1034092544 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034680040120361083, + "loss": 1.3505, + "theoretical_loss": 3.6660461917642344, + "tokens_seen": 1034158080 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034679037111334007, + "loss": 1.3326, + "theoretical_loss": 3.666024390896424, + "tokens_seen": 1034223616 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003467803410230692, + "loss": 1.2273, + "theoretical_loss": 3.666002591796814, + "tokens_seen": 1034289152 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034677031093279843, + "loss": 1.3588, + "theoretical_loss": 3.665980794465149, + "tokens_seen": 1034354688 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003467602808425276, + "loss": 1.4499, + "theoretical_loss": 3.6659589989011754, + "tokens_seen": 1034420224 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003467502507522568, + "loss": 1.4644, + "theoretical_loss": 3.665937205104636, + "tokens_seen": 1034485760 + }, + { + "epoch": 0.31, + "learning_rate": 0.000346740220661986, + "loss": 1.4102, + "theoretical_loss": 3.665915413075276, + "tokens_seen": 1034551296 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034673019057171516, + "loss": 1.243, + "theoretical_loss": 3.66589362281284, + "tokens_seen": 1034616832 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034672016048144434, + "loss": 1.2464, + "theoretical_loss": 3.6658718343170738, + "tokens_seen": 1034682368 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034671013039117357, + "loss": 1.3177, + "theoretical_loss": 3.665850047587721, + "tokens_seen": 1034747904 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003467001003009027, + "loss": 1.326, + "theoretical_loss": 3.6658282626245278, + "tokens_seen": 1034813440 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034669007021063193, + "loss": 1.2948, + "theoretical_loss": 3.6658064794272383, + "tokens_seen": 1034878976 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034668004012036106, + "loss": 1.386, + "theoretical_loss": 3.6657846979955973, + "tokens_seen": 1034944512 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003466700100300903, + "loss": 1.3482, + "theoretical_loss": 3.6657629183293508, + "tokens_seen": 1035010048 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003466599799398195, + "loss": 1.3274, + "theoretical_loss": 3.665741140428243, + "tokens_seen": 1035075584 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034664994984954866, + "loss": 1.3586, + "theoretical_loss": 3.66571936429202, + "tokens_seen": 1035141120 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034663991975927784, + "loss": 1.3707, + "theoretical_loss": 3.6656975899204265, + "tokens_seen": 1035206656 + }, + { + "epoch": 0.31, + "learning_rate": 0.000346629889669007, + "loss": 1.3046, + "theoretical_loss": 3.665675817313208, + "tokens_seen": 1035272192 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003466198595787362, + "loss": 1.4153, + "theoretical_loss": 3.6656540464701095, + "tokens_seen": 1035337728 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034660982948846544, + "loss": 1.3662, + "theoretical_loss": 3.665632277390877, + "tokens_seen": 1035403264 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 347864, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2771682739257812, + "objective/train/theoretical_loss": 3.665610510075256, + "objective/train/tokens_used": 1035468800, + "theoretical_loss": 3.665610510075256, + "tokens_seen": 1035468800 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034659979939819456, + "loss": 1.3185, + "theoretical_loss": 3.665610510075256, + "tokens_seen": 1035468800 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003465897693079238, + "loss": 1.3261, + "theoretical_loss": 3.6655887445229913, + "tokens_seen": 1035534336 + }, + { + "epoch": 0.31, + "learning_rate": 0.000346579739217653, + "loss": 1.4033, + "theoretical_loss": 3.6655669807338285, + "tokens_seen": 1035599872 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034656970912738216, + "loss": 1.1948, + "theoretical_loss": 3.665545218707514, + "tokens_seen": 1035665408 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034655967903711134, + "loss": 1.3448, + "theoretical_loss": 3.6655234584437926, + "tokens_seen": 1035730944 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003465496489468405, + "loss": 1.3766, + "theoretical_loss": 3.665501699942411, + "tokens_seen": 1035796480 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003465396188565697, + "loss": 1.2569, + "theoretical_loss": 3.665479943203114, + "tokens_seen": 1035862016 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034652958876629894, + "loss": 1.348, + "theoretical_loss": 3.6654581882256485, + "tokens_seen": 1035927552 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034651955867602807, + "loss": 1.4081, + "theoretical_loss": 3.6654364350097595, + "tokens_seen": 1035993088 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003465095285857573, + "loss": 1.3822, + "theoretical_loss": 3.665414683555193, + "tokens_seen": 1036058624 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034649949849548643, + "loss": 1.41, + "theoretical_loss": 3.665392933861695, + "tokens_seen": 1036124160 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034648946840521567, + "loss": 1.2556, + "theoretical_loss": 3.665371185929012, + "tokens_seen": 1036189696 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034647943831494485, + "loss": 1.2563, + "theoretical_loss": 3.6653494397568895, + "tokens_seen": 1036255232 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034646940822467403, + "loss": 1.4066, + "theoretical_loss": 3.6653276953450744, + "tokens_seen": 1036320768 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003464593781344032, + "loss": 1.3516, + "theoretical_loss": 3.6653059526933123, + "tokens_seen": 1036386304 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034644934804413244, + "loss": 1.3604, + "theoretical_loss": 3.6652842118013496, + "tokens_seen": 1036451840 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034643931795386157, + "loss": 1.2521, + "theoretical_loss": 3.6652624726689327, + "tokens_seen": 1036517376 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003464292878635908, + "loss": 1.3102, + "theoretical_loss": 3.6652407352958076, + "tokens_seen": 1036582912 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034641925777331993, + "loss": 1.3678, + "theoretical_loss": 3.665218999681721, + "tokens_seen": 1036648448 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034640922768304917, + "loss": 1.314, + "theoretical_loss": 3.6651972658264196, + "tokens_seen": 1036713984 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034639919759277835, + "loss": 1.3378, + "theoretical_loss": 3.66517553372965, + "tokens_seen": 1036779520 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034638916750250753, + "loss": 1.4261, + "theoretical_loss": 3.6651538033911577, + "tokens_seen": 1036845056 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003463791374122367, + "loss": 1.3613, + "theoretical_loss": 3.6651320748106904, + "tokens_seen": 1036910592 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003463691073219659, + "loss": 1.4373, + "theoretical_loss": 3.6651103479879943, + "tokens_seen": 1036976128 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003463590772316951, + "loss": 1.4088, + "theoretical_loss": 3.665088622922817, + "tokens_seen": 1037041664 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003463490471414243, + "loss": 1.4662, + "theoretical_loss": 3.665066899614904, + "tokens_seen": 1037107200 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034633901705115344, + "loss": 1.3562, + "theoretical_loss": 3.665045178064002, + "tokens_seen": 1037172736 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034632898696088267, + "loss": 1.4353, + "theoretical_loss": 3.6650234582698595, + "tokens_seen": 1037238272 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003463189568706118, + "loss": 1.3189, + "theoretical_loss": 3.665001740232222, + "tokens_seen": 1037303808 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034630892678034103, + "loss": 1.3323, + "theoretical_loss": 3.664980023950837, + "tokens_seen": 1037369344 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003462988966900702, + "loss": 1.4033, + "theoretical_loss": 3.664958309425452, + "tokens_seen": 1037434880 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003462888665997994, + "loss": 1.3439, + "theoretical_loss": 3.664936596655813, + "tokens_seen": 1037500416 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003462788365095286, + "loss": 1.2549, + "theoretical_loss": 3.664914885641668, + "tokens_seen": 1037565952 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003462688064192578, + "loss": 1.2725, + "theoretical_loss": 3.664893176382764, + "tokens_seen": 1037631488 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034625877632898694, + "loss": 1.2504, + "theoretical_loss": 3.6648714688788484, + "tokens_seen": 1037697024 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003462487462387162, + "loss": 1.4165, + "theoretical_loss": 3.664849763129668, + "tokens_seen": 1037762560 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003462387161484453, + "loss": 1.3232, + "theoretical_loss": 3.6648280591349707, + "tokens_seen": 1037828096 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034622868605817454, + "loss": 1.3178, + "theoretical_loss": 3.6648063568945033, + "tokens_seen": 1037893632 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003462186559679037, + "loss": 1.2776, + "theoretical_loss": 3.6647846564080138, + "tokens_seen": 1037959168 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003462086258776329, + "loss": 1.2982, + "theoretical_loss": 3.66476295767525, + "tokens_seen": 1038024704 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003461985957873621, + "loss": 1.3405, + "theoretical_loss": 3.6647412606959584, + "tokens_seen": 1038090240 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034618856569709126, + "loss": 1.431, + "theoretical_loss": 3.664719565469887, + "tokens_seen": 1038155776 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034617853560682044, + "loss": 1.3037, + "theoretical_loss": 3.6646978719967844, + "tokens_seen": 1038221312 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003461685055165497, + "loss": 1.3553, + "theoretical_loss": 3.664676180276397, + "tokens_seen": 1038286848 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003461584754262788, + "loss": 1.3571, + "theoretical_loss": 3.6646544903084735, + "tokens_seen": 1038352384 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034614844533600804, + "loss": 1.4131, + "theoretical_loss": 3.664632802092761, + "tokens_seen": 1038417920 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034613841524573717, + "loss": 1.2329, + "theoretical_loss": 3.6646111156290084, + "tokens_seen": 1038483456 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003461283851554664, + "loss": 1.2802, + "theoretical_loss": 3.664589430916963, + "tokens_seen": 1038548992 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003461183550651956, + "loss": 1.3461, + "theoretical_loss": 3.6645677479563723, + "tokens_seen": 1038614528 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034610832497492476, + "loss": 1.3706, + "theoretical_loss": 3.6645460667469854, + "tokens_seen": 1038680064 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 348747, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2420824766159058, + "objective/train/theoretical_loss": 3.66452438728855, + "objective/train/tokens_used": 1038745600, + "theoretical_loss": 3.66452438728855, + "tokens_seen": 1038745600 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034609829488465395, + "loss": 1.2909, + "theoretical_loss": 3.66452438728855, + "tokens_seen": 1038745600 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003460882647943832, + "loss": 1.2941, + "theoretical_loss": 3.6645027095808134, + "tokens_seen": 1038811136 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034607823470411236, + "loss": 1.3289, + "theoretical_loss": 3.6644810336235243, + "tokens_seen": 1038876672 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034606820461384154, + "loss": 1.3375, + "theoretical_loss": 3.6644593594164316, + "tokens_seen": 1038942208 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003460581745235707, + "loss": 1.3852, + "theoretical_loss": 3.6644376869592827, + "tokens_seen": 1039007744 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003460481444332999, + "loss": 1.4296, + "theoretical_loss": 3.6644160162518267, + "tokens_seen": 1039073280 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034603811434302914, + "loss": 1.3586, + "theoretical_loss": 3.664394347293811, + "tokens_seen": 1039138816 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034602808425275827, + "loss": 1.4004, + "theoretical_loss": 3.664372680084986, + "tokens_seen": 1039204352 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003460180541624875, + "loss": 1.3359, + "theoretical_loss": 3.6643510146250975, + "tokens_seen": 1039269888 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034600802407221663, + "loss": 1.2918, + "theoretical_loss": 3.664329350913896, + "tokens_seen": 1039335424 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034599799398194587, + "loss": 1.406, + "theoretical_loss": 3.6643076889511295, + "tokens_seen": 1039400960 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034598796389167505, + "loss": 1.2877, + "theoretical_loss": 3.6642860287365466, + "tokens_seen": 1039466496 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034597793380140423, + "loss": 1.2617, + "theoretical_loss": 3.6642643702698963, + "tokens_seen": 1039532032 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003459679037111334, + "loss": 1.3092, + "theoretical_loss": 3.6642427135509275, + "tokens_seen": 1039597568 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034595787362086264, + "loss": 1.242, + "theoretical_loss": 3.664221058579388, + "tokens_seen": 1039663104 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034594784353059177, + "loss": 1.3148, + "theoretical_loss": 3.6641994053550277, + "tokens_seen": 1039728640 + }, + { + "epoch": 0.32, + "learning_rate": 0.000345937813440321, + "loss": 1.3033, + "theoretical_loss": 3.664177753877595, + "tokens_seen": 1039794176 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034592778335005013, + "loss": 1.3358, + "theoretical_loss": 3.664156104146839, + "tokens_seen": 1039859712 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034591775325977937, + "loss": 1.3187, + "theoretical_loss": 3.664134456162509, + "tokens_seen": 1039925248 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034590772316950855, + "loss": 1.3769, + "theoretical_loss": 3.6641128099243536, + "tokens_seen": 1039990784 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034589769307923773, + "loss": 1.3543, + "theoretical_loss": 3.664091165432122, + "tokens_seen": 1040056320 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003458876629889669, + "loss": 1.3237, + "theoretical_loss": 3.6640695226855646, + "tokens_seen": 1040121856 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003458776328986961, + "loss": 1.4112, + "theoretical_loss": 3.6640478816844286, + "tokens_seen": 1040187392 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003458676028084253, + "loss": 1.3448, + "theoretical_loss": 3.6640262424284646, + "tokens_seen": 1040252928 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003458575727181545, + "loss": 1.2807, + "theoretical_loss": 3.6640046049174213, + "tokens_seen": 1040318464 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034584754262788364, + "loss": 1.3452, + "theoretical_loss": 3.663982969151049, + "tokens_seen": 1040384000 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034583751253761287, + "loss": 1.1921, + "theoretical_loss": 3.6639613351290956, + "tokens_seen": 1040449536 + }, + { + "epoch": 0.32, + "learning_rate": 0.000345827482447342, + "loss": 1.3291, + "theoretical_loss": 3.6639397028513123, + "tokens_seen": 1040515072 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034581745235707123, + "loss": 1.3337, + "theoretical_loss": 3.663918072317448, + "tokens_seen": 1040580608 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003458074222668004, + "loss": 1.3266, + "theoretical_loss": 3.663896443527251, + "tokens_seen": 1040646144 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003457973921765296, + "loss": 1.3307, + "theoretical_loss": 3.663874816480473, + "tokens_seen": 1040711680 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003457873620862588, + "loss": 1.4548, + "theoretical_loss": 3.6638531911768624, + "tokens_seen": 1040777216 + }, + { + "epoch": 0.32, + "learning_rate": 0.000345777331995988, + "loss": 1.3343, + "theoretical_loss": 3.6638315676161697, + "tokens_seen": 1040842752 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034576730190571714, + "loss": 1.3241, + "theoretical_loss": 3.6638099457981443, + "tokens_seen": 1040908288 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003457572718154464, + "loss": 1.2635, + "theoretical_loss": 3.6637883257225354, + "tokens_seen": 1040973824 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003457472417251755, + "loss": 1.2467, + "theoretical_loss": 3.6637667073890943, + "tokens_seen": 1041039360 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034573721163490474, + "loss": 1.3017, + "theoretical_loss": 3.66374509079757, + "tokens_seen": 1041104896 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003457271815446339, + "loss": 1.3655, + "theoretical_loss": 3.6637234759477124, + "tokens_seen": 1041170432 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003457171514543631, + "loss": 1.2587, + "theoretical_loss": 3.6637018628392726, + "tokens_seen": 1041235968 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003457071213640923, + "loss": 1.2173, + "theoretical_loss": 3.663680251471999, + "tokens_seen": 1041301504 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034569709127382146, + "loss": 1.407, + "theoretical_loss": 3.6636586418456436, + "tokens_seen": 1041367040 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034568706118355064, + "loss": 1.4233, + "theoretical_loss": 3.6636370339599553, + "tokens_seen": 1041432576 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003456770310932799, + "loss": 1.3452, + "theoretical_loss": 3.663615427814685, + "tokens_seen": 1041498112 + }, + { + "epoch": 0.32, + "learning_rate": 0.000345667001003009, + "loss": 1.3674, + "theoretical_loss": 3.663593823409583, + "tokens_seen": 1041563648 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034565697091273824, + "loss": 1.2953, + "theoretical_loss": 3.6635722207444, + "tokens_seen": 1041629184 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034564694082246737, + "loss": 1.268, + "theoretical_loss": 3.663550619818885, + "tokens_seen": 1041694720 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003456369107321966, + "loss": 1.3126, + "theoretical_loss": 3.6635290206327897, + "tokens_seen": 1041760256 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003456268806419258, + "loss": 1.2873, + "theoretical_loss": 3.663507423185864, + "tokens_seen": 1041825792 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034561685055165496, + "loss": 1.2647, + "theoretical_loss": 3.6634858274778597, + "tokens_seen": 1041891328 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034560682046138415, + "loss": 1.3966, + "theoretical_loss": 3.6634642335085257, + "tokens_seen": 1041956864 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 349934, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3734266757965088, + "objective/train/theoretical_loss": 3.663442641277614, + "objective/train/tokens_used": 1042022400, + "theoretical_loss": 3.663442641277614, + "tokens_seen": 1042022400 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003455967903711134, + "loss": 1.4227, + "theoretical_loss": 3.663442641277614, + "tokens_seen": 1042022400 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003455867602808425, + "loss": 1.3368, + "theoretical_loss": 3.6634210507848746, + "tokens_seen": 1042087936 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034557673019057174, + "loss": 1.2654, + "theoretical_loss": 3.6633994620300587, + "tokens_seen": 1042153472 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034556670010030087, + "loss": 1.2175, + "theoretical_loss": 3.663377875012917, + "tokens_seen": 1042219008 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003455566700100301, + "loss": 1.3153, + "theoretical_loss": 3.6633562897331995, + "tokens_seen": 1042284544 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003455466399197593, + "loss": 1.3679, + "theoretical_loss": 3.6633347061906587, + "tokens_seen": 1042350080 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034553660982948847, + "loss": 1.2318, + "theoretical_loss": 3.663313124385045, + "tokens_seen": 1042415616 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034552657973921765, + "loss": 1.2599, + "theoretical_loss": 3.6632915443161087, + "tokens_seen": 1042481152 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034551654964894683, + "loss": 1.3043, + "theoretical_loss": 3.663269965983601, + "tokens_seen": 1042546688 + }, + { + "epoch": 0.32, + "learning_rate": 0.000345506519558676, + "loss": 1.2796, + "theoretical_loss": 3.6632483893872747, + "tokens_seen": 1042612224 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034549648946840525, + "loss": 1.4156, + "theoretical_loss": 3.6632268145268796, + "tokens_seen": 1042677760 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003454864593781344, + "loss": 1.3158, + "theoretical_loss": 3.6632052414021663, + "tokens_seen": 1042743296 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003454764292878636, + "loss": 1.2522, + "theoretical_loss": 3.6631836700128875, + "tokens_seen": 1042808832 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034546639919759274, + "loss": 1.2824, + "theoretical_loss": 3.663162100358794, + "tokens_seen": 1042874368 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034545636910732197, + "loss": 1.3149, + "theoretical_loss": 3.6631405324396367, + "tokens_seen": 1042939904 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034544633901705115, + "loss": 1.3866, + "theoretical_loss": 3.663118966255168, + "tokens_seen": 1043005440 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034543630892678033, + "loss": 1.32, + "theoretical_loss": 3.6630974018051385, + "tokens_seen": 1043070976 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003454262788365095, + "loss": 1.244, + "theoretical_loss": 3.6630758390893003, + "tokens_seen": 1043136512 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034541624874623875, + "loss": 1.3391, + "theoretical_loss": 3.663054278107405, + "tokens_seen": 1043202048 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003454062186559679, + "loss": 1.3251, + "theoretical_loss": 3.6630327188592036, + "tokens_seen": 1043267584 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003453961885656971, + "loss": 1.303, + "theoretical_loss": 3.663011161344449, + "tokens_seen": 1043333120 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034538615847542624, + "loss": 1.2634, + "theoretical_loss": 3.662989605562892, + "tokens_seen": 1043398656 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003453761283851555, + "loss": 1.3303, + "theoretical_loss": 3.6629680515142846, + "tokens_seen": 1043464192 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034536609829488466, + "loss": 1.417, + "theoretical_loss": 3.6629464991983784, + "tokens_seen": 1043529728 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034535606820461384, + "loss": 1.2782, + "theoretical_loss": 3.6629249486149256, + "tokens_seen": 1043595264 + }, + { + "epoch": 0.32, + "learning_rate": 0.000345346038114343, + "loss": 1.2901, + "theoretical_loss": 3.662903399763678, + "tokens_seen": 1043660800 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003453360080240722, + "loss": 1.3068, + "theoretical_loss": 3.6628818526443885, + "tokens_seen": 1043726336 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034532597793380143, + "loss": 1.3305, + "theoretical_loss": 3.6628603072568082, + "tokens_seen": 1043791872 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003453159478435306, + "loss": 1.339, + "theoretical_loss": 3.662838763600689, + "tokens_seen": 1043857408 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003453059177532598, + "loss": 1.2525, + "theoretical_loss": 3.6628172216757844, + "tokens_seen": 1043922944 + }, + { + "epoch": 0.32, + "learning_rate": 0.000345295887662989, + "loss": 1.3385, + "theoretical_loss": 3.662795681481845, + "tokens_seen": 1043988480 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003452858575727182, + "loss": 1.2585, + "theoretical_loss": 3.662774143018624, + "tokens_seen": 1044054016 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034527582748244734, + "loss": 1.3122, + "theoretical_loss": 3.6627526062858733, + "tokens_seen": 1044119552 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003452657973921766, + "loss": 1.3202, + "theoretical_loss": 3.6627310712833454, + "tokens_seen": 1044185088 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003452557673019057, + "loss": 1.4449, + "theoretical_loss": 3.6627095380107932, + "tokens_seen": 1044250624 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034524573721163494, + "loss": 1.2906, + "theoretical_loss": 3.6626880064679685, + "tokens_seen": 1044316160 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003452357071213641, + "loss": 1.3033, + "theoretical_loss": 3.662666476654624, + "tokens_seen": 1044381696 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003452256770310933, + "loss": 1.4721, + "theoretical_loss": 3.6626449485705126, + "tokens_seen": 1044447232 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003452156469408225, + "loss": 1.2978, + "theoretical_loss": 3.662623422215386, + "tokens_seen": 1044512768 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034520561685055166, + "loss": 1.3446, + "theoretical_loss": 3.6626018975889982, + "tokens_seen": 1044578304 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034519558676028084, + "loss": 1.2452, + "theoretical_loss": 3.6625803746911014, + "tokens_seen": 1044643840 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003451855566700101, + "loss": 1.3336, + "theoretical_loss": 3.6625588535214475, + "tokens_seen": 1044709376 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003451755265797392, + "loss": 1.2369, + "theoretical_loss": 3.6625373340797904, + "tokens_seen": 1044774912 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034516549648946844, + "loss": 1.2911, + "theoretical_loss": 3.6625158163658824, + "tokens_seen": 1044840448 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034515546639919757, + "loss": 1.2364, + "theoretical_loss": 3.6624943003794765, + "tokens_seen": 1044905984 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003451454363089268, + "loss": 1.2723, + "theoretical_loss": 3.6624727861203263, + "tokens_seen": 1044971520 + }, + { + "epoch": 0.32, + "learning_rate": 0.000345135406218656, + "loss": 1.3556, + "theoretical_loss": 3.6624512735881845, + "tokens_seen": 1045037056 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034512537612838517, + "loss": 1.3581, + "theoretical_loss": 3.6624297627828035, + "tokens_seen": 1045102592 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034511534603811435, + "loss": 1.3953, + "theoretical_loss": 3.662408253703937, + "tokens_seen": 1045168128 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003451053159478436, + "loss": 1.3259, + "theoretical_loss": 3.6623867463513387, + "tokens_seen": 1045233664 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 351167, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3379461765289307, + "objective/train/theoretical_loss": 3.662365240724761, + "objective/train/tokens_used": 1045299200, + "theoretical_loss": 3.662365240724761, + "tokens_seen": 1045299200 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003450952858575727, + "loss": 1.3873, + "theoretical_loss": 3.662365240724761, + "tokens_seen": 1045299200 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034508525576730194, + "loss": 1.2599, + "theoretical_loss": 3.6623437368239573, + "tokens_seen": 1045364736 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034507522567703107, + "loss": 1.2974, + "theoretical_loss": 3.6623222346486806, + "tokens_seen": 1045430272 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003450651955867603, + "loss": 1.2722, + "theoretical_loss": 3.662300734198685, + "tokens_seen": 1045495808 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003450551654964895, + "loss": 1.3871, + "theoretical_loss": 3.6622792354737244, + "tokens_seen": 1045561344 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034504513540621867, + "loss": 1.3703, + "theoretical_loss": 3.662257738473551, + "tokens_seen": 1045626880 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034503510531594785, + "loss": 1.2331, + "theoretical_loss": 3.6622362431979187, + "tokens_seen": 1045692416 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034502507522567703, + "loss": 1.3196, + "theoretical_loss": 3.662214749646582, + "tokens_seen": 1045757952 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003450150451354062, + "loss": 1.3491, + "theoretical_loss": 3.6621932578192933, + "tokens_seen": 1045823488 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034500501504513545, + "loss": 1.4179, + "theoretical_loss": 3.6621717677158068, + "tokens_seen": 1045889024 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003449949849548646, + "loss": 1.3437, + "theoretical_loss": 3.6621502793358762, + "tokens_seen": 1045954560 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003449849548645938, + "loss": 1.2807, + "theoretical_loss": 3.6621287926792556, + "tokens_seen": 1046020096 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034497492477432294, + "loss": 1.3864, + "theoretical_loss": 3.6621073077456985, + "tokens_seen": 1046085632 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034496489468405217, + "loss": 1.2774, + "theoretical_loss": 3.662085824534959, + "tokens_seen": 1046151168 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034495486459378135, + "loss": 1.5115, + "theoretical_loss": 3.6620643430467905, + "tokens_seen": 1046216704 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034494483450351053, + "loss": 1.3826, + "theoretical_loss": 3.6620428632809476, + "tokens_seen": 1046282240 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003449348044132397, + "loss": 1.2984, + "theoretical_loss": 3.6620213852371846, + "tokens_seen": 1046347776 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034492477432296895, + "loss": 1.4008, + "theoretical_loss": 3.661999908915254, + "tokens_seen": 1046413312 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003449147442326981, + "loss": 1.2519, + "theoretical_loss": 3.6619784343149124, + "tokens_seen": 1046478848 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003449047141424273, + "loss": 1.3379, + "theoretical_loss": 3.6619569614359118, + "tokens_seen": 1046544384 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034489468405215644, + "loss": 1.2811, + "theoretical_loss": 3.6619354902780072, + "tokens_seen": 1046609920 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003448846539618857, + "loss": 1.4403, + "theoretical_loss": 3.6619140208409533, + "tokens_seen": 1046675456 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034487462387161486, + "loss": 1.3786, + "theoretical_loss": 3.6618925531245043, + "tokens_seen": 1046740992 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034486459378134404, + "loss": 1.3005, + "theoretical_loss": 3.6618710871284144, + "tokens_seen": 1046806528 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003448545636910732, + "loss": 1.3666, + "theoretical_loss": 3.661849622852438, + "tokens_seen": 1046872064 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003448445336008024, + "loss": 1.4006, + "theoretical_loss": 3.6618281602963294, + "tokens_seen": 1046937600 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003448345035105316, + "loss": 1.3644, + "theoretical_loss": 3.6618066994598437, + "tokens_seen": 1047003136 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003448244734202608, + "loss": 1.2649, + "theoretical_loss": 3.661785240342735, + "tokens_seen": 1047068672 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034481444332998994, + "loss": 1.2997, + "theoretical_loss": 3.6617637829447585, + "tokens_seen": 1047134208 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003448044132397192, + "loss": 1.3856, + "theoretical_loss": 3.661742327265668, + "tokens_seen": 1047199744 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034479438314944836, + "loss": 1.3857, + "theoretical_loss": 3.6617208733052196, + "tokens_seen": 1047265280 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034478435305917754, + "loss": 1.2923, + "theoretical_loss": 3.6616994210631666, + "tokens_seen": 1047330816 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003447743229689067, + "loss": 1.3416, + "theoretical_loss": 3.661677970539265, + "tokens_seen": 1047396352 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003447642928786359, + "loss": 1.2637, + "theoretical_loss": 3.661656521733269, + "tokens_seen": 1047461888 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003447542627883651, + "loss": 1.2588, + "theoretical_loss": 3.6616350746449338, + "tokens_seen": 1047527424 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003447442326980943, + "loss": 1.2817, + "theoretical_loss": 3.6616136292740142, + "tokens_seen": 1047592960 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034473420260782345, + "loss": 1.4861, + "theoretical_loss": 3.6615921856202656, + "tokens_seen": 1047658496 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003447241725175527, + "loss": 1.2812, + "theoretical_loss": 3.661570743683443, + "tokens_seen": 1047724032 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003447141424272818, + "loss": 1.2839, + "theoretical_loss": 3.6615493034633015, + "tokens_seen": 1047789568 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034470411233701104, + "loss": 1.4278, + "theoretical_loss": 3.6615278649595964, + "tokens_seen": 1047855104 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003446940822467402, + "loss": 1.4043, + "theoretical_loss": 3.661506428172083, + "tokens_seen": 1047920640 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003446840521564694, + "loss": 1.3737, + "theoretical_loss": 3.661484993100516, + "tokens_seen": 1047986176 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003446740220661986, + "loss": 1.3811, + "theoretical_loss": 3.661463559744651, + "tokens_seen": 1048051712 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034466399197592777, + "loss": 1.3101, + "theoretical_loss": 3.6614421281042437, + "tokens_seen": 1048117248 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034465396188565695, + "loss": 1.4855, + "theoretical_loss": 3.66142069817905, + "tokens_seen": 1048182784 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003446439317953862, + "loss": 1.3918, + "theoretical_loss": 3.6613992699688245, + "tokens_seen": 1048248320 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003446339017051153, + "loss": 1.4818, + "theoretical_loss": 3.6613778434733226, + "tokens_seen": 1048313856 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034462387161484455, + "loss": 1.355, + "theoretical_loss": 3.661356418692301, + "tokens_seen": 1048379392 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034461384152457373, + "loss": 1.361, + "theoretical_loss": 3.6613349956255146, + "tokens_seen": 1048444928 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003446038114343029, + "loss": 1.4628, + "theoretical_loss": 3.661313574272719, + "tokens_seen": 1048510464 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 351997, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4105134010314941, + "objective/train/theoretical_loss": 3.6612921546336703, + "objective/train/tokens_used": 1048576000, + "theoretical_loss": 3.6612921546336703, + "tokens_seen": 1048576000 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034459378134403214, + "loss": 1.3962, + "theoretical_loss": 3.6612921546336703, + "tokens_seen": 1048576000 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034458375125376127, + "loss": 1.3524, + "theoretical_loss": 3.6612707367081243, + "tokens_seen": 1048641536 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003445737211634905, + "loss": 1.2406, + "theoretical_loss": 3.6612493204958367, + "tokens_seen": 1048707072 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003445636910732197, + "loss": 1.4162, + "theoretical_loss": 3.6612279059965633, + "tokens_seen": 1048772608 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034455366098294887, + "loss": 1.3462, + "theoretical_loss": 3.6612064932100603, + "tokens_seen": 1048838144 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034454363089267805, + "loss": 1.359, + "theoretical_loss": 3.6611850821360834, + "tokens_seen": 1048903680 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034453360080240723, + "loss": 1.2507, + "theoretical_loss": 3.6611636727743893, + "tokens_seen": 1048969216 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003445235707121364, + "loss": 1.3269, + "theoretical_loss": 3.661142265124733, + "tokens_seen": 1049034752 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034451354062186565, + "loss": 1.4244, + "theoretical_loss": 3.661120859186872, + "tokens_seen": 1049100288 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003445035105315948, + "loss": 1.4167, + "theoretical_loss": 3.6610994549605613, + "tokens_seen": 1049165824 + }, + { + "epoch": 0.32, + "learning_rate": 0.000344493480441324, + "loss": 1.3091, + "theoretical_loss": 3.661078052445558, + "tokens_seen": 1049231360 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034448345035105314, + "loss": 1.4257, + "theoretical_loss": 3.6610566516416174, + "tokens_seen": 1049296896 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034447342026078237, + "loss": 1.3631, + "theoretical_loss": 3.661035252548497, + "tokens_seen": 1049362432 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034446339017051155, + "loss": 1.2886, + "theoretical_loss": 3.661013855165953, + "tokens_seen": 1049427968 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034445336008024073, + "loss": 1.2856, + "theoretical_loss": 3.660992459493741, + "tokens_seen": 1049493504 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003444433299899699, + "loss": 1.3251, + "theoretical_loss": 3.6609710655316183, + "tokens_seen": 1049559040 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034443329989969915, + "loss": 1.2689, + "theoretical_loss": 3.660949673279341, + "tokens_seen": 1049624576 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003444232698094283, + "loss": 1.2883, + "theoretical_loss": 3.6609282827366663, + "tokens_seen": 1049690112 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003444132397191575, + "loss": 1.3867, + "theoretical_loss": 3.66090689390335, + "tokens_seen": 1049755648 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034440320962888664, + "loss": 1.2246, + "theoretical_loss": 3.660885506779149, + "tokens_seen": 1049821184 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003443931795386159, + "loss": 1.3176, + "theoretical_loss": 3.660864121363821, + "tokens_seen": 1049886720 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034438314944834506, + "loss": 1.3608, + "theoretical_loss": 3.6608427376571218, + "tokens_seen": 1049952256 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034437311935807424, + "loss": 1.3632, + "theoretical_loss": 3.6608213556588085, + "tokens_seen": 1050017792 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003443630892678034, + "loss": 1.2562, + "theoretical_loss": 3.660799975368638, + "tokens_seen": 1050083328 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003443530591775326, + "loss": 1.297, + "theoretical_loss": 3.6607785967863675, + "tokens_seen": 1050148864 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003443430290872618, + "loss": 1.3948, + "theoretical_loss": 3.6607572199117535, + "tokens_seen": 1050214400 + }, + { + "epoch": 0.32, + "learning_rate": 0.000344332998996991, + "loss": 1.361, + "theoretical_loss": 3.6607358447445533, + "tokens_seen": 1050279936 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034432296890672014, + "loss": 1.4048, + "theoretical_loss": 3.6607144712845243, + "tokens_seen": 1050345472 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003443129388164494, + "loss": 1.2934, + "theoretical_loss": 3.6606930995314233, + "tokens_seen": 1050411008 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034430290872617856, + "loss": 1.3553, + "theoretical_loss": 3.6606717294850077, + "tokens_seen": 1050476544 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034429287863590774, + "loss": 1.2021, + "theoretical_loss": 3.6606503611450343, + "tokens_seen": 1050542080 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003442828485456369, + "loss": 1.3335, + "theoretical_loss": 3.6606289945112613, + "tokens_seen": 1050607616 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003442728184553661, + "loss": 1.4305, + "theoretical_loss": 3.660607629583445, + "tokens_seen": 1050673152 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003442627883650953, + "loss": 1.2972, + "theoretical_loss": 3.6605862663613435, + "tokens_seen": 1050738688 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003442527582748245, + "loss": 1.1858, + "theoretical_loss": 3.6605649048447138, + "tokens_seen": 1050804224 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034424272818455365, + "loss": 1.2772, + "theoretical_loss": 3.660543545033314, + "tokens_seen": 1050869760 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003442326980942829, + "loss": 1.3385, + "theoretical_loss": 3.660522186926901, + "tokens_seen": 1050935296 + }, + { + "epoch": 0.32, + "learning_rate": 0.000344222668004012, + "loss": 1.3789, + "theoretical_loss": 3.6605008305252325, + "tokens_seen": 1051000832 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034421263791374124, + "loss": 1.4034, + "theoretical_loss": 3.6604794758280663, + "tokens_seen": 1051066368 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003442026078234704, + "loss": 1.3071, + "theoretical_loss": 3.66045812283516, + "tokens_seen": 1051131904 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003441925777331996, + "loss": 1.3452, + "theoretical_loss": 3.660436771546272, + "tokens_seen": 1051197440 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003441825476429288, + "loss": 1.3172, + "theoretical_loss": 3.660415421961159, + "tokens_seen": 1051262976 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034417251755265797, + "loss": 1.3554, + "theoretical_loss": 3.6603940740795795, + "tokens_seen": 1051328512 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034416248746238715, + "loss": 1.2472, + "theoretical_loss": 3.660372727901291, + "tokens_seen": 1051394048 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003441524573721164, + "loss": 1.3471, + "theoretical_loss": 3.660351383426052, + "tokens_seen": 1051459584 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003441424272818455, + "loss": 1.2805, + "theoretical_loss": 3.6603300406536206, + "tokens_seen": 1051525120 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034413239719157475, + "loss": 1.2724, + "theoretical_loss": 3.660308699583754, + "tokens_seen": 1051590656 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034412236710130393, + "loss": 1.3235, + "theoretical_loss": 3.660287360216211, + "tokens_seen": 1051656192 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003441123370110331, + "loss": 1.3107, + "theoretical_loss": 3.660266022550749, + "tokens_seen": 1051721728 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003441023069207623, + "loss": 1.319, + "theoretical_loss": 3.6602446865871263, + "tokens_seen": 1051787264 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 353175, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2638063430786133, + "objective/train/theoretical_loss": 3.660223352325102, + "objective/train/tokens_used": 1051852800, + "theoretical_loss": 3.660223352325102, + "tokens_seen": 1051852800 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034409227683049147, + "loss": 1.3012, + "theoretical_loss": 3.660223352325102, + "tokens_seen": 1051852800 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034408224674022065, + "loss": 1.3796, + "theoretical_loss": 3.660202019764434, + "tokens_seen": 1051918336 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003440722166499499, + "loss": 1.3282, + "theoretical_loss": 3.6601806889048802, + "tokens_seen": 1051983872 + }, + { + "epoch": 0.32, + "learning_rate": 0.000344062186559679, + "loss": 1.2921, + "theoretical_loss": 3.6601593597461997, + "tokens_seen": 1052049408 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034405215646940825, + "loss": 1.3406, + "theoretical_loss": 3.6601380322881503, + "tokens_seen": 1052114944 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003440421263791374, + "loss": 1.3638, + "theoretical_loss": 3.6601167065304905, + "tokens_seen": 1052180480 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003440320962888666, + "loss": 1.2721, + "theoretical_loss": 3.6600953824729796, + "tokens_seen": 1052246016 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003440220661985958, + "loss": 1.3035, + "theoretical_loss": 3.660074060115375, + "tokens_seen": 1052311552 + }, + { + "epoch": 0.32, + "learning_rate": 0.000344012036108325, + "loss": 1.3134, + "theoretical_loss": 3.660052739457436, + "tokens_seen": 1052377088 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034400200601805416, + "loss": 1.3485, + "theoretical_loss": 3.660031420498922, + "tokens_seen": 1052442624 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034399197592778334, + "loss": 1.4051, + "theoretical_loss": 3.66001010323959, + "tokens_seen": 1052508160 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003439819458375125, + "loss": 1.3939, + "theoretical_loss": 3.6599887876792003, + "tokens_seen": 1052573696 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034397191574724175, + "loss": 1.32, + "theoretical_loss": 3.6599674738175114, + "tokens_seen": 1052639232 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003439618856569709, + "loss": 1.1922, + "theoretical_loss": 3.6599461616542817, + "tokens_seen": 1052704768 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003439518555667001, + "loss": 1.3406, + "theoretical_loss": 3.6599248511892704, + "tokens_seen": 1052770304 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003439418254764293, + "loss": 1.2236, + "theoretical_loss": 3.659903542422237, + "tokens_seen": 1052835840 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003439317953861585, + "loss": 1.277, + "theoretical_loss": 3.6598822353529386, + "tokens_seen": 1052901376 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034392176529588766, + "loss": 1.2094, + "theoretical_loss": 3.6598609299811367, + "tokens_seen": 1052966912 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034391173520561684, + "loss": 1.2999, + "theoretical_loss": 3.6598396263065895, + "tokens_seen": 1053032448 + }, + { + "epoch": 0.32, + "learning_rate": 0.000343901705115346, + "loss": 1.3413, + "theoretical_loss": 3.6598183243290556, + "tokens_seen": 1053097984 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034389167502507526, + "loss": 1.3705, + "theoretical_loss": 3.659797024048295, + "tokens_seen": 1053163520 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003438816449348044, + "loss": 1.267, + "theoretical_loss": 3.6597757254640664, + "tokens_seen": 1053229056 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003438716148445336, + "loss": 1.2517, + "theoretical_loss": 3.6597544285761296, + "tokens_seen": 1053294592 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034386158475426275, + "loss": 1.2988, + "theoretical_loss": 3.6597331333842438, + "tokens_seen": 1053360128 + }, + { + "epoch": 0.32, + "learning_rate": 0.000343851554663992, + "loss": 1.3617, + "theoretical_loss": 3.6597118398881685, + "tokens_seen": 1053425664 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003438415245737212, + "loss": 1.4064, + "theoretical_loss": 3.6596905480876627, + "tokens_seen": 1053491200 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034383149448345034, + "loss": 1.3627, + "theoretical_loss": 3.6596692579824865, + "tokens_seen": 1053556736 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003438214643931796, + "loss": 1.332, + "theoretical_loss": 3.659647969572399, + "tokens_seen": 1053622272 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034381143430290876, + "loss": 1.3073, + "theoretical_loss": 3.6596266828571604, + "tokens_seen": 1053687808 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034380140421263794, + "loss": 1.2556, + "theoretical_loss": 3.65960539783653, + "tokens_seen": 1053753344 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003437913741223671, + "loss": 1.245, + "theoretical_loss": 3.659584114510267, + "tokens_seen": 1053818880 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003437813440320963, + "loss": 1.3837, + "theoretical_loss": 3.6595628328781324, + "tokens_seen": 1053884416 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003437713139418255, + "loss": 1.2924, + "theoretical_loss": 3.659541552939885, + "tokens_seen": 1053949952 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003437612838515547, + "loss": 1.3664, + "theoretical_loss": 3.659520274695285, + "tokens_seen": 1054015488 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034375125376128385, + "loss": 1.2337, + "theoretical_loss": 3.659498998144092, + "tokens_seen": 1054081024 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003437412236710131, + "loss": 1.2773, + "theoretical_loss": 3.6594777232860665, + "tokens_seen": 1054146560 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003437311935807422, + "loss": 1.3999, + "theoretical_loss": 3.6594564501209685, + "tokens_seen": 1054212096 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034372116349047144, + "loss": 1.3132, + "theoretical_loss": 3.6594351786485575, + "tokens_seen": 1054277632 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003437111334002006, + "loss": 1.3102, + "theoretical_loss": 3.659413908868594, + "tokens_seen": 1054343168 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003437011033099298, + "loss": 1.3116, + "theoretical_loss": 3.6593926407808377, + "tokens_seen": 1054408704 + }, + { + "epoch": 0.32, + "learning_rate": 0.000343691073219659, + "loss": 1.3317, + "theoretical_loss": 3.65937137438505, + "tokens_seen": 1054474240 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034368104312938817, + "loss": 1.3202, + "theoretical_loss": 3.6593501096809895, + "tokens_seen": 1054539776 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034367101303911735, + "loss": 1.2001, + "theoretical_loss": 3.6593288466684175, + "tokens_seen": 1054605312 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003436609829488466, + "loss": 1.3599, + "theoretical_loss": 3.6593075853470944, + "tokens_seen": 1054670848 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003436509528585757, + "loss": 1.3526, + "theoretical_loss": 3.6592863257167805, + "tokens_seen": 1054736384 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034364092276830495, + "loss": 1.3666, + "theoretical_loss": 3.659265067777236, + "tokens_seen": 1054801920 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034363089267803413, + "loss": 1.3154, + "theoretical_loss": 3.6592438115282215, + "tokens_seen": 1054867456 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003436208625877633, + "loss": 1.1718, + "theoretical_loss": 3.6592225569694974, + "tokens_seen": 1054932992 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003436108324974925, + "loss": 1.2685, + "theoretical_loss": 3.659201304100825, + "tokens_seen": 1054998528 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034360080240722167, + "loss": 1.3313, + "theoretical_loss": 3.659180052921964, + "tokens_seen": 1055064064 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 354036, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1976933479309082, + "objective/train/theoretical_loss": 3.659158803432675, + "objective/train/tokens_used": 1055129600, + "theoretical_loss": 3.659158803432675, + "tokens_seen": 1055129600 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034359077231695085, + "loss": 1.28, + "theoretical_loss": 3.659158803432675, + "tokens_seen": 1055129600 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003435807422266801, + "loss": 1.3208, + "theoretical_loss": 3.6591375556327197, + "tokens_seen": 1055195136 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003435707121364092, + "loss": 1.2647, + "theoretical_loss": 3.659116309521859, + "tokens_seen": 1055260672 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034356068204613845, + "loss": 1.3108, + "theoretical_loss": 3.6590950650998524, + "tokens_seen": 1055326208 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003435506519558676, + "loss": 1.4775, + "theoretical_loss": 3.6590738223664614, + "tokens_seen": 1055391744 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003435406218655968, + "loss": 1.2974, + "theoretical_loss": 3.659052581321448, + "tokens_seen": 1055457280 + }, + { + "epoch": 0.32, + "learning_rate": 0.000343530591775326, + "loss": 1.2096, + "theoretical_loss": 3.6590313419645715, + "tokens_seen": 1055522816 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003435205616850552, + "loss": 1.3771, + "theoretical_loss": 3.659010104295594, + "tokens_seen": 1055588352 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034351053159478436, + "loss": 1.387, + "theoretical_loss": 3.6589888683142764, + "tokens_seen": 1055653888 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034350050150451354, + "loss": 1.4098, + "theoretical_loss": 3.6589676340203794, + "tokens_seen": 1055719424 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003434904714142427, + "loss": 1.2302, + "theoretical_loss": 3.6589464014136643, + "tokens_seen": 1055784960 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034348044132397195, + "loss": 1.3022, + "theoretical_loss": 3.6589251704938928, + "tokens_seen": 1055850496 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003434704112337011, + "loss": 1.4409, + "theoretical_loss": 3.658903941260826, + "tokens_seen": 1055916032 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003434603811434303, + "loss": 1.3651, + "theoretical_loss": 3.6588827137142252, + "tokens_seen": 1055981568 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003434503510531595, + "loss": 1.3684, + "theoretical_loss": 3.6588614878538515, + "tokens_seen": 1056047104 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003434403209628887, + "loss": 1.2231, + "theoretical_loss": 3.658840263679467, + "tokens_seen": 1056112640 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034343029087261786, + "loss": 1.2537, + "theoretical_loss": 3.6588190411908323, + "tokens_seen": 1056178176 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034342026078234704, + "loss": 1.3248, + "theoretical_loss": 3.658797820387709, + "tokens_seen": 1056243712 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003434102306920762, + "loss": 1.3939, + "theoretical_loss": 3.6587766012698593, + "tokens_seen": 1056309248 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034340020060180546, + "loss": 1.4248, + "theoretical_loss": 3.658755383837045, + "tokens_seen": 1056374784 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003433901705115346, + "loss": 1.2369, + "theoretical_loss": 3.658734168089026, + "tokens_seen": 1056440320 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003433801404212638, + "loss": 1.3454, + "theoretical_loss": 3.6587129540255665, + "tokens_seen": 1056505856 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034337011033099295, + "loss": 1.3047, + "theoretical_loss": 3.6586917416464266, + "tokens_seen": 1056571392 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003433600802407222, + "loss": 1.4054, + "theoretical_loss": 3.658670530951368, + "tokens_seen": 1056636928 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034335005015045136, + "loss": 1.3942, + "theoretical_loss": 3.658649321940154, + "tokens_seen": 1056702464 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034334002006018054, + "loss": 1.2916, + "theoretical_loss": 3.6586281146125454, + "tokens_seen": 1056768000 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003433299899699097, + "loss": 1.3997, + "theoretical_loss": 3.658606908968304, + "tokens_seen": 1056833536 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003433199598796389, + "loss": 1.4176, + "theoretical_loss": 3.6585857050071926, + "tokens_seen": 1056899072 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003433099297893681, + "loss": 1.2494, + "theoretical_loss": 3.6585645027289724, + "tokens_seen": 1056964608 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003432998996990973, + "loss": 1.2689, + "theoretical_loss": 3.658543302133406, + "tokens_seen": 1057030144 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034328986960882645, + "loss": 1.3284, + "theoretical_loss": 3.658522103220256, + "tokens_seen": 1057095680 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003432798395185557, + "loss": 1.3275, + "theoretical_loss": 3.6585009059892832, + "tokens_seen": 1057161216 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034326980942828487, + "loss": 1.2816, + "theoretical_loss": 3.6584797104402513, + "tokens_seen": 1057226752 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034325977933801405, + "loss": 1.2063, + "theoretical_loss": 3.6584585165729218, + "tokens_seen": 1057292288 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034324974924774323, + "loss": 1.2881, + "theoretical_loss": 3.6584373243870574, + "tokens_seen": 1057357824 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003432397191574724, + "loss": 1.387, + "theoretical_loss": 3.6584161338824206, + "tokens_seen": 1057423360 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003432296890672016, + "loss": 1.3473, + "theoretical_loss": 3.658394945058773, + "tokens_seen": 1057488896 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003432196589769308, + "loss": 1.3747, + "theoretical_loss": 3.6583737579158777, + "tokens_seen": 1057554432 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034320962888665995, + "loss": 1.291, + "theoretical_loss": 3.6583525724534978, + "tokens_seen": 1057619968 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003431995987963892, + "loss": 1.3812, + "theoretical_loss": 3.6583313886713946, + "tokens_seen": 1057685504 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003431895687061183, + "loss": 1.2489, + "theoretical_loss": 3.6583102065693316, + "tokens_seen": 1057751040 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034317953861584755, + "loss": 1.3536, + "theoretical_loss": 3.6582890261470715, + "tokens_seen": 1057816576 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034316950852557673, + "loss": 1.2477, + "theoretical_loss": 3.658267847404377, + "tokens_seen": 1057882112 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003431594784353059, + "loss": 1.2516, + "theoretical_loss": 3.6582466703410104, + "tokens_seen": 1057947648 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003431494483450351, + "loss": 1.3423, + "theoretical_loss": 3.658225494956735, + "tokens_seen": 1058013184 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034313941825476433, + "loss": 1.2407, + "theoretical_loss": 3.658204321251313, + "tokens_seen": 1058078720 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034312938816449345, + "loss": 1.3215, + "theoretical_loss": 3.6581831492245085, + "tokens_seen": 1058144256 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003431193580742227, + "loss": 1.3941, + "theoretical_loss": 3.658161978876083, + "tokens_seen": 1058209792 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003431093279839518, + "loss": 1.2308, + "theoretical_loss": 3.6581408102058015, + "tokens_seen": 1058275328 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034309929789368105, + "loss": 1.3419, + "theoretical_loss": 3.658119643213425, + "tokens_seen": 1058340864 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 355252, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3750050067901611, + "objective/train/theoretical_loss": 3.658098477898718, + "objective/train/tokens_used": 1058406400, + "theoretical_loss": 3.658098477898718, + "tokens_seen": 1058406400 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003430892678034103, + "loss": 1.339, + "theoretical_loss": 3.658098477898718, + "tokens_seen": 1058406400 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003430792377131394, + "loss": 1.3072, + "theoretical_loss": 3.6580773142614422, + "tokens_seen": 1058471936 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034306920762286865, + "loss": 1.3558, + "theoretical_loss": 3.658056152301363, + "tokens_seen": 1058537472 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003430591775325978, + "loss": 1.3011, + "theoretical_loss": 3.6580349920182416, + "tokens_seen": 1058603008 + }, + { + "epoch": 0.32, + "learning_rate": 0.000343049147442327, + "loss": 1.3515, + "theoretical_loss": 3.658013833411842, + "tokens_seen": 1058668544 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003430391173520562, + "loss": 1.2577, + "theoretical_loss": 3.6579926764819284, + "tokens_seen": 1058734080 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003430290872617854, + "loss": 1.3491, + "theoretical_loss": 3.657971521228263, + "tokens_seen": 1058799616 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034301905717151456, + "loss": 1.4626, + "theoretical_loss": 3.65795036765061, + "tokens_seen": 1058865152 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034300902708124374, + "loss": 1.327, + "theoretical_loss": 3.6579292157487324, + "tokens_seen": 1058930688 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003429989969909729, + "loss": 1.3071, + "theoretical_loss": 3.6579080655223937, + "tokens_seen": 1058996224 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034298896690070215, + "loss": 1.2546, + "theoretical_loss": 3.657886916971359, + "tokens_seen": 1059061760 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003429789368104313, + "loss": 1.2684, + "theoretical_loss": 3.6578657700953894, + "tokens_seen": 1059127296 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003429689067201605, + "loss": 1.4091, + "theoretical_loss": 3.6578446248942504, + "tokens_seen": 1059192832 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003429588766298897, + "loss": 1.2946, + "theoretical_loss": 3.6578234813677053, + "tokens_seen": 1059258368 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003429488465396189, + "loss": 1.4191, + "theoretical_loss": 3.6578023395155173, + "tokens_seen": 1059323904 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034293881644934806, + "loss": 1.3629, + "theoretical_loss": 3.6577811993374514, + "tokens_seen": 1059389440 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034292878635907724, + "loss": 1.2714, + "theoretical_loss": 3.6577600608332705, + "tokens_seen": 1059454976 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003429187562688064, + "loss": 1.2346, + "theoretical_loss": 3.6577389240027394, + "tokens_seen": 1059520512 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034290872617853566, + "loss": 1.3056, + "theoretical_loss": 3.657717788845621, + "tokens_seen": 1059586048 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003428986960882648, + "loss": 1.2553, + "theoretical_loss": 3.65769665536168, + "tokens_seen": 1059651584 + }, + { + "epoch": 0.32, + "learning_rate": 0.000342888665997994, + "loss": 1.2955, + "theoretical_loss": 3.6576755235506804, + "tokens_seen": 1059717120 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034287863590772315, + "loss": 1.1918, + "theoretical_loss": 3.6576543934123866, + "tokens_seen": 1059782656 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003428686058174524, + "loss": 1.4856, + "theoretical_loss": 3.657633264946562, + "tokens_seen": 1059848192 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034285857572718156, + "loss": 1.3112, + "theoretical_loss": 3.6576121381529716, + "tokens_seen": 1059913728 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034284854563691074, + "loss": 1.2087, + "theoretical_loss": 3.657591013031379, + "tokens_seen": 1059979264 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003428385155466399, + "loss": 1.4298, + "theoretical_loss": 3.6575698895815494, + "tokens_seen": 1060044800 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003428284854563691, + "loss": 1.2844, + "theoretical_loss": 3.657548767803246, + "tokens_seen": 1060110336 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003428184553660983, + "loss": 1.3144, + "theoretical_loss": 3.6575276476962335, + "tokens_seen": 1060175872 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003428084252758275, + "loss": 1.3413, + "theoretical_loss": 3.6575065292602775, + "tokens_seen": 1060241408 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034279839518555665, + "loss": 1.3291, + "theoretical_loss": 3.6574854124951415, + "tokens_seen": 1060306944 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003427883650952859, + "loss": 1.2915, + "theoretical_loss": 3.65746429740059, + "tokens_seen": 1060372480 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034277833500501507, + "loss": 1.3951, + "theoretical_loss": 3.6574431839763877, + "tokens_seen": 1060438016 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034276830491474425, + "loss": 1.3668, + "theoretical_loss": 3.6574220722222996, + "tokens_seen": 1060503552 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034275827482447343, + "loss": 1.365, + "theoretical_loss": 3.65740096213809, + "tokens_seen": 1060569088 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003427482447342026, + "loss": 1.348, + "theoretical_loss": 3.657379853723524, + "tokens_seen": 1060634624 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003427382146439318, + "loss": 1.4574, + "theoretical_loss": 3.657358746978366, + "tokens_seen": 1060700160 + }, + { + "epoch": 0.32, + "learning_rate": 0.000342728184553661, + "loss": 1.3747, + "theoretical_loss": 3.6573376419023815, + "tokens_seen": 1060765696 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034271815446339015, + "loss": 1.2211, + "theoretical_loss": 3.6573165384953343, + "tokens_seen": 1060831232 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003427081243731194, + "loss": 1.2602, + "theoretical_loss": 3.6572954367569905, + "tokens_seen": 1060896768 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003426980942828485, + "loss": 1.3247, + "theoretical_loss": 3.657274336687114, + "tokens_seen": 1060962304 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034268806419257775, + "loss": 1.2781, + "theoretical_loss": 3.6572532382854708, + "tokens_seen": 1061027840 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034267803410230693, + "loss": 1.2704, + "theoretical_loss": 3.6572321415518254, + "tokens_seen": 1061093376 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003426680040120361, + "loss": 1.258, + "theoretical_loss": 3.657211046485943, + "tokens_seen": 1061158912 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003426579739217653, + "loss": 1.3942, + "theoretical_loss": 3.6571899530875887, + "tokens_seen": 1061224448 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034264794383149453, + "loss": 1.4007, + "theoretical_loss": 3.657168861356528, + "tokens_seen": 1061289984 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034263791374122366, + "loss": 1.3252, + "theoretical_loss": 3.657147771292526, + "tokens_seen": 1061355520 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003426278836509529, + "loss": 1.3465, + "theoretical_loss": 3.657126682895348, + "tokens_seen": 1061421056 + }, + { + "epoch": 0.32, + "learning_rate": 0.000342617853560682, + "loss": 1.3054, + "theoretical_loss": 3.65710559616476, + "tokens_seen": 1061486592 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034260782347041125, + "loss": 1.2441, + "theoretical_loss": 3.6570845111005257, + "tokens_seen": 1061552128 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034259779338014043, + "loss": 1.2644, + "theoretical_loss": 3.657063427702412, + "tokens_seen": 1061617664 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 356305, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2597023248672485, + "objective/train/theoretical_loss": 3.6570423459701846, + "objective/train/tokens_used": 1061683200, + "theoretical_loss": 3.6570423459701846, + "tokens_seen": 1061683200 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003425877632898696, + "loss": 1.3646, + "theoretical_loss": 3.6570423459701846, + "tokens_seen": 1061683200 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003425777331995988, + "loss": 1.2931, + "theoretical_loss": 3.657021265903608, + "tokens_seen": 1061748736 + }, + { + "epoch": 0.32, + "learning_rate": 0.000342567703109328, + "loss": 1.3205, + "theoretical_loss": 3.6570001875024483, + "tokens_seen": 1061814272 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034255767301905716, + "loss": 1.2925, + "theoretical_loss": 3.6569791107664713, + "tokens_seen": 1061879808 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003425476429287864, + "loss": 1.2059, + "theoretical_loss": 3.6569580356954425, + "tokens_seen": 1061945344 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003425376128385155, + "loss": 1.4006, + "theoretical_loss": 3.6569369622891283, + "tokens_seen": 1062010880 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034252758274824476, + "loss": 1.565, + "theoretical_loss": 3.6569158905472934, + "tokens_seen": 1062076416 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003425175526579739, + "loss": 1.3227, + "theoretical_loss": 3.656894820469704, + "tokens_seen": 1062141952 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003425075225677031, + "loss": 1.2879, + "theoretical_loss": 3.656873752056126, + "tokens_seen": 1062207488 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003424974924774323, + "loss": 1.4256, + "theoretical_loss": 3.6568526853063252, + "tokens_seen": 1062273024 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003424874623871615, + "loss": 1.3253, + "theoretical_loss": 3.6568316202200686, + "tokens_seen": 1062338560 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034247743229689066, + "loss": 1.2519, + "theoretical_loss": 3.656810556797121, + "tokens_seen": 1062404096 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003424674022066199, + "loss": 1.4537, + "theoretical_loss": 3.6567894950372493, + "tokens_seen": 1062469632 + }, + { + "epoch": 0.32, + "learning_rate": 0.000342457372116349, + "loss": 1.2905, + "theoretical_loss": 3.6567684349402185, + "tokens_seen": 1062535168 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034244734202607826, + "loss": 1.2686, + "theoretical_loss": 3.6567473765057956, + "tokens_seen": 1062600704 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003424373119358074, + "loss": 1.3369, + "theoretical_loss": 3.656726319733747, + "tokens_seen": 1062666240 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003424272818455366, + "loss": 1.3182, + "theoretical_loss": 3.656705264623839, + "tokens_seen": 1062731776 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003424172517552658, + "loss": 1.3862, + "theoretical_loss": 3.656684211175837, + "tokens_seen": 1062797312 + }, + { + "epoch": 0.32, + "learning_rate": 0.000342407221664995, + "loss": 1.3196, + "theoretical_loss": 3.6566631593895083, + "tokens_seen": 1062862848 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034239719157472416, + "loss": 1.2676, + "theoretical_loss": 3.656642109264619, + "tokens_seen": 1062928384 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034238716148445335, + "loss": 1.3444, + "theoretical_loss": 3.6566210608009353, + "tokens_seen": 1062993920 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003423771313941825, + "loss": 1.2952, + "theoretical_loss": 3.6566000139982235, + "tokens_seen": 1063059456 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034236710130391176, + "loss": 1.3072, + "theoretical_loss": 3.656578968856251, + "tokens_seen": 1063124992 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003423570712136409, + "loss": 1.3173, + "theoretical_loss": 3.656557925374784, + "tokens_seen": 1063190528 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003423470411233701, + "loss": 1.2986, + "theoretical_loss": 3.6565368835535885, + "tokens_seen": 1063256064 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003423370110330993, + "loss": 1.4081, + "theoretical_loss": 3.656515843392432, + "tokens_seen": 1063321600 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003423269809428285, + "loss": 1.3537, + "theoretical_loss": 3.6564948048910813, + "tokens_seen": 1063387136 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003423169508525577, + "loss": 1.3032, + "theoretical_loss": 3.656473768049303, + "tokens_seen": 1063452672 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034230692076228685, + "loss": 1.2668, + "theoretical_loss": 3.6564527328668635, + "tokens_seen": 1063518208 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003422968906720161, + "loss": 1.3307, + "theoretical_loss": 3.6564316993435297, + "tokens_seen": 1063583744 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034228686058174527, + "loss": 1.273, + "theoretical_loss": 3.656410667479069, + "tokens_seen": 1063649280 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034227683049147445, + "loss": 1.2616, + "theoretical_loss": 3.6563896372732483, + "tokens_seen": 1063714816 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034226680040120363, + "loss": 1.218, + "theoretical_loss": 3.6563686087258347, + "tokens_seen": 1063780352 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003422567703109328, + "loss": 1.3037, + "theoretical_loss": 3.6563475818365943, + "tokens_seen": 1063845888 + }, + { + "epoch": 0.32, + "learning_rate": 0.000342246740220662, + "loss": 1.2308, + "theoretical_loss": 3.6563265566052956, + "tokens_seen": 1063911424 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003422367101303912, + "loss": 1.3617, + "theoretical_loss": 3.6563055330317047, + "tokens_seen": 1063976960 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034222668004012035, + "loss": 1.3917, + "theoretical_loss": 3.6562845111155893, + "tokens_seen": 1064042496 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003422166499498496, + "loss": 1.288, + "theoretical_loss": 3.656263490856717, + "tokens_seen": 1064108032 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003422066198595787, + "loss": 1.2207, + "theoretical_loss": 3.6562424722548545, + "tokens_seen": 1064173568 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034219658976930795, + "loss": 1.349, + "theoretical_loss": 3.656221455309769, + "tokens_seen": 1064239104 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034218655967903713, + "loss": 1.2913, + "theoretical_loss": 3.6562004400212285, + "tokens_seen": 1064304640 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003421765295887663, + "loss": 1.2965, + "theoretical_loss": 3.6561794263890004, + "tokens_seen": 1064370176 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003421664994984955, + "loss": 1.386, + "theoretical_loss": 3.656158414412851, + "tokens_seen": 1064435712 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034215646940822473, + "loss": 1.4194, + "theoretical_loss": 3.65613740409255, + "tokens_seen": 1064501248 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034214643931795386, + "loss": 1.2617, + "theoretical_loss": 3.656116395427863, + "tokens_seen": 1064566784 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003421364092276831, + "loss": 1.3573, + "theoretical_loss": 3.6560953884185583, + "tokens_seen": 1064632320 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003421263791374122, + "loss": 1.2494, + "theoretical_loss": 3.6560743830644036, + "tokens_seen": 1064697856 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034211634904714145, + "loss": 1.2552, + "theoretical_loss": 3.656053379365167, + "tokens_seen": 1064763392 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034210631895687063, + "loss": 1.3462, + "theoretical_loss": 3.656032377320616, + "tokens_seen": 1064828928 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003420962888665998, + "loss": 1.3375, + "theoretical_loss": 3.6560113769305183, + "tokens_seen": 1064894464 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 357559, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4927496910095215, + "objective/train/theoretical_loss": 3.655990378194642, + "objective/train/tokens_used": 1064960000, + "theoretical_loss": 3.655990378194642, + "tokens_seen": 1064960000 + }, + { + "epoch": 0.32, + "learning_rate": 0.000342086258776329, + "loss": 1.4463, + "theoretical_loss": 3.655990378194642, + "tokens_seen": 1064960000 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003420762286860582, + "loss": 1.2306, + "theoretical_loss": 3.655969381112754, + "tokens_seen": 1065025536 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034206619859578736, + "loss": 1.3178, + "theoretical_loss": 3.655948385684624, + "tokens_seen": 1065091072 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003420561685055166, + "loss": 1.2379, + "theoretical_loss": 3.6559273919100184, + "tokens_seen": 1065156608 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003420461384152457, + "loss": 1.2229, + "theoretical_loss": 3.6559063997887065, + "tokens_seen": 1065222144 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034203610832497496, + "loss": 1.4009, + "theoretical_loss": 3.6558854093204554, + "tokens_seen": 1065287680 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003420260782347041, + "loss": 1.2776, + "theoretical_loss": 3.655864420505034, + "tokens_seen": 1065353216 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003420160481444333, + "loss": 1.3746, + "theoretical_loss": 3.65584343334221, + "tokens_seen": 1065418752 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003420060180541625, + "loss": 1.2515, + "theoretical_loss": 3.655822447831752, + "tokens_seen": 1065484288 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003419959879638917, + "loss": 1.3717, + "theoretical_loss": 3.655801463973428, + "tokens_seen": 1065549824 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034198595787362086, + "loss": 1.3152, + "theoretical_loss": 3.6557804817670063, + "tokens_seen": 1065615360 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003419759277833501, + "loss": 1.3052, + "theoretical_loss": 3.6557595012122555, + "tokens_seen": 1065680896 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003419658976930792, + "loss": 1.3429, + "theoretical_loss": 3.655738522308944, + "tokens_seen": 1065746432 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034195586760280846, + "loss": 1.2836, + "theoretical_loss": 3.65571754505684, + "tokens_seen": 1065811968 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003419458375125376, + "loss": 1.3588, + "theoretical_loss": 3.6556965694557126, + "tokens_seen": 1065877504 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003419358074222668, + "loss": 1.3376, + "theoretical_loss": 3.6556755955053295, + "tokens_seen": 1065943040 + }, + { + "epoch": 0.32, + "learning_rate": 0.000341925777331996, + "loss": 1.2654, + "theoretical_loss": 3.65565462320546, + "tokens_seen": 1066008576 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003419157472417252, + "loss": 1.4546, + "theoretical_loss": 3.6556336525558724, + "tokens_seen": 1066074112 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034190571715145436, + "loss": 1.4678, + "theoretical_loss": 3.6556126835563356, + "tokens_seen": 1066139648 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034189568706118355, + "loss": 1.3483, + "theoretical_loss": 3.6555917162066187, + "tokens_seen": 1066205184 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003418856569709127, + "loss": 1.3096, + "theoretical_loss": 3.65557075050649, + "tokens_seen": 1066270720 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034187562688064196, + "loss": 1.2506, + "theoretical_loss": 3.6555497864557176, + "tokens_seen": 1066336256 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003418655967903711, + "loss": 1.3091, + "theoretical_loss": 3.655528824054072, + "tokens_seen": 1066401792 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003418555667001003, + "loss": 1.2612, + "theoretical_loss": 3.655507863301321, + "tokens_seen": 1066467328 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034184553660982945, + "loss": 1.3917, + "theoretical_loss": 3.655486904197234, + "tokens_seen": 1066532864 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003418355065195587, + "loss": 1.3912, + "theoretical_loss": 3.65546594674158, + "tokens_seen": 1066598400 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034182547642928787, + "loss": 1.2989, + "theoretical_loss": 3.6554449909341287, + "tokens_seen": 1066663936 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034181544633901705, + "loss": 1.1829, + "theoretical_loss": 3.6554240367746473, + "tokens_seen": 1066729472 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034180541624874623, + "loss": 1.3441, + "theoretical_loss": 3.6554030842629075, + "tokens_seen": 1066795008 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034179538615847547, + "loss": 1.3653, + "theoretical_loss": 3.6553821333986765, + "tokens_seen": 1066860544 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003417853560682046, + "loss": 1.3623, + "theoretical_loss": 3.6553611841817246, + "tokens_seen": 1066926080 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034177532597793383, + "loss": 1.3493, + "theoretical_loss": 3.6553402366118206, + "tokens_seen": 1066991616 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034176529588766295, + "loss": 1.432, + "theoretical_loss": 3.6553192906887344, + "tokens_seen": 1067057152 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003417552657973922, + "loss": 1.324, + "theoretical_loss": 3.655298346412235, + "tokens_seen": 1067122688 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034174523570712137, + "loss": 1.2742, + "theoretical_loss": 3.655277403782092, + "tokens_seen": 1067188224 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034173520561685055, + "loss": 1.3578, + "theoretical_loss": 3.6552564627980746, + "tokens_seen": 1067253760 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034172517552657973, + "loss": 1.3229, + "theoretical_loss": 3.655235523459953, + "tokens_seen": 1067319296 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003417151454363089, + "loss": 1.2975, + "theoretical_loss": 3.655214585767496, + "tokens_seen": 1067384832 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003417051153460381, + "loss": 1.3088, + "theoretical_loss": 3.6551936497204736, + "tokens_seen": 1067450368 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034169508525576733, + "loss": 1.3019, + "theoretical_loss": 3.6551727153186553, + "tokens_seen": 1067515904 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034168505516549646, + "loss": 1.3515, + "theoretical_loss": 3.6551517825618114, + "tokens_seen": 1067581440 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003416750250752257, + "loss": 1.1852, + "theoretical_loss": 3.6551308514497114, + "tokens_seen": 1067646976 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003416649949849548, + "loss": 1.3946, + "theoretical_loss": 3.6551099219821244, + "tokens_seen": 1067712512 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034165496489468406, + "loss": 1.3523, + "theoretical_loss": 3.655088994158821, + "tokens_seen": 1067778048 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034164493480441324, + "loss": 1.2942, + "theoretical_loss": 3.6550680679795713, + "tokens_seen": 1067843584 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003416349047141424, + "loss": 1.4229, + "theoretical_loss": 3.655047143444145, + "tokens_seen": 1067909120 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003416248746238716, + "loss": 1.2678, + "theoretical_loss": 3.655026220552312, + "tokens_seen": 1067974656 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034161484453360083, + "loss": 1.3659, + "theoretical_loss": 3.655005299303842, + "tokens_seen": 1068040192 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034160481444332996, + "loss": 1.1886, + "theoretical_loss": 3.6549843796985053, + "tokens_seen": 1068105728 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003415947843530592, + "loss": 1.3343, + "theoretical_loss": 3.654963461736073, + "tokens_seen": 1068171264 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 358459, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2238588333129883, + "objective/train/theoretical_loss": 3.654942545416314, + "objective/train/tokens_used": 1068236800, + "theoretical_loss": 3.654942545416314, + "tokens_seen": 1068236800 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003415847542627884, + "loss": 1.2786, + "theoretical_loss": 3.654942545416314, + "tokens_seen": 1068236800 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034157472417251756, + "loss": 1.2524, + "theoretical_loss": 3.6549216307389996, + "tokens_seen": 1068302336 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003415646940822468, + "loss": 1.3449, + "theoretical_loss": 3.6549007177038986, + "tokens_seen": 1068367872 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003415546639919759, + "loss": 1.2867, + "theoretical_loss": 3.654879806310783, + "tokens_seen": 1068433408 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034154463390170516, + "loss": 1.2785, + "theoretical_loss": 3.654858896559422, + "tokens_seen": 1068498944 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003415346038114343, + "loss": 1.2202, + "theoretical_loss": 3.654837988449587, + "tokens_seen": 1068564480 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003415245737211635, + "loss": 1.4208, + "theoretical_loss": 3.6548170819810477, + "tokens_seen": 1068630016 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003415145436308927, + "loss": 1.3342, + "theoretical_loss": 3.6547961771535746, + "tokens_seen": 1068695552 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003415045135406219, + "loss": 1.3071, + "theoretical_loss": 3.654775273966939, + "tokens_seen": 1068761088 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034149448345035106, + "loss": 1.3841, + "theoretical_loss": 3.6547543724209106, + "tokens_seen": 1068826624 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003414844533600803, + "loss": 1.3133, + "theoretical_loss": 3.654733472515261, + "tokens_seen": 1068892160 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003414744232698094, + "loss": 1.3248, + "theoretical_loss": 3.6547125742497597, + "tokens_seen": 1068957696 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034146439317953866, + "loss": 1.3512, + "theoretical_loss": 3.6546916776241787, + "tokens_seen": 1069023232 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003414543630892678, + "loss": 1.3683, + "theoretical_loss": 3.654670782638288, + "tokens_seen": 1069088768 + }, + { + "epoch": 0.32, + "learning_rate": 0.000341444332998997, + "loss": 1.2314, + "theoretical_loss": 3.654649889291858, + "tokens_seen": 1069154304 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003414343029087262, + "loss": 1.2531, + "theoretical_loss": 3.654628997584661, + "tokens_seen": 1069219840 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003414242728184554, + "loss": 1.3921, + "theoretical_loss": 3.6546081075164665, + "tokens_seen": 1069285376 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034141424272818456, + "loss": 1.3971, + "theoretical_loss": 3.6545872190870465, + "tokens_seen": 1069350912 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034140421263791375, + "loss": 1.2556, + "theoretical_loss": 3.6545663322961714, + "tokens_seen": 1069416448 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034139418254764293, + "loss": 1.296, + "theoretical_loss": 3.6545454471436125, + "tokens_seen": 1069481984 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034138415245737216, + "loss": 1.3753, + "theoretical_loss": 3.654524563629141, + "tokens_seen": 1069547520 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003413741223671013, + "loss": 1.3006, + "theoretical_loss": 3.6545036817525283, + "tokens_seen": 1069613056 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003413640922768305, + "loss": 1.4189, + "theoretical_loss": 3.654482801513545, + "tokens_seen": 1069678592 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034135406218655965, + "loss": 1.3319, + "theoretical_loss": 3.654461922911963, + "tokens_seen": 1069744128 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003413440320962889, + "loss": 1.3095, + "theoretical_loss": 3.6544410459475527, + "tokens_seen": 1069809664 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034133400200601807, + "loss": 1.2754, + "theoretical_loss": 3.654420170620086, + "tokens_seen": 1069875200 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034132397191574725, + "loss": 1.3256, + "theoretical_loss": 3.654399296929335, + "tokens_seen": 1069940736 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034131394182547643, + "loss": 1.3221, + "theoretical_loss": 3.6543784248750697, + "tokens_seen": 1070006272 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034130391173520567, + "loss": 1.3322, + "theoretical_loss": 3.6543575544570626, + "tokens_seen": 1070071808 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003412938816449348, + "loss": 1.2247, + "theoretical_loss": 3.6543366856750845, + "tokens_seen": 1070137344 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034128385155466403, + "loss": 1.3377, + "theoretical_loss": 3.6543158185289077, + "tokens_seen": 1070202880 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034127382146439315, + "loss": 1.2364, + "theoretical_loss": 3.6542949530183035, + "tokens_seen": 1070268416 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003412637913741224, + "loss": 1.2959, + "theoretical_loss": 3.654274089143043, + "tokens_seen": 1070333952 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034125376128385157, + "loss": 1.4338, + "theoretical_loss": 3.654253226902899, + "tokens_seen": 1070399488 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034124373119358075, + "loss": 1.4226, + "theoretical_loss": 3.6542323662976424, + "tokens_seen": 1070465024 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034123370110330993, + "loss": 1.3024, + "theoretical_loss": 3.654211507327046, + "tokens_seen": 1070530560 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003412236710130391, + "loss": 1.2193, + "theoretical_loss": 3.6541906499908805, + "tokens_seen": 1070596096 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003412136409227683, + "loss": 1.3686, + "theoretical_loss": 3.6541697942889177, + "tokens_seen": 1070661632 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034120361083249753, + "loss": 1.266, + "theoretical_loss": 3.654148940220931, + "tokens_seen": 1070727168 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034119358074222666, + "loss": 1.3021, + "theoretical_loss": 3.654128087786691, + "tokens_seen": 1070792704 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003411835506519559, + "loss": 1.382, + "theoretical_loss": 3.65410723698597, + "tokens_seen": 1070858240 + }, + { + "epoch": 0.32, + "learning_rate": 0.000341173520561685, + "loss": 1.343, + "theoretical_loss": 3.654086387818541, + "tokens_seen": 1070923776 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034116349047141426, + "loss": 1.338, + "theoretical_loss": 3.654065540284175, + "tokens_seen": 1070989312 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034115346038114344, + "loss": 1.2364, + "theoretical_loss": 3.6540446943826446, + "tokens_seen": 1071054848 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003411434302908726, + "loss": 1.3832, + "theoretical_loss": 3.654023850113722, + "tokens_seen": 1071120384 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003411334002006018, + "loss": 1.3092, + "theoretical_loss": 3.654003007477179, + "tokens_seen": 1071185920 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034112337011033103, + "loss": 1.2862, + "theoretical_loss": 3.6539821664727894, + "tokens_seen": 1071251456 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034111334002006016, + "loss": 1.396, + "theoretical_loss": 3.6539613271003235, + "tokens_seen": 1071316992 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003411033099297894, + "loss": 1.3757, + "theoretical_loss": 3.653940489359555, + "tokens_seen": 1071382528 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003410932798395185, + "loss": 1.4856, + "theoretical_loss": 3.6539196532502562, + "tokens_seen": 1071448064 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 359663, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.296759843826294, + "objective/train/theoretical_loss": 3.6538988187721997, + "objective/train/tokens_used": 1071513600, + "theoretical_loss": 3.6538988187721997, + "tokens_seen": 1071513600 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034108324974924776, + "loss": 1.3464, + "theoretical_loss": 3.6538988187721997, + "tokens_seen": 1071513600 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034107321965897694, + "loss": 1.3518, + "theoretical_loss": 3.653877985925157, + "tokens_seen": 1071579136 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003410631895687061, + "loss": 1.4159, + "theoretical_loss": 3.653857154708902, + "tokens_seen": 1071644672 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003410531594784353, + "loss": 1.3483, + "theoretical_loss": 3.6538363251232067, + "tokens_seen": 1071710208 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003410431293881645, + "loss": 1.3387, + "theoretical_loss": 3.6538154971678436, + "tokens_seen": 1071775744 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034103309929789366, + "loss": 1.2933, + "theoretical_loss": 3.6537946708425864, + "tokens_seen": 1071841280 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003410230692076229, + "loss": 1.3211, + "theoretical_loss": 3.653773846147206, + "tokens_seen": 1071906816 + }, + { + "epoch": 0.32, + "learning_rate": 0.000341013039117352, + "loss": 1.3712, + "theoretical_loss": 3.653753023081477, + "tokens_seen": 1071972352 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034100300902708126, + "loss": 1.3416, + "theoretical_loss": 3.6537322016451714, + "tokens_seen": 1072037888 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034099297893681044, + "loss": 1.419, + "theoretical_loss": 3.653711381838062, + "tokens_seen": 1072103424 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003409829488465396, + "loss": 1.2775, + "theoretical_loss": 3.6536905636599224, + "tokens_seen": 1072168960 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003409729187562688, + "loss": 1.358, + "theoretical_loss": 3.6536697471105253, + "tokens_seen": 1072234496 + }, + { + "epoch": 0.32, + "learning_rate": 0.000340962888665998, + "loss": 1.2829, + "theoretical_loss": 3.6536489321896433, + "tokens_seen": 1072300032 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034095285857572717, + "loss": 1.3786, + "theoretical_loss": 3.65362811889705, + "tokens_seen": 1072365568 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003409428284854564, + "loss": 1.3057, + "theoretical_loss": 3.653607307232518, + "tokens_seen": 1072431104 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034093279839518553, + "loss": 1.2474, + "theoretical_loss": 3.653586497195821, + "tokens_seen": 1072496640 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034092276830491477, + "loss": 1.3967, + "theoretical_loss": 3.653565688786732, + "tokens_seen": 1072562176 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003409127382146439, + "loss": 1.3522, + "theoretical_loss": 3.6535448820050247, + "tokens_seen": 1072627712 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034090270812437313, + "loss": 1.3565, + "theoretical_loss": 3.6535240768504718, + "tokens_seen": 1072693248 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003408926780341023, + "loss": 1.2719, + "theoretical_loss": 3.6535032733228467, + "tokens_seen": 1072758784 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003408826479438315, + "loss": 1.3424, + "theoretical_loss": 3.6534824714219227, + "tokens_seen": 1072824320 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034087261785356067, + "loss": 1.2938, + "theoretical_loss": 3.6534616711474737, + "tokens_seen": 1072889856 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034086258776328985, + "loss": 1.4119, + "theoretical_loss": 3.653440872499273, + "tokens_seen": 1072955392 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034085255767301903, + "loss": 1.3764, + "theoretical_loss": 3.6534200754770945, + "tokens_seen": 1073020928 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034084252758274827, + "loss": 1.3865, + "theoretical_loss": 3.653399280080711, + "tokens_seen": 1073086464 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034083249749247745, + "loss": 1.3781, + "theoretical_loss": 3.6533784863098964, + "tokens_seen": 1073152000 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034082246740220663, + "loss": 1.2183, + "theoretical_loss": 3.6533576941644244, + "tokens_seen": 1073217536 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034081243731193587, + "loss": 1.3272, + "theoretical_loss": 3.6533369036440693, + "tokens_seen": 1073283072 + }, + { + "epoch": 0.33, + "learning_rate": 0.000340802407221665, + "loss": 1.1062, + "theoretical_loss": 3.6533161147486037, + "tokens_seen": 1073348608 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034079237713139423, + "loss": 1.3017, + "theoretical_loss": 3.6532953274778026, + "tokens_seen": 1073414144 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034078234704112335, + "loss": 1.2981, + "theoretical_loss": 3.653274541831439, + "tokens_seen": 1073479680 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003407723169508526, + "loss": 1.319, + "theoretical_loss": 3.6532537578092876, + "tokens_seen": 1073545216 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034076228686058177, + "loss": 1.4021, + "theoretical_loss": 3.6532329754111217, + "tokens_seen": 1073610752 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034075225677031095, + "loss": 1.3625, + "theoretical_loss": 3.6532121946367146, + "tokens_seen": 1073676288 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034074222668004013, + "loss": 1.3194, + "theoretical_loss": 3.653191415485842, + "tokens_seen": 1073741824 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003407321965897693, + "loss": 1.3852, + "theoretical_loss": 3.6531706379582776, + "tokens_seen": 1073807360 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003407221664994985, + "loss": 1.439, + "theoretical_loss": 3.653149862053794, + "tokens_seen": 1073872896 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034071213640922773, + "loss": 1.4822, + "theoretical_loss": 3.653129087772167, + "tokens_seen": 1073938432 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034070210631895686, + "loss": 1.3458, + "theoretical_loss": 3.65310831511317, + "tokens_seen": 1074003968 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003406920762286861, + "loss": 1.2956, + "theoretical_loss": 3.6530875440765778, + "tokens_seen": 1074069504 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003406820461384152, + "loss": 1.3489, + "theoretical_loss": 3.6530667746621646, + "tokens_seen": 1074135040 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034067201604814446, + "loss": 1.3269, + "theoretical_loss": 3.6530460068697037, + "tokens_seen": 1074200576 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034066198595787364, + "loss": 1.2648, + "theoretical_loss": 3.6530252406989714, + "tokens_seen": 1074266112 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003406519558676028, + "loss": 1.3674, + "theoretical_loss": 3.65300447614974, + "tokens_seen": 1074331648 + }, + { + "epoch": 0.33, + "learning_rate": 0.000340641925777332, + "loss": 1.3716, + "theoretical_loss": 3.6529837132217864, + "tokens_seen": 1074397184 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034063189568706123, + "loss": 1.3762, + "theoretical_loss": 3.6529629519148825, + "tokens_seen": 1074462720 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034062186559679036, + "loss": 1.2737, + "theoretical_loss": 3.6529421922288052, + "tokens_seen": 1074528256 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003406118355065196, + "loss": 1.452, + "theoretical_loss": 3.6529214341633276, + "tokens_seen": 1074593792 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003406018054162487, + "loss": 1.3086, + "theoretical_loss": 3.6529006777182254, + "tokens_seen": 1074659328 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034059177532597796, + "loss": 1.2998, + "theoretical_loss": 3.652879922893272, + "tokens_seen": 1074724864 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 360904, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2254668474197388, + "objective/train/theoretical_loss": 3.6528591696882433, + "objective/train/tokens_used": 1074790400, + "theoretical_loss": 3.6528591696882433, + "tokens_seen": 1074790400 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034058174523570714, + "loss": 1.3186, + "theoretical_loss": 3.6528591696882433, + "tokens_seen": 1074790400 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003405717151454363, + "loss": 1.3857, + "theoretical_loss": 3.652838418102914, + "tokens_seen": 1074855936 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003405616850551655, + "loss": 1.3067, + "theoretical_loss": 3.6528176681370583, + "tokens_seen": 1074921472 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003405516549648947, + "loss": 1.4328, + "theoretical_loss": 3.6527969197904513, + "tokens_seen": 1074987008 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034054162487462386, + "loss": 1.2938, + "theoretical_loss": 3.6527761730628687, + "tokens_seen": 1075052544 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003405315947843531, + "loss": 1.3415, + "theoretical_loss": 3.6527554279540846, + "tokens_seen": 1075118080 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003405215646940822, + "loss": 1.2725, + "theoretical_loss": 3.6527346844638746, + "tokens_seen": 1075183616 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034051153460381146, + "loss": 1.33, + "theoretical_loss": 3.6527139425920128, + "tokens_seen": 1075249152 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034050150451354064, + "loss": 1.3656, + "theoretical_loss": 3.652693202338275, + "tokens_seen": 1075314688 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003404914744232698, + "loss": 1.3098, + "theoretical_loss": 3.6526724637024373, + "tokens_seen": 1075380224 + }, + { + "epoch": 0.33, + "learning_rate": 0.000340481444332999, + "loss": 1.2922, + "theoretical_loss": 3.6526517266842733, + "tokens_seen": 1075445760 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003404714142427282, + "loss": 1.3095, + "theoretical_loss": 3.652630991283559, + "tokens_seen": 1075511296 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034046138415245737, + "loss": 1.3536, + "theoretical_loss": 3.6526102575000694, + "tokens_seen": 1075576832 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003404513540621866, + "loss": 1.4169, + "theoretical_loss": 3.6525895253335805, + "tokens_seen": 1075642368 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034044132397191573, + "loss": 1.4127, + "theoretical_loss": 3.6525687947838668, + "tokens_seen": 1075707904 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034043129388164497, + "loss": 1.3107, + "theoretical_loss": 3.652548065850705, + "tokens_seen": 1075773440 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003404212637913741, + "loss": 1.3024, + "theoretical_loss": 3.6525273385338686, + "tokens_seen": 1075838976 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034041123370110333, + "loss": 1.2559, + "theoretical_loss": 3.6525066128331347, + "tokens_seen": 1075904512 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003404012036108325, + "loss": 1.4388, + "theoretical_loss": 3.6524858887482785, + "tokens_seen": 1075970048 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003403911735205617, + "loss": 1.304, + "theoretical_loss": 3.6524651662790752, + "tokens_seen": 1076035584 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034038114343029087, + "loss": 1.4328, + "theoretical_loss": 3.652444445425301, + "tokens_seen": 1076101120 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034037111334002005, + "loss": 1.3349, + "theoretical_loss": 3.652423726186732, + "tokens_seen": 1076166656 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034036108324974923, + "loss": 1.2597, + "theoretical_loss": 3.6524030085631427, + "tokens_seen": 1076232192 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034035105315947847, + "loss": 1.3119, + "theoretical_loss": 3.652382292554309, + "tokens_seen": 1076297728 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003403410230692076, + "loss": 1.2844, + "theoretical_loss": 3.652361578160008, + "tokens_seen": 1076363264 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034033099297893683, + "loss": 1.2938, + "theoretical_loss": 3.6523408653800145, + "tokens_seen": 1076428800 + }, + { + "epoch": 0.33, + "learning_rate": 0.000340320962888666, + "loss": 1.4451, + "theoretical_loss": 3.652320154214105, + "tokens_seen": 1076494336 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003403109327983952, + "loss": 1.3387, + "theoretical_loss": 3.652299444662055, + "tokens_seen": 1076559872 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003403009027081244, + "loss": 1.3847, + "theoretical_loss": 3.6522787367236407, + "tokens_seen": 1076625408 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034029087261785356, + "loss": 1.4483, + "theoretical_loss": 3.652258030398638, + "tokens_seen": 1076690944 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034028084252758274, + "loss": 1.3042, + "theoretical_loss": 3.6522373256868237, + "tokens_seen": 1076756480 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034027081243731197, + "loss": 1.2076, + "theoretical_loss": 3.652216622587973, + "tokens_seen": 1076822016 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003402607823470411, + "loss": 1.3869, + "theoretical_loss": 3.6521959211018626, + "tokens_seen": 1076887552 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034025075225677033, + "loss": 1.1995, + "theoretical_loss": 3.6521752212282688, + "tokens_seen": 1076953088 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034024072216649946, + "loss": 1.3222, + "theoretical_loss": 3.652154522966968, + "tokens_seen": 1077018624 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003402306920762287, + "loss": 1.388, + "theoretical_loss": 3.652133826317735, + "tokens_seen": 1077084160 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003402206619859579, + "loss": 1.3646, + "theoretical_loss": 3.6521131312803483, + "tokens_seen": 1077149696 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034021063189568706, + "loss": 1.2898, + "theoretical_loss": 3.652092437854584, + "tokens_seen": 1077215232 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034020060180541624, + "loss": 1.2966, + "theoretical_loss": 3.652071746040217, + "tokens_seen": 1077280768 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003401905717151454, + "loss": 1.3795, + "theoretical_loss": 3.652051055837025, + "tokens_seen": 1077346304 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003401805416248746, + "loss": 1.2795, + "theoretical_loss": 3.6520303672447847, + "tokens_seen": 1077411840 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034017051153460384, + "loss": 1.3569, + "theoretical_loss": 3.652009680263272, + "tokens_seen": 1077477376 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034016048144433296, + "loss": 1.4528, + "theoretical_loss": 3.651988994892264, + "tokens_seen": 1077542912 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003401504513540622, + "loss": 1.2433, + "theoretical_loss": 3.6519683111315375, + "tokens_seen": 1077608448 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003401404212637914, + "loss": 1.3318, + "theoretical_loss": 3.651947628980869, + "tokens_seen": 1077673984 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034013039117352056, + "loss": 1.2824, + "theoretical_loss": 3.6519269484400345, + "tokens_seen": 1077739520 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034012036108324974, + "loss": 1.2303, + "theoretical_loss": 3.651906269508812, + "tokens_seen": 1077805056 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003401103309929789, + "loss": 1.44, + "theoretical_loss": 3.651885592186978, + "tokens_seen": 1077870592 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003401003009027081, + "loss": 1.3468, + "theoretical_loss": 3.6518649164743096, + "tokens_seen": 1077936128 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034009027081243734, + "loss": 1.3599, + "theoretical_loss": 3.651844242370583, + "tokens_seen": 1078001664 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 361855, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3081943988800049, + "objective/train/theoretical_loss": 3.651823569875576, + "objective/train/tokens_used": 1078067200, + "theoretical_loss": 3.651823569875576, + "tokens_seen": 1078067200 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003400802407221665, + "loss": 1.3128, + "theoretical_loss": 3.651823569875576, + "tokens_seen": 1078067200 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003400702106318957, + "loss": 1.2917, + "theoretical_loss": 3.6518028989890654, + "tokens_seen": 1078132736 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003400601805416249, + "loss": 1.3862, + "theoretical_loss": 3.651782229710828, + "tokens_seen": 1078198272 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034005015045135406, + "loss": 1.3517, + "theoretical_loss": 3.6517615620406407, + "tokens_seen": 1078263808 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003400401203610833, + "loss": 1.2631, + "theoretical_loss": 3.6517408959782816, + "tokens_seen": 1078329344 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003400300902708124, + "loss": 1.3566, + "theoretical_loss": 3.651720231523527, + "tokens_seen": 1078394880 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034002006018054166, + "loss": 1.3361, + "theoretical_loss": 3.651699568676155, + "tokens_seen": 1078460416 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034001003009027084, + "loss": 1.2752, + "theoretical_loss": 3.651678907435942, + "tokens_seen": 1078525952 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034, + "loss": 1.2784, + "theoretical_loss": 3.6516582478026662, + "tokens_seen": 1078591488 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003399899699097292, + "loss": 1.2638, + "theoretical_loss": 3.6516375897761044, + "tokens_seen": 1078657024 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003399799398194584, + "loss": 1.2929, + "theoretical_loss": 3.651616933356034, + "tokens_seen": 1078722560 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033996990972918757, + "loss": 1.3965, + "theoretical_loss": 3.6515962785422333, + "tokens_seen": 1078788096 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003399598796389168, + "loss": 1.3453, + "theoretical_loss": 3.6515756253344787, + "tokens_seen": 1078853632 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033994984954864593, + "loss": 1.3004, + "theoretical_loss": 3.651554973732549, + "tokens_seen": 1078919168 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033993981945837517, + "loss": 1.2402, + "theoretical_loss": 3.6515343237362203, + "tokens_seen": 1078984704 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003399297893681043, + "loss": 1.3149, + "theoretical_loss": 3.6515136753452717, + "tokens_seen": 1079050240 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033991975927783353, + "loss": 1.2831, + "theoretical_loss": 3.65149302855948, + "tokens_seen": 1079115776 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003399097291875627, + "loss": 1.2597, + "theoretical_loss": 3.651472383378623, + "tokens_seen": 1079181312 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003398996990972919, + "loss": 1.2985, + "theoretical_loss": 3.651451739802479, + "tokens_seen": 1079246848 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033988966900702107, + "loss": 1.2679, + "theoretical_loss": 3.6514310978308258, + "tokens_seen": 1079312384 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033987963891675025, + "loss": 1.2934, + "theoretical_loss": 3.6514104574634407, + "tokens_seen": 1079377920 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033986960882647943, + "loss": 1.321, + "theoretical_loss": 3.6513898187001024, + "tokens_seen": 1079443456 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033985957873620867, + "loss": 1.3199, + "theoretical_loss": 3.6513691815405878, + "tokens_seen": 1079508992 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003398495486459378, + "loss": 1.3628, + "theoretical_loss": 3.6513485459846757, + "tokens_seen": 1079574528 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033983951855566703, + "loss": 1.3045, + "theoretical_loss": 3.6513279120321442, + "tokens_seen": 1079640064 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003398294884653962, + "loss": 1.2808, + "theoretical_loss": 3.6513072796827712, + "tokens_seen": 1079705600 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003398194583751254, + "loss": 1.2637, + "theoretical_loss": 3.651286648936335, + "tokens_seen": 1079771136 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003398094282848546, + "loss": 1.3338, + "theoretical_loss": 3.6512660197926134, + "tokens_seen": 1079836672 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033979939819458376, + "loss": 1.3176, + "theoretical_loss": 3.6512453922513854, + "tokens_seen": 1079902208 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033978936810431294, + "loss": 1.2968, + "theoretical_loss": 3.6512247663124278, + "tokens_seen": 1079967744 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033977933801404217, + "loss": 1.3476, + "theoretical_loss": 3.6512041419755206, + "tokens_seen": 1080033280 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003397693079237713, + "loss": 1.3712, + "theoretical_loss": 3.6511835192404414, + "tokens_seen": 1080098816 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033975927783350053, + "loss": 1.3014, + "theoretical_loss": 3.6511628981069686, + "tokens_seen": 1080164352 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033974924774322966, + "loss": 1.3345, + "theoretical_loss": 3.6511422785748806, + "tokens_seen": 1080229888 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003397392176529589, + "loss": 1.3238, + "theoretical_loss": 3.651121660643956, + "tokens_seen": 1080295424 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003397291875626881, + "loss": 1.3754, + "theoretical_loss": 3.651101044313973, + "tokens_seen": 1080360960 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033971915747241726, + "loss": 1.4207, + "theoretical_loss": 3.651080429584711, + "tokens_seen": 1080426496 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033970912738214644, + "loss": 1.3166, + "theoretical_loss": 3.651059816455948, + "tokens_seen": 1080492032 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003396990972918756, + "loss": 1.3633, + "theoretical_loss": 3.6510392049274625, + "tokens_seen": 1080557568 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003396890672016048, + "loss": 1.3824, + "theoretical_loss": 3.651018594999034, + "tokens_seen": 1080623104 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033967903711133404, + "loss": 1.3664, + "theoretical_loss": 3.6509979866704407, + "tokens_seen": 1080688640 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033966900702106316, + "loss": 1.3379, + "theoretical_loss": 3.6509773799414615, + "tokens_seen": 1080754176 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003396589769307924, + "loss": 1.2566, + "theoretical_loss": 3.6509567748118754, + "tokens_seen": 1080819712 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003396489468405216, + "loss": 1.3017, + "theoretical_loss": 3.650936171281461, + "tokens_seen": 1080885248 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033963891675025076, + "loss": 1.2541, + "theoretical_loss": 3.6509155693499977, + "tokens_seen": 1080950784 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033962888665997994, + "loss": 1.3983, + "theoretical_loss": 3.650894969017264, + "tokens_seen": 1081016320 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003396188565697091, + "loss": 1.2967, + "theoretical_loss": 3.650874370283039, + "tokens_seen": 1081081856 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003396088264794383, + "loss": 1.3165, + "theoretical_loss": 3.650853773147102, + "tokens_seen": 1081147392 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033959879638916754, + "loss": 1.3842, + "theoretical_loss": 3.6508331776092318, + "tokens_seen": 1081212928 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033958876629889667, + "loss": 1.27, + "theoretical_loss": 3.6508125836692082, + "tokens_seen": 1081278464 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 363163, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2364295721054077, + "objective/train/theoretical_loss": 3.6507919913268094, + "objective/train/tokens_used": 1081344000, + "theoretical_loss": 3.6507919913268094, + "tokens_seen": 1081344000 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003395787362086259, + "loss": 1.2847, + "theoretical_loss": 3.6507919913268094, + "tokens_seen": 1081344000 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033956870611835503, + "loss": 1.3779, + "theoretical_loss": 3.6507714005818155, + "tokens_seen": 1081409536 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033955867602808426, + "loss": 1.3217, + "theoretical_loss": 3.6507508114340057, + "tokens_seen": 1081475072 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033954864593781345, + "loss": 1.4022, + "theoretical_loss": 3.6507302238831585, + "tokens_seen": 1081540608 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003395386158475426, + "loss": 1.3639, + "theoretical_loss": 3.6507096379290545, + "tokens_seen": 1081606144 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003395285857572718, + "loss": 1.3167, + "theoretical_loss": 3.6506890535714724, + "tokens_seen": 1081671680 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033951855566700104, + "loss": 1.2744, + "theoretical_loss": 3.6506684708101913, + "tokens_seen": 1081737216 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033950852557673017, + "loss": 1.3388, + "theoretical_loss": 3.6506478896449917, + "tokens_seen": 1081802752 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003394984954864594, + "loss": 1.2512, + "theoretical_loss": 3.6506273100756528, + "tokens_seen": 1081868288 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033948846539618853, + "loss": 1.3801, + "theoretical_loss": 3.6506067321019535, + "tokens_seen": 1081933824 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033947843530591777, + "loss": 1.353, + "theoretical_loss": 3.6505861557236745, + "tokens_seen": 1081999360 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033946840521564695, + "loss": 1.2374, + "theoretical_loss": 3.6505655809405946, + "tokens_seen": 1082064896 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033945837512537613, + "loss": 1.2829, + "theoretical_loss": 3.6505450077524944, + "tokens_seen": 1082130432 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003394483450351053, + "loss": 1.4221, + "theoretical_loss": 3.6505244361591527, + "tokens_seen": 1082195968 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003394383149448345, + "loss": 1.3532, + "theoretical_loss": 3.65050386616035, + "tokens_seen": 1082261504 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003394282848545637, + "loss": 1.4233, + "theoretical_loss": 3.6504832977558657, + "tokens_seen": 1082327040 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003394182547642929, + "loss": 1.3236, + "theoretical_loss": 3.6504627309454802, + "tokens_seen": 1082392576 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033940822467402204, + "loss": 1.4217, + "theoretical_loss": 3.6504421657289727, + "tokens_seen": 1082458112 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033939819458375127, + "loss": 1.4016, + "theoretical_loss": 3.6504216021061247, + "tokens_seen": 1082523648 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003393881644934804, + "loss": 1.3101, + "theoretical_loss": 3.6504010400767144, + "tokens_seen": 1082589184 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033937813440320963, + "loss": 1.3233, + "theoretical_loss": 3.650380479640523, + "tokens_seen": 1082654720 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003393681043129388, + "loss": 1.3536, + "theoretical_loss": 3.65035992079733, + "tokens_seen": 1082720256 + }, + { + "epoch": 0.33, + "learning_rate": 0.000339358074222668, + "loss": 1.2854, + "theoretical_loss": 3.650339363546916, + "tokens_seen": 1082785792 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003393480441323972, + "loss": 1.254, + "theoretical_loss": 3.6503188078890614, + "tokens_seen": 1082851328 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003393380140421264, + "loss": 1.3812, + "theoretical_loss": 3.650298253823546, + "tokens_seen": 1082916864 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003393279839518556, + "loss": 1.3443, + "theoretical_loss": 3.6502777013501504, + "tokens_seen": 1082982400 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003393179538615848, + "loss": 1.3554, + "theoretical_loss": 3.6502571504686543, + "tokens_seen": 1083047936 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033930792377131396, + "loss": 1.3212, + "theoretical_loss": 3.6502366011788387, + "tokens_seen": 1083113472 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033929789368104314, + "loss": 1.4117, + "theoretical_loss": 3.650216053480484, + "tokens_seen": 1083179008 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033928786359077237, + "loss": 1.3222, + "theoretical_loss": 3.650195507373371, + "tokens_seen": 1083244544 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003392778335005015, + "loss": 1.346, + "theoretical_loss": 3.650174962857279, + "tokens_seen": 1083310080 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033926780341023073, + "loss": 1.2814, + "theoretical_loss": 3.6501544199319897, + "tokens_seen": 1083375616 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033925777331995986, + "loss": 1.3247, + "theoretical_loss": 3.650133878597283, + "tokens_seen": 1083441152 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003392477432296891, + "loss": 1.2574, + "theoretical_loss": 3.6501133388529405, + "tokens_seen": 1083506688 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003392377131394183, + "loss": 1.4295, + "theoretical_loss": 3.650092800698742, + "tokens_seen": 1083572224 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033922768304914746, + "loss": 1.3146, + "theoretical_loss": 3.6500722641344683, + "tokens_seen": 1083637760 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033921765295887664, + "loss": 1.2701, + "theoretical_loss": 3.6500517291599004, + "tokens_seen": 1083703296 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003392076228686058, + "loss": 1.3538, + "theoretical_loss": 3.6500311957748193, + "tokens_seen": 1083768832 + }, + { + "epoch": 0.33, + "learning_rate": 0.000339197592778335, + "loss": 1.3163, + "theoretical_loss": 3.6500106639790055, + "tokens_seen": 1083834368 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033918756268806424, + "loss": 1.3736, + "theoretical_loss": 3.6499901337722402, + "tokens_seen": 1083899904 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033917753259779336, + "loss": 1.4189, + "theoretical_loss": 3.649969605154304, + "tokens_seen": 1083965440 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003391675025075226, + "loss": 1.3893, + "theoretical_loss": 3.649949078124978, + "tokens_seen": 1084030976 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003391574724172518, + "loss": 1.3976, + "theoretical_loss": 3.6499285526840435, + "tokens_seen": 1084096512 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033914744232698096, + "loss": 1.3279, + "theoretical_loss": 3.6499080288312813, + "tokens_seen": 1084162048 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033913741223671014, + "loss": 1.3421, + "theoretical_loss": 3.649887506566473, + "tokens_seen": 1084227584 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003391273821464393, + "loss": 1.4306, + "theoretical_loss": 3.6498669858893993, + "tokens_seen": 1084293120 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003391173520561685, + "loss": 1.2678, + "theoretical_loss": 3.6498464667998416, + "tokens_seen": 1084358656 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033910732196589774, + "loss": 1.3986, + "theoretical_loss": 3.6498259492975804, + "tokens_seen": 1084424192 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033909729187562687, + "loss": 1.3301, + "theoretical_loss": 3.649805433382398, + "tokens_seen": 1084489728 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003390872617853561, + "loss": 1.29, + "theoretical_loss": 3.649784919054076, + "tokens_seen": 1084555264 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 364115, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.0788460969924927, + "objective/train/theoretical_loss": 3.649764406312394, + "objective/train/tokens_used": 1084620800, + "theoretical_loss": 3.649764406312394, + "tokens_seen": 1084620800 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033907723169508523, + "loss": 1.2054, + "theoretical_loss": 3.649764406312394, + "tokens_seen": 1084620800 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033906720160481446, + "loss": 1.4295, + "theoretical_loss": 3.649743895157135, + "tokens_seen": 1084686336 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033905717151454365, + "loss": 1.3224, + "theoretical_loss": 3.6497233855880804, + "tokens_seen": 1084751872 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033904714142427283, + "loss": 1.3342, + "theoretical_loss": 3.6497028776050113, + "tokens_seen": 1084817408 + }, + { + "epoch": 0.33, + "learning_rate": 0.000339037111334002, + "loss": 1.3333, + "theoretical_loss": 3.6496823712077093, + "tokens_seen": 1084882944 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033902708124373124, + "loss": 1.3621, + "theoretical_loss": 3.649661866395957, + "tokens_seen": 1084948480 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033901705115346037, + "loss": 1.3091, + "theoretical_loss": 3.6496413631695335, + "tokens_seen": 1085014016 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003390070210631896, + "loss": 1.5112, + "theoretical_loss": 3.6496208615282226, + "tokens_seen": 1085079552 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033899699097291873, + "loss": 1.3109, + "theoretical_loss": 3.6496003614718058, + "tokens_seen": 1085145088 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033898696088264797, + "loss": 1.3635, + "theoretical_loss": 3.6495798630000644, + "tokens_seen": 1085210624 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033897693079237715, + "loss": 1.3306, + "theoretical_loss": 3.64955936611278, + "tokens_seen": 1085276160 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033896690070210633, + "loss": 1.295, + "theoretical_loss": 3.649538870809735, + "tokens_seen": 1085341696 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003389568706118355, + "loss": 1.3243, + "theoretical_loss": 3.649518377090712, + "tokens_seen": 1085407232 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003389468405215647, + "loss": 1.4427, + "theoretical_loss": 3.6494978849554913, + "tokens_seen": 1085472768 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003389368104312939, + "loss": 1.3175, + "theoretical_loss": 3.6494773944038554, + "tokens_seen": 1085538304 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003389267803410231, + "loss": 1.3595, + "theoretical_loss": 3.6494569054355877, + "tokens_seen": 1085603840 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033891675025075224, + "loss": 1.4239, + "theoretical_loss": 3.649436418050468, + "tokens_seen": 1085669376 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033890672016048147, + "loss": 1.3001, + "theoretical_loss": 3.64941593224828, + "tokens_seen": 1085734912 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003388966900702106, + "loss": 1.432, + "theoretical_loss": 3.6493954480288058, + "tokens_seen": 1085800448 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033888665997993983, + "loss": 1.3463, + "theoretical_loss": 3.649374965391827, + "tokens_seen": 1085865984 + }, + { + "epoch": 0.33, + "learning_rate": 0.000338876629889669, + "loss": 1.411, + "theoretical_loss": 3.649354484337126, + "tokens_seen": 1085931520 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003388665997993982, + "loss": 1.3272, + "theoretical_loss": 3.6493340048644853, + "tokens_seen": 1085997056 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003388565697091274, + "loss": 1.3883, + "theoretical_loss": 3.649313526973687, + "tokens_seen": 1086062592 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003388465396188566, + "loss": 1.4488, + "theoretical_loss": 3.649293050664514, + "tokens_seen": 1086128128 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033883650952858574, + "loss": 1.3127, + "theoretical_loss": 3.649272575936748, + "tokens_seen": 1086193664 + }, + { + "epoch": 0.33, + "learning_rate": 0.000338826479438315, + "loss": 1.4024, + "theoretical_loss": 3.6492521027901716, + "tokens_seen": 1086259200 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003388164493480441, + "loss": 1.3307, + "theoretical_loss": 3.649231631224568, + "tokens_seen": 1086324736 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033880641925777334, + "loss": 1.3024, + "theoretical_loss": 3.649211161239719, + "tokens_seen": 1086390272 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003387963891675025, + "loss": 1.237, + "theoretical_loss": 3.6491906928354076, + "tokens_seen": 1086455808 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003387863590772317, + "loss": 1.3374, + "theoretical_loss": 3.649170226011416, + "tokens_seen": 1086521344 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003387763289869609, + "loss": 1.2028, + "theoretical_loss": 3.6491497607675276, + "tokens_seen": 1086586880 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033876629889669006, + "loss": 1.3345, + "theoretical_loss": 3.6491292971035243, + "tokens_seen": 1086652416 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033875626880641924, + "loss": 1.3332, + "theoretical_loss": 3.649108835019189, + "tokens_seen": 1086717952 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003387462387161485, + "loss": 1.3772, + "theoretical_loss": 3.6490883745143052, + "tokens_seen": 1086783488 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003387362086258776, + "loss": 1.3524, + "theoretical_loss": 3.649067915588655, + "tokens_seen": 1086849024 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033872617853560684, + "loss": 1.3651, + "theoretical_loss": 3.6490474582420216, + "tokens_seen": 1086914560 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033871614844533597, + "loss": 1.295, + "theoretical_loss": 3.649027002474188, + "tokens_seen": 1086980096 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003387061183550652, + "loss": 1.3969, + "theoretical_loss": 3.6490065482849374, + "tokens_seen": 1087045632 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003386960882647944, + "loss": 1.3092, + "theoretical_loss": 3.648986095674052, + "tokens_seen": 1087111168 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033868605817452356, + "loss": 1.4182, + "theoretical_loss": 3.6489656446413155, + "tokens_seen": 1087176704 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033867602808425275, + "loss": 1.3077, + "theoretical_loss": 3.648945195186511, + "tokens_seen": 1087242240 + }, + { + "epoch": 0.33, + "learning_rate": 0.000338665997993982, + "loss": 1.3154, + "theoretical_loss": 3.6489247473094215, + "tokens_seen": 1087307776 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003386559679037111, + "loss": 1.2147, + "theoretical_loss": 3.6489043010098303, + "tokens_seen": 1087373312 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033864593781344034, + "loss": 1.2959, + "theoretical_loss": 3.6488838562875205, + "tokens_seen": 1087438848 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033863590772316947, + "loss": 1.3175, + "theoretical_loss": 3.6488634131422755, + "tokens_seen": 1087504384 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003386258776328987, + "loss": 1.2477, + "theoretical_loss": 3.6488429715738784, + "tokens_seen": 1087569920 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003386158475426279, + "loss": 1.4, + "theoretical_loss": 3.6488225315821126, + "tokens_seen": 1087635456 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033860581745235707, + "loss": 1.3497, + "theoretical_loss": 3.648802093166762, + "tokens_seen": 1087700992 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033859578736208625, + "loss": 1.2913, + "theoretical_loss": 3.6487816563276096, + "tokens_seen": 1087766528 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033858575727181543, + "loss": 1.3306, + "theoretical_loss": 3.648761221064439, + "tokens_seen": 1087832064 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 365331, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1914818286895752, + "objective/train/theoretical_loss": 3.6487407873770334, + "objective/train/tokens_used": 1087897600, + "theoretical_loss": 3.6487407873770334, + "tokens_seen": 1087897600 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033857572718154467, + "loss": 1.3453, + "theoretical_loss": 3.6487407873770334, + "tokens_seen": 1087897600 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033856569709127385, + "loss": 1.4294, + "theoretical_loss": 3.6487203552651772, + "tokens_seen": 1087963136 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033855566700100303, + "loss": 1.3018, + "theoretical_loss": 3.648699924728653, + "tokens_seen": 1088028672 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003385456369107322, + "loss": 1.3702, + "theoretical_loss": 3.6486794957672455, + "tokens_seen": 1088094208 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033853560682046144, + "loss": 1.3076, + "theoretical_loss": 3.648659068380737, + "tokens_seen": 1088159744 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033852557673019057, + "loss": 1.3568, + "theoretical_loss": 3.648638642568913, + "tokens_seen": 1088225280 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003385155466399198, + "loss": 1.4429, + "theoretical_loss": 3.6486182183315563, + "tokens_seen": 1088290816 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033850551654964893, + "loss": 1.3084, + "theoretical_loss": 3.6485977956684503, + "tokens_seen": 1088356352 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033849548645937817, + "loss": 1.2969, + "theoretical_loss": 3.64857737457938, + "tokens_seen": 1088421888 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033848545636910735, + "loss": 1.2792, + "theoretical_loss": 3.6485569550641284, + "tokens_seen": 1088487424 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033847542627883653, + "loss": 1.2363, + "theoretical_loss": 3.64853653712248, + "tokens_seen": 1088552960 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003384653961885657, + "loss": 1.33, + "theoretical_loss": 3.6485161207542185, + "tokens_seen": 1088618496 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003384553660982949, + "loss": 1.3985, + "theoretical_loss": 3.648495705959128, + "tokens_seen": 1088684032 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003384453360080241, + "loss": 1.3374, + "theoretical_loss": 3.648475292736993, + "tokens_seen": 1088749568 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003384353059177533, + "loss": 1.407, + "theoretical_loss": 3.6484548810875967, + "tokens_seen": 1088815104 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033842527582748244, + "loss": 1.3789, + "theoretical_loss": 3.6484344710107246, + "tokens_seen": 1088880640 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033841524573721167, + "loss": 1.3146, + "theoretical_loss": 3.6484140625061596, + "tokens_seen": 1088946176 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003384052156469408, + "loss": 1.3471, + "theoretical_loss": 3.648393655573687, + "tokens_seen": 1089011712 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033839518555667003, + "loss": 1.3279, + "theoretical_loss": 3.6483732502130906, + "tokens_seen": 1089077248 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003383851554663992, + "loss": 1.3585, + "theoretical_loss": 3.6483528464241544, + "tokens_seen": 1089142784 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003383751253761284, + "loss": 1.3409, + "theoretical_loss": 3.6483324442066634, + "tokens_seen": 1089208320 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003383650952858576, + "loss": 1.5371, + "theoretical_loss": 3.6483120435604017, + "tokens_seen": 1089273856 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003383550651955868, + "loss": 1.4602, + "theoretical_loss": 3.648291644485154, + "tokens_seen": 1089339392 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033834503510531594, + "loss": 1.4545, + "theoretical_loss": 3.6482712469807046, + "tokens_seen": 1089404928 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003383350050150452, + "loss": 1.423, + "theoretical_loss": 3.6482508510468383, + "tokens_seen": 1089470464 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003383249749247743, + "loss": 1.3881, + "theoretical_loss": 3.6482304566833395, + "tokens_seen": 1089536000 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033831494483450354, + "loss": 1.3066, + "theoretical_loss": 3.6482100638899926, + "tokens_seen": 1089601536 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003383049147442327, + "loss": 1.2932, + "theoretical_loss": 3.6481896726665832, + "tokens_seen": 1089667072 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003382948846539619, + "loss": 1.4362, + "theoretical_loss": 3.648169283012895, + "tokens_seen": 1089732608 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003382848545636911, + "loss": 1.3474, + "theoretical_loss": 3.6481488949287133, + "tokens_seen": 1089798144 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033827482447342026, + "loss": 1.3603, + "theoretical_loss": 3.648128508413823, + "tokens_seen": 1089863680 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033826479438314944, + "loss": 1.2623, + "theoretical_loss": 3.648108123468009, + "tokens_seen": 1089929216 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003382547642928787, + "loss": 1.3338, + "theoretical_loss": 3.648087740091055, + "tokens_seen": 1089994752 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003382447342026078, + "loss": 1.4751, + "theoretical_loss": 3.6480673582827476, + "tokens_seen": 1090060288 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033823470411233704, + "loss": 1.502, + "theoretical_loss": 3.6480469780428706, + "tokens_seen": 1090125824 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033822467402206617, + "loss": 1.328, + "theoretical_loss": 3.64802659937121, + "tokens_seen": 1090191360 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003382146439317954, + "loss": 1.5032, + "theoretical_loss": 3.6480062222675502, + "tokens_seen": 1090256896 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003382046138415246, + "loss": 1.3035, + "theoretical_loss": 3.647985846731676, + "tokens_seen": 1090322432 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033819458375125376, + "loss": 1.373, + "theoretical_loss": 3.647965472763374, + "tokens_seen": 1090387968 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033818455366098295, + "loss": 1.377, + "theoretical_loss": 3.6479451003624277, + "tokens_seen": 1090453504 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003381745235707122, + "loss": 1.3389, + "theoretical_loss": 3.647924729528624, + "tokens_seen": 1090519040 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003381644934804413, + "loss": 1.4071, + "theoretical_loss": 3.6479043602617462, + "tokens_seen": 1090584576 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033815446339017054, + "loss": 1.3208, + "theoretical_loss": 3.6478839925615807, + "tokens_seen": 1090650112 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033814443329989967, + "loss": 1.3653, + "theoretical_loss": 3.647863626427913, + "tokens_seen": 1090715648 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003381344032096289, + "loss": 1.3278, + "theoretical_loss": 3.6478432618605288, + "tokens_seen": 1090781184 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003381243731193581, + "loss": 1.3936, + "theoretical_loss": 3.647822898859213, + "tokens_seen": 1090846720 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033811434302908727, + "loss": 1.4157, + "theoretical_loss": 3.647802537423751, + "tokens_seen": 1090912256 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033810431293881645, + "loss": 1.3704, + "theoretical_loss": 3.6477821775539283, + "tokens_seen": 1090977792 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033809428284854563, + "loss": 1.3479, + "theoretical_loss": 3.647761819249531, + "tokens_seen": 1091043328 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003380842527582748, + "loss": 1.3655, + "theoretical_loss": 3.647741462510344, + "tokens_seen": 1091108864 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 366232, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1822326183319092, + "objective/train/theoretical_loss": 3.647721107336154, + "objective/train/tokens_used": 1091174400, + "theoretical_loss": 3.647721107336154, + "tokens_seen": 1091174400 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033807422266800405, + "loss": 1.3251, + "theoretical_loss": 3.647721107336154, + "tokens_seen": 1091174400 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003380641925777332, + "loss": 1.2593, + "theoretical_loss": 3.647700753726746, + "tokens_seen": 1091239936 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003380541624874624, + "loss": 1.3434, + "theoretical_loss": 3.6476804016819053, + "tokens_seen": 1091305472 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033804413239719154, + "loss": 1.2814, + "theoretical_loss": 3.6476600512014183, + "tokens_seen": 1091371008 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033803410230692077, + "loss": 1.3291, + "theoretical_loss": 3.647639702285071, + "tokens_seen": 1091436544 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033802407221664995, + "loss": 1.2823, + "theoretical_loss": 3.647619354932649, + "tokens_seen": 1091502080 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033801404212637913, + "loss": 1.2914, + "theoretical_loss": 3.647599009143938, + "tokens_seen": 1091567616 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003380040120361083, + "loss": 1.3241, + "theoretical_loss": 3.6475786649187247, + "tokens_seen": 1091633152 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033799398194583755, + "loss": 1.2808, + "theoretical_loss": 3.6475583222567938, + "tokens_seen": 1091698688 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003379839518555667, + "loss": 1.37, + "theoretical_loss": 3.6475379811579325, + "tokens_seen": 1091764224 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003379739217652959, + "loss": 1.3151, + "theoretical_loss": 3.647517641621927, + "tokens_seen": 1091829760 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033796389167502504, + "loss": 1.3262, + "theoretical_loss": 3.647497303648562, + "tokens_seen": 1091895296 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003379538615847543, + "loss": 1.3846, + "theoretical_loss": 3.647476967237625, + "tokens_seen": 1091960832 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033794383149448346, + "loss": 1.3217, + "theoretical_loss": 3.6474566323889017, + "tokens_seen": 1092026368 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033793380140421264, + "loss": 1.3079, + "theoretical_loss": 3.6474362991021785, + "tokens_seen": 1092091904 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003379237713139418, + "loss": 1.4262, + "theoretical_loss": 3.647415967377242, + "tokens_seen": 1092157440 + }, + { + "epoch": 0.33, + "learning_rate": 0.000337913741223671, + "loss": 1.2443, + "theoretical_loss": 3.6473956372138776, + "tokens_seen": 1092222976 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003379037111334002, + "loss": 1.422, + "theoretical_loss": 3.6473753086118723, + "tokens_seen": 1092288512 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003378936810431294, + "loss": 1.3736, + "theoretical_loss": 3.647354981571013, + "tokens_seen": 1092354048 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033788365095285854, + "loss": 1.3634, + "theoretical_loss": 3.6473346560910853, + "tokens_seen": 1092419584 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003378736208625878, + "loss": 1.3428, + "theoretical_loss": 3.6473143321718755, + "tokens_seen": 1092485120 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003378635907723169, + "loss": 1.3472, + "theoretical_loss": 3.647294009813171, + "tokens_seen": 1092550656 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033785356068204614, + "loss": 1.3438, + "theoretical_loss": 3.6472736890147583, + "tokens_seen": 1092616192 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003378435305917753, + "loss": 1.3409, + "theoretical_loss": 3.647253369776424, + "tokens_seen": 1092681728 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003378335005015045, + "loss": 1.3618, + "theoretical_loss": 3.6472330520979543, + "tokens_seen": 1092747264 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033782347041123374, + "loss": 1.3414, + "theoretical_loss": 3.647212735979136, + "tokens_seen": 1092812800 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003378134403209629, + "loss": 1.2602, + "theoretical_loss": 3.647192421419756, + "tokens_seen": 1092878336 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003378034102306921, + "loss": 1.2605, + "theoretical_loss": 3.6471721084196016, + "tokens_seen": 1092943872 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003377933801404213, + "loss": 1.2837, + "theoretical_loss": 3.6471517969784584, + "tokens_seen": 1093009408 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033778335005015046, + "loss": 1.2681, + "theoretical_loss": 3.6471314870961145, + "tokens_seen": 1093074944 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033777331995987964, + "loss": 1.3522, + "theoretical_loss": 3.6471111787723567, + "tokens_seen": 1093140480 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003377632898696089, + "loss": 1.459, + "theoretical_loss": 3.647090872006971, + "tokens_seen": 1093206016 + }, + { + "epoch": 0.33, + "learning_rate": 0.000337753259779338, + "loss": 1.2674, + "theoretical_loss": 3.647070566799745, + "tokens_seen": 1093271552 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033774322968906724, + "loss": 1.1771, + "theoretical_loss": 3.6470502631504664, + "tokens_seen": 1093337088 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033773319959879637, + "loss": 1.4538, + "theoretical_loss": 3.647029961058921, + "tokens_seen": 1093402624 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003377231695085256, + "loss": 1.3831, + "theoretical_loss": 3.6470096605248976, + "tokens_seen": 1093468160 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003377131394182548, + "loss": 1.2472, + "theoretical_loss": 3.646989361548181, + "tokens_seen": 1093533696 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033770310932798396, + "loss": 1.3709, + "theoretical_loss": 3.646969064128561, + "tokens_seen": 1093599232 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033769307923771315, + "loss": 1.356, + "theoretical_loss": 3.646948768265823, + "tokens_seen": 1093664768 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003376830491474424, + "loss": 1.2799, + "theoretical_loss": 3.646928473959755, + "tokens_seen": 1093730304 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003376730190571715, + "loss": 1.3075, + "theoretical_loss": 3.6469081812101445, + "tokens_seen": 1093795840 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033766298896690074, + "loss": 1.369, + "theoretical_loss": 3.6468878900167785, + "tokens_seen": 1093861376 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033765295887662987, + "loss": 1.2589, + "theoretical_loss": 3.6468676003794447, + "tokens_seen": 1093926912 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003376429287863591, + "loss": 1.4452, + "theoretical_loss": 3.6468473122979304, + "tokens_seen": 1093992448 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003376328986960883, + "loss": 1.4076, + "theoretical_loss": 3.6468270257720232, + "tokens_seen": 1094057984 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033762286860581747, + "loss": 1.3884, + "theoretical_loss": 3.6468067408015106, + "tokens_seen": 1094123520 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033761283851554665, + "loss": 1.3598, + "theoretical_loss": 3.64678645738618, + "tokens_seen": 1094189056 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033760280842527583, + "loss": 1.3916, + "theoretical_loss": 3.6467661755258196, + "tokens_seen": 1094254592 + }, + { + "epoch": 0.33, + "learning_rate": 0.000337592778335005, + "loss": 1.3369, + "theoretical_loss": 3.6467458952202163, + "tokens_seen": 1094320128 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033758274824473425, + "loss": 1.395, + "theoretical_loss": 3.6467256164691584, + "tokens_seen": 1094385664 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 367505, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.269169569015503, + "objective/train/theoretical_loss": 3.646705339272434, + "objective/train/tokens_used": 1094451200, + "theoretical_loss": 3.646705339272434, + "tokens_seen": 1094451200 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003375727181544634, + "loss": 1.342, + "theoretical_loss": 3.646705339272434, + "tokens_seen": 1094451200 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003375626880641926, + "loss": 1.3968, + "theoretical_loss": 3.64668506362983, + "tokens_seen": 1094516736 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033755265797392174, + "loss": 1.3279, + "theoretical_loss": 3.646664789541134, + "tokens_seen": 1094582272 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033754262788365097, + "loss": 1.3461, + "theoretical_loss": 3.6466445170061355, + "tokens_seen": 1094647808 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033753259779338015, + "loss": 1.2483, + "theoretical_loss": 3.646624246024621, + "tokens_seen": 1094713344 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033752256770310933, + "loss": 1.4481, + "theoretical_loss": 3.6466039765963787, + "tokens_seen": 1094778880 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003375125376128385, + "loss": 1.3266, + "theoretical_loss": 3.646583708721197, + "tokens_seen": 1094844416 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033750250752256775, + "loss": 1.2449, + "theoretical_loss": 3.646563442398864, + "tokens_seen": 1094909952 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003374924774322969, + "loss": 1.2791, + "theoretical_loss": 3.6465431776291677, + "tokens_seen": 1094975488 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003374824473420261, + "loss": 1.3747, + "theoretical_loss": 3.6465229144118956, + "tokens_seen": 1095041024 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033747241725175524, + "loss": 1.3258, + "theoretical_loss": 3.6465026527468365, + "tokens_seen": 1095106560 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003374623871614845, + "loss": 1.3091, + "theoretical_loss": 3.6464823926337786, + "tokens_seen": 1095172096 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033745235707121366, + "loss": 1.2901, + "theoretical_loss": 3.64646213407251, + "tokens_seen": 1095237632 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033744232698094284, + "loss": 1.3107, + "theoretical_loss": 3.6464418770628186, + "tokens_seen": 1095303168 + }, + { + "epoch": 0.33, + "learning_rate": 0.000337432296890672, + "loss": 1.3125, + "theoretical_loss": 3.646421621604494, + "tokens_seen": 1095368704 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003374222668004012, + "loss": 1.282, + "theoretical_loss": 3.6464013676973233, + "tokens_seen": 1095434240 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003374122367101304, + "loss": 1.3033, + "theoretical_loss": 3.6463811153410948, + "tokens_seen": 1095499776 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003374022066198596, + "loss": 1.3599, + "theoretical_loss": 3.6463608645355983, + "tokens_seen": 1095565312 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033739217652958874, + "loss": 1.2967, + "theoretical_loss": 3.6463406152806215, + "tokens_seen": 1095630848 + }, + { + "epoch": 0.33, + "learning_rate": 0.000337382146439318, + "loss": 1.2935, + "theoretical_loss": 3.646320367575953, + "tokens_seen": 1095696384 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003373721163490471, + "loss": 1.2458, + "theoretical_loss": 3.6463001214213815, + "tokens_seen": 1095761920 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033736208625877634, + "loss": 1.379, + "theoretical_loss": 3.646279876816695, + "tokens_seen": 1095827456 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003373520561685055, + "loss": 1.4283, + "theoretical_loss": 3.646259633761683, + "tokens_seen": 1095892992 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003373420260782347, + "loss": 1.3282, + "theoretical_loss": 3.646239392256134, + "tokens_seen": 1095958528 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003373319959879639, + "loss": 1.2974, + "theoretical_loss": 3.646219152299836, + "tokens_seen": 1096024064 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003373219658976931, + "loss": 1.2648, + "theoretical_loss": 3.6461989138925794, + "tokens_seen": 1096089600 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033731193580742225, + "loss": 1.4035, + "theoretical_loss": 3.6461786770341518, + "tokens_seen": 1096155136 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003373019057171515, + "loss": 1.3884, + "theoretical_loss": 3.6461584417243422, + "tokens_seen": 1096220672 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003372918756268806, + "loss": 1.4644, + "theoretical_loss": 3.64613820796294, + "tokens_seen": 1096286208 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033728184553660984, + "loss": 1.2995, + "theoretical_loss": 3.6461179757497337, + "tokens_seen": 1096351744 + }, + { + "epoch": 0.33, + "learning_rate": 0.000337271815446339, + "loss": 1.4088, + "theoretical_loss": 3.6460977450845125, + "tokens_seen": 1096417280 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003372617853560682, + "loss": 1.2837, + "theoretical_loss": 3.646077515967065, + "tokens_seen": 1096482816 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003372517552657974, + "loss": 1.3239, + "theoretical_loss": 3.6460572883971816, + "tokens_seen": 1096548352 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033724172517552657, + "loss": 1.3714, + "theoretical_loss": 3.6460370623746496, + "tokens_seen": 1096613888 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033723169508525575, + "loss": 1.2169, + "theoretical_loss": 3.6460168378992597, + "tokens_seen": 1096679424 + }, + { + "epoch": 0.33, + "learning_rate": 0.000337221664994985, + "loss": 1.3489, + "theoretical_loss": 3.6459966149708, + "tokens_seen": 1096744960 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003372116349047141, + "loss": 1.2591, + "theoretical_loss": 3.6459763935890606, + "tokens_seen": 1096810496 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033720160481444335, + "loss": 1.2747, + "theoretical_loss": 3.6459561737538304, + "tokens_seen": 1096876032 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003371915747241725, + "loss": 1.3723, + "theoretical_loss": 3.645935955464899, + "tokens_seen": 1096941568 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003371815446339017, + "loss": 1.3423, + "theoretical_loss": 3.645915738722055, + "tokens_seen": 1097007104 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003371715145436309, + "loss": 1.4266, + "theoretical_loss": 3.645895523525089, + "tokens_seen": 1097072640 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033716148445336007, + "loss": 1.4015, + "theoretical_loss": 3.6458753098737895, + "tokens_seen": 1097138176 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033715145436308925, + "loss": 1.431, + "theoretical_loss": 3.6458550977679467, + "tokens_seen": 1097203712 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003371414242728185, + "loss": 1.419, + "theoretical_loss": 3.645834887207349, + "tokens_seen": 1097269248 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003371313941825476, + "loss": 1.2842, + "theoretical_loss": 3.6458146781917877, + "tokens_seen": 1097334784 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033712136409227685, + "loss": 1.4724, + "theoretical_loss": 3.645794470721051, + "tokens_seen": 1097400320 + }, + { + "epoch": 0.33, + "learning_rate": 0.000337111334002006, + "loss": 1.3564, + "theoretical_loss": 3.6457742647949294, + "tokens_seen": 1097465856 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003371013039117352, + "loss": 1.3912, + "theoretical_loss": 3.6457540604132115, + "tokens_seen": 1097531392 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003370912738214644, + "loss": 1.3674, + "theoretical_loss": 3.645733857575688, + "tokens_seen": 1097596928 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003370812437311936, + "loss": 1.2526, + "theoretical_loss": 3.6457136562821493, + "tokens_seen": 1097662464 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 368349, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3915784358978271, + "objective/train/theoretical_loss": 3.645693456532384, + "objective/train/tokens_used": 1097728000, + "theoretical_loss": 3.645693456532384, + "tokens_seen": 1097728000 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003370712136409228, + "loss": 1.3393, + "theoretical_loss": 3.645693456532384, + "tokens_seen": 1097728000 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033706118355065194, + "loss": 1.3725, + "theoretical_loss": 3.645673258326182, + "tokens_seen": 1097793536 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033705115346038117, + "loss": 1.3157, + "theoretical_loss": 3.6456530616633334, + "tokens_seen": 1097859072 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033704112337011035, + "loss": 1.2965, + "theoretical_loss": 3.645632866543629, + "tokens_seen": 1097924608 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033703109327983953, + "loss": 1.1691, + "theoretical_loss": 3.645612672966858, + "tokens_seen": 1097990144 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003370210631895687, + "loss": 1.3709, + "theoretical_loss": 3.6455924809328106, + "tokens_seen": 1098055680 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033701103309929795, + "loss": 1.3464, + "theoretical_loss": 3.645572290441277, + "tokens_seen": 1098121216 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003370010030090271, + "loss": 1.3502, + "theoretical_loss": 3.645552101492047, + "tokens_seen": 1098186752 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003369909729187563, + "loss": 1.2758, + "theoretical_loss": 3.645531914084911, + "tokens_seen": 1098252288 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033698094282848544, + "loss": 1.4203, + "theoretical_loss": 3.6455117282196587, + "tokens_seen": 1098317824 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003369709127382147, + "loss": 1.4058, + "theoretical_loss": 3.6454915438960813, + "tokens_seen": 1098383360 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033696088264794386, + "loss": 1.3049, + "theoretical_loss": 3.6454713611139686, + "tokens_seen": 1098448896 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033695085255767304, + "loss": 1.3669, + "theoretical_loss": 3.6454511798731106, + "tokens_seen": 1098514432 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003369408224674022, + "loss": 1.2162, + "theoretical_loss": 3.645431000173298, + "tokens_seen": 1098579968 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003369307923771314, + "loss": 1.3168, + "theoretical_loss": 3.6454108220143215, + "tokens_seen": 1098645504 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003369207622868606, + "loss": 1.3653, + "theoretical_loss": 3.6453906453959704, + "tokens_seen": 1098711040 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003369107321965898, + "loss": 1.338, + "theoretical_loss": 3.6453704703180367, + "tokens_seen": 1098776576 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033690070210631894, + "loss": 1.4192, + "theoretical_loss": 3.6453502967803098, + "tokens_seen": 1098842112 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003368906720160482, + "loss": 1.327, + "theoretical_loss": 3.645330124782581, + "tokens_seen": 1098907648 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003368806419257773, + "loss": 1.2804, + "theoretical_loss": 3.6453099543246403, + "tokens_seen": 1098973184 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033687061183550654, + "loss": 1.3116, + "theoretical_loss": 3.6452897854062787, + "tokens_seen": 1099038720 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003368605817452357, + "loss": 1.2474, + "theoretical_loss": 3.645269618027287, + "tokens_seen": 1099104256 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003368505516549649, + "loss": 1.298, + "theoretical_loss": 3.6452494521874557, + "tokens_seen": 1099169792 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003368405215646941, + "loss": 1.359, + "theoretical_loss": 3.6452292878865755, + "tokens_seen": 1099235328 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003368304914744233, + "loss": 1.2934, + "theoretical_loss": 3.6452091251244374, + "tokens_seen": 1099300864 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033682046138415245, + "loss": 1.2891, + "theoretical_loss": 3.645188963900832, + "tokens_seen": 1099366400 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003368104312938817, + "loss": 1.3127, + "theoretical_loss": 3.645168804215551, + "tokens_seen": 1099431936 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003368004012036108, + "loss": 1.3282, + "theoretical_loss": 3.6451486460683844, + "tokens_seen": 1099497472 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033679037111334004, + "loss": 1.3107, + "theoretical_loss": 3.645128489459123, + "tokens_seen": 1099563008 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003367803410230692, + "loss": 1.4262, + "theoretical_loss": 3.645108334387559, + "tokens_seen": 1099628544 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003367703109327984, + "loss": 1.3783, + "theoretical_loss": 3.6450881808534827, + "tokens_seen": 1099694080 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003367602808425276, + "loss": 1.2969, + "theoretical_loss": 3.6450680288566852, + "tokens_seen": 1099759616 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033675025075225677, + "loss": 1.3017, + "theoretical_loss": 3.6450478783969573, + "tokens_seen": 1099825152 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033674022066198595, + "loss": 1.3134, + "theoretical_loss": 3.6450277294740916, + "tokens_seen": 1099890688 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003367301905717152, + "loss": 1.2589, + "theoretical_loss": 3.645007582087877, + "tokens_seen": 1099956224 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003367201604814443, + "loss": 1.4204, + "theoretical_loss": 3.6449874362381074, + "tokens_seen": 1100021760 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033671013039117355, + "loss": 1.3534, + "theoretical_loss": 3.644967291924572, + "tokens_seen": 1100087296 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033670010030090273, + "loss": 1.3426, + "theoretical_loss": 3.6449471491470633, + "tokens_seen": 1100152832 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003366900702106319, + "loss": 1.31, + "theoretical_loss": 3.6449270079053724, + "tokens_seen": 1100218368 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003366800401203611, + "loss": 1.4035, + "theoretical_loss": 3.64490686819929, + "tokens_seen": 1100283904 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033667001003009027, + "loss": 1.2565, + "theoretical_loss": 3.6448867300286087, + "tokens_seen": 1100349440 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033665997993981945, + "loss": 1.4098, + "theoretical_loss": 3.64486659339312, + "tokens_seen": 1100414976 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003366499498495487, + "loss": 1.3802, + "theoretical_loss": 3.6448464582926143, + "tokens_seen": 1100480512 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003366399197592778, + "loss": 1.435, + "theoretical_loss": 3.6448263247268846, + "tokens_seen": 1100546048 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033662988966900705, + "loss": 1.3585, + "theoretical_loss": 3.6448061926957207, + "tokens_seen": 1100611584 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003366198595787362, + "loss": 1.2983, + "theoretical_loss": 3.644786062198916, + "tokens_seen": 1100677120 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003366098294884654, + "loss": 1.2918, + "theoretical_loss": 3.6447659332362616, + "tokens_seen": 1100742656 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003365997993981946, + "loss": 1.314, + "theoretical_loss": 3.6447458058075495, + "tokens_seen": 1100808192 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003365897693079238, + "loss": 1.3942, + "theoretical_loss": 3.644725679912571, + "tokens_seen": 1100873728 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033657973921765295, + "loss": 1.2032, + "theoretical_loss": 3.644705555551118, + "tokens_seen": 1100939264 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 369550, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2648725509643555, + "objective/train/theoretical_loss": 3.6446854327229827, + "objective/train/tokens_used": 1101004800, + "theoretical_loss": 3.6446854327229827, + "tokens_seen": 1101004800 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033656970912738214, + "loss": 1.3494, + "theoretical_loss": 3.6446854327229827, + "tokens_seen": 1101004800 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003365596790371113, + "loss": 1.4462, + "theoretical_loss": 3.6446653114279566, + "tokens_seen": 1101070336 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033654964894684055, + "loss": 1.3726, + "theoretical_loss": 3.6446451916658322, + "tokens_seen": 1101135872 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003365396188565697, + "loss": 1.3305, + "theoretical_loss": 3.6446250734364014, + "tokens_seen": 1101201408 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003365295887662989, + "loss": 1.3469, + "theoretical_loss": 3.644604956739456, + "tokens_seen": 1101266944 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003365195586760281, + "loss": 1.3062, + "theoretical_loss": 3.6445848415747877, + "tokens_seen": 1101332480 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003365095285857573, + "loss": 1.3241, + "theoretical_loss": 3.644564727942189, + "tokens_seen": 1101398016 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033649949849548646, + "loss": 1.3797, + "theoretical_loss": 3.6445446158414523, + "tokens_seen": 1101463552 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033648946840521564, + "loss": 1.3857, + "theoretical_loss": 3.6445245052723694, + "tokens_seen": 1101529088 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003364794383149448, + "loss": 1.4403, + "theoretical_loss": 3.644504396234733, + "tokens_seen": 1101594624 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033646940822467406, + "loss": 1.3428, + "theoretical_loss": 3.6444842887283353, + "tokens_seen": 1101660160 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003364593781344032, + "loss": 1.3953, + "theoretical_loss": 3.644464182752968, + "tokens_seen": 1101725696 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003364493480441324, + "loss": 1.3811, + "theoretical_loss": 3.644444078308424, + "tokens_seen": 1101791232 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033643931795386154, + "loss": 1.3613, + "theoretical_loss": 3.644423975394496, + "tokens_seen": 1101856768 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003364292878635908, + "loss": 1.3239, + "theoretical_loss": 3.6444038740109757, + "tokens_seen": 1101922304 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033641925777331996, + "loss": 1.4285, + "theoretical_loss": 3.6443837741576557, + "tokens_seen": 1101987840 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033640922768304914, + "loss": 1.4055, + "theoretical_loss": 3.644363675834329, + "tokens_seen": 1102053376 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003363991975927783, + "loss": 1.402, + "theoretical_loss": 3.644343579040788, + "tokens_seen": 1102118912 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003363891675025075, + "loss": 1.3852, + "theoretical_loss": 3.6443234837768252, + "tokens_seen": 1102184448 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003363791374122367, + "loss": 1.3129, + "theoretical_loss": 3.644303390042233, + "tokens_seen": 1102249984 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003363691073219659, + "loss": 1.3127, + "theoretical_loss": 3.6442832978368047, + "tokens_seen": 1102315520 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033635907723169505, + "loss": 1.3804, + "theoretical_loss": 3.6442632071603325, + "tokens_seen": 1102381056 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003363490471414243, + "loss": 1.3054, + "theoretical_loss": 3.6442431180126094, + "tokens_seen": 1102446592 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033633901705115346, + "loss": 1.3243, + "theoretical_loss": 3.6442230303934284, + "tokens_seen": 1102512128 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033632898696088265, + "loss": 1.4139, + "theoretical_loss": 3.6442029443025814, + "tokens_seen": 1102577664 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003363189568706119, + "loss": 1.3616, + "theoretical_loss": 3.6441828597398622, + "tokens_seen": 1102643200 + }, + { + "epoch": 0.33, + "learning_rate": 0.000336308926780341, + "loss": 1.2981, + "theoretical_loss": 3.6441627767050635, + "tokens_seen": 1102708736 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033629889669007024, + "loss": 1.3075, + "theoretical_loss": 3.6441426951979787, + "tokens_seen": 1102774272 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003362888665997994, + "loss": 1.4603, + "theoretical_loss": 3.6441226152184, + "tokens_seen": 1102839808 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003362788365095286, + "loss": 1.3099, + "theoretical_loss": 3.6441025367661206, + "tokens_seen": 1102905344 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003362688064192578, + "loss": 1.3908, + "theoretical_loss": 3.6440824598409347, + "tokens_seen": 1102970880 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033625877632898697, + "loss": 1.3, + "theoretical_loss": 3.644062384442634, + "tokens_seen": 1103036416 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033624874623871615, + "loss": 1.4188, + "theoretical_loss": 3.6440423105710122, + "tokens_seen": 1103101952 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003362387161484454, + "loss": 1.3402, + "theoretical_loss": 3.6440222382258622, + "tokens_seen": 1103167488 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003362286860581745, + "loss": 1.3327, + "theoretical_loss": 3.644002167406978, + "tokens_seen": 1103233024 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033621865596790375, + "loss": 1.3599, + "theoretical_loss": 3.643982098114152, + "tokens_seen": 1103298560 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033620862587763293, + "loss": 1.4816, + "theoretical_loss": 3.6439620303471782, + "tokens_seen": 1103364096 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003361985957873621, + "loss": 1.2714, + "theoretical_loss": 3.64394196410585, + "tokens_seen": 1103429632 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003361885656970913, + "loss": 1.3752, + "theoretical_loss": 3.6439218993899605, + "tokens_seen": 1103495168 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033617853560682047, + "loss": 1.398, + "theoretical_loss": 3.643901836199303, + "tokens_seen": 1103560704 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033616850551654965, + "loss": 1.3436, + "theoretical_loss": 3.643881774533672, + "tokens_seen": 1103626240 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003361584754262789, + "loss": 1.3187, + "theoretical_loss": 3.6438617143928598, + "tokens_seen": 1103691776 + }, + { + "epoch": 0.33, + "learning_rate": 0.000336148445336008, + "loss": 1.3012, + "theoretical_loss": 3.6438416557766597, + "tokens_seen": 1103757312 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033613841524573725, + "loss": 1.2682, + "theoretical_loss": 3.643821598684867, + "tokens_seen": 1103822848 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003361283851554664, + "loss": 1.3539, + "theoretical_loss": 3.6438015431172737, + "tokens_seen": 1103888384 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003361183550651956, + "loss": 1.4341, + "theoretical_loss": 3.6437814890736746, + "tokens_seen": 1103953920 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003361083249749248, + "loss": 1.3143, + "theoretical_loss": 3.643761436553863, + "tokens_seen": 1104019456 + }, + { + "epoch": 0.33, + "learning_rate": 0.000336098294884654, + "loss": 1.3936, + "theoretical_loss": 3.6437413855576324, + "tokens_seen": 1104084992 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033608826479438316, + "loss": 1.497, + "theoretical_loss": 3.643721336084777, + "tokens_seen": 1104150528 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033607823470411234, + "loss": 1.4833, + "theoretical_loss": 3.6437012881350905, + "tokens_seen": 1104216064 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 370683, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.30669367313385, + "objective/train/theoretical_loss": 3.643681241708367, + "objective/train/tokens_used": 1104281600, + "theoretical_loss": 3.643681241708367, + "tokens_seen": 1104281600 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003360682046138415, + "loss": 1.4215, + "theoretical_loss": 3.643681241708367, + "tokens_seen": 1104281600 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033605817452357075, + "loss": 1.3339, + "theoretical_loss": 3.6436611968044, + "tokens_seen": 1104347136 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003360481444332999, + "loss": 1.3436, + "theoretical_loss": 3.6436411534229842, + "tokens_seen": 1104412672 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003360381143430291, + "loss": 1.3855, + "theoretical_loss": 3.6436211115639128, + "tokens_seen": 1104478208 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003360280842527583, + "loss": 1.3578, + "theoretical_loss": 3.64360107122698, + "tokens_seen": 1104543744 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003360180541624875, + "loss": 1.4848, + "theoretical_loss": 3.643581032411981, + "tokens_seen": 1104609280 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033600802407221666, + "loss": 1.2836, + "theoretical_loss": 3.6435609951187082, + "tokens_seen": 1104674816 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033599799398194584, + "loss": 1.2603, + "theoretical_loss": 3.643540959346957, + "tokens_seen": 1104740352 + }, + { + "epoch": 0.33, + "learning_rate": 0.000335987963891675, + "loss": 1.4318, + "theoretical_loss": 3.643520925096521, + "tokens_seen": 1104805888 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033597793380140426, + "loss": 1.3832, + "theoretical_loss": 3.6435008923671948, + "tokens_seen": 1104871424 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003359679037111334, + "loss": 1.3343, + "theoretical_loss": 3.6434808611587726, + "tokens_seen": 1104936960 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003359578736208626, + "loss": 1.3668, + "theoretical_loss": 3.6434608314710486, + "tokens_seen": 1105002496 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033594784353059174, + "loss": 1.3135, + "theoretical_loss": 3.6434408033038173, + "tokens_seen": 1105068032 + }, + { + "epoch": 0.33, + "learning_rate": 0.000335937813440321, + "loss": 1.2516, + "theoretical_loss": 3.6434207766568734, + "tokens_seen": 1105133568 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033592778335005016, + "loss": 1.4104, + "theoretical_loss": 3.643400751530011, + "tokens_seen": 1105199104 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033591775325977934, + "loss": 1.4488, + "theoretical_loss": 3.6433807279230246, + "tokens_seen": 1105264640 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003359077231695085, + "loss": 1.4141, + "theoretical_loss": 3.6433607058357085, + "tokens_seen": 1105330176 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003358976930792377, + "loss": 1.2579, + "theoretical_loss": 3.643340685267858, + "tokens_seen": 1105395712 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003358876629889669, + "loss": 1.2878, + "theoretical_loss": 3.6433206662192665, + "tokens_seen": 1105461248 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003358776328986961, + "loss": 1.2983, + "theoretical_loss": 3.6433006486897304, + "tokens_seen": 1105526784 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033586760280842525, + "loss": 1.3229, + "theoretical_loss": 3.6432806326790423, + "tokens_seen": 1105592320 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003358575727181545, + "loss": 1.2584, + "theoretical_loss": 3.643260618186999, + "tokens_seen": 1105657856 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033584754262788366, + "loss": 1.3113, + "theoretical_loss": 3.643240605213394, + "tokens_seen": 1105723392 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033583751253761285, + "loss": 1.3403, + "theoretical_loss": 3.6432205937580227, + "tokens_seen": 1105788928 + }, + { + "epoch": 0.34, + "learning_rate": 0.000335827482447342, + "loss": 1.4463, + "theoretical_loss": 3.643200583820679, + "tokens_seen": 1105854464 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003358174523570712, + "loss": 1.3832, + "theoretical_loss": 3.6431805754011592, + "tokens_seen": 1105920000 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003358074222668004, + "loss": 1.3117, + "theoretical_loss": 3.6431605684992574, + "tokens_seen": 1105985536 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003357973921765296, + "loss": 1.3363, + "theoretical_loss": 3.6431405631147684, + "tokens_seen": 1106051072 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033578736208625875, + "loss": 1.3384, + "theoretical_loss": 3.6431205592474876, + "tokens_seen": 1106116608 + }, + { + "epoch": 0.34, + "learning_rate": 0.000335777331995988, + "loss": 1.3311, + "theoretical_loss": 3.64310055689721, + "tokens_seen": 1106182144 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003357673019057171, + "loss": 1.3108, + "theoretical_loss": 3.6430805560637305, + "tokens_seen": 1106247680 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033575727181544635, + "loss": 1.3505, + "theoretical_loss": 3.643060556746845, + "tokens_seen": 1106313216 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033574724172517553, + "loss": 1.3626, + "theoretical_loss": 3.643040558946347, + "tokens_seen": 1106378752 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003357372116349047, + "loss": 1.3493, + "theoretical_loss": 3.643020562662034, + "tokens_seen": 1106444288 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003357271815446339, + "loss": 1.3929, + "theoretical_loss": 3.643000567893699, + "tokens_seen": 1106509824 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033571715145436313, + "loss": 1.3401, + "theoretical_loss": 3.642980574641139, + "tokens_seen": 1106575360 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033570712136409225, + "loss": 1.3594, + "theoretical_loss": 3.6429605829041476, + "tokens_seen": 1106640896 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003356970912738215, + "loss": 1.3441, + "theoretical_loss": 3.642940592682522, + "tokens_seen": 1106706432 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003356870611835506, + "loss": 1.394, + "theoretical_loss": 3.642920603976057, + "tokens_seen": 1106771968 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033567703109327985, + "loss": 1.3579, + "theoretical_loss": 3.6429006167845475, + "tokens_seen": 1106837504 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033566700100300903, + "loss": 1.1891, + "theoretical_loss": 3.642880631107789, + "tokens_seen": 1106903040 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003356569709127382, + "loss": 1.328, + "theoretical_loss": 3.642860646945578, + "tokens_seen": 1106968576 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003356469408224674, + "loss": 1.4049, + "theoretical_loss": 3.64284066429771, + "tokens_seen": 1107034112 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003356369107321966, + "loss": 1.4187, + "theoretical_loss": 3.642820683163979, + "tokens_seen": 1107099648 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033562688064192576, + "loss": 1.3525, + "theoretical_loss": 3.6428007035441823, + "tokens_seen": 1107165184 + }, + { + "epoch": 0.34, + "learning_rate": 0.000335616850551655, + "loss": 1.4326, + "theoretical_loss": 3.6427807254381146, + "tokens_seen": 1107230720 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003356068204613841, + "loss": 1.25, + "theoretical_loss": 3.642760748845572, + "tokens_seen": 1107296256 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033559679037111336, + "loss": 1.3713, + "theoretical_loss": 3.6427407737663504, + "tokens_seen": 1107361792 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003355867602808425, + "loss": 1.3549, + "theoretical_loss": 3.642720800200246, + "tokens_seen": 1107427328 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003355767301905717, + "loss": 1.3618, + "theoretical_loss": 3.6427008281470536, + "tokens_seen": 1107492864 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 371546, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2879154682159424, + "objective/train/theoretical_loss": 3.64268085760657, + "objective/train/tokens_used": 1107558400, + "theoretical_loss": 3.64268085760657, + "tokens_seen": 1107558400 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033556670010030095, + "loss": 1.2693, + "theoretical_loss": 3.64268085760657, + "tokens_seen": 1107558400 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003355566700100301, + "loss": 1.3592, + "theoretical_loss": 3.6426608885785905, + "tokens_seen": 1107623936 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003355466399197593, + "loss": 1.2757, + "theoretical_loss": 3.6426409210629114, + "tokens_seen": 1107689472 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003355366098294885, + "loss": 1.445, + "theoretical_loss": 3.6426209550593285, + "tokens_seen": 1107755008 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003355265797392177, + "loss": 1.423, + "theoretical_loss": 3.6426009905676384, + "tokens_seen": 1107820544 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033551654964894686, + "loss": 1.3699, + "theoretical_loss": 3.642581027587636, + "tokens_seen": 1107886080 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033550651955867604, + "loss": 1.4306, + "theoretical_loss": 3.6425610661191192, + "tokens_seen": 1107951616 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003354964894684052, + "loss": 1.3669, + "theoretical_loss": 3.6425411061618824, + "tokens_seen": 1108017152 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033548645937813446, + "loss": 1.3201, + "theoretical_loss": 3.6425211477157227, + "tokens_seen": 1108082688 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003354764292878636, + "loss": 1.333, + "theoretical_loss": 3.6425011907804365, + "tokens_seen": 1108148224 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003354663991975928, + "loss": 1.3344, + "theoretical_loss": 3.64248123535582, + "tokens_seen": 1108213760 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033545636910732195, + "loss": 1.3911, + "theoretical_loss": 3.6424612814416686, + "tokens_seen": 1108279296 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003354463390170512, + "loss": 1.359, + "theoretical_loss": 3.64244132903778, + "tokens_seen": 1108344832 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033543630892678036, + "loss": 1.2619, + "theoretical_loss": 3.6424213781439496, + "tokens_seen": 1108410368 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033542627883650954, + "loss": 1.3081, + "theoretical_loss": 3.642401428759974, + "tokens_seen": 1108475904 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003354162487462387, + "loss": 1.2956, + "theoretical_loss": 3.6423814808856507, + "tokens_seen": 1108541440 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003354062186559679, + "loss": 1.3171, + "theoretical_loss": 3.6423615345207745, + "tokens_seen": 1108606976 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003353961885656971, + "loss": 1.4339, + "theoretical_loss": 3.6423415896651434, + "tokens_seen": 1108672512 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003353861584754263, + "loss": 1.3388, + "theoretical_loss": 3.6423216463185533, + "tokens_seen": 1108738048 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033537612838515545, + "loss": 1.2415, + "theoretical_loss": 3.642301704480801, + "tokens_seen": 1108803584 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003353660982948847, + "loss": 1.2921, + "theoretical_loss": 3.642281764151683, + "tokens_seen": 1108869120 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033535606820461386, + "loss": 1.3586, + "theoretical_loss": 3.642261825330997, + "tokens_seen": 1108934656 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033534603811434305, + "loss": 1.3511, + "theoretical_loss": 3.642241888018538, + "tokens_seen": 1109000192 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003353360080240722, + "loss": 1.3198, + "theoretical_loss": 3.642221952214104, + "tokens_seen": 1109065728 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003353259779338014, + "loss": 1.3055, + "theoretical_loss": 3.6422020179174917, + "tokens_seen": 1109131264 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003353159478435306, + "loss": 1.3866, + "theoretical_loss": 3.642182085128498, + "tokens_seen": 1109196800 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003353059177532598, + "loss": 1.4316, + "theoretical_loss": 3.642162153846919, + "tokens_seen": 1109262336 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033529588766298895, + "loss": 1.3607, + "theoretical_loss": 3.642142224072553, + "tokens_seen": 1109327872 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003352858575727182, + "loss": 1.3415, + "theoretical_loss": 3.642122295805197, + "tokens_seen": 1109393408 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003352758274824473, + "loss": 1.3968, + "theoretical_loss": 3.642102369044646, + "tokens_seen": 1109458944 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033526579739217655, + "loss": 1.3337, + "theoretical_loss": 3.642082443790699, + "tokens_seen": 1109524480 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033525576730190573, + "loss": 1.3512, + "theoretical_loss": 3.6420625200431527, + "tokens_seen": 1109590016 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003352457372116349, + "loss": 1.3183, + "theoretical_loss": 3.6420425978018036, + "tokens_seen": 1109655552 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003352357071213641, + "loss": 1.4135, + "theoretical_loss": 3.64202267706645, + "tokens_seen": 1109721088 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033522567703109333, + "loss": 1.3543, + "theoretical_loss": 3.642002757836888, + "tokens_seen": 1109786624 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033521564694082245, + "loss": 1.4972, + "theoretical_loss": 3.6419828401129153, + "tokens_seen": 1109852160 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003352056168505517, + "loss": 1.3357, + "theoretical_loss": 3.6419629238943294, + "tokens_seen": 1109917696 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003351955867602808, + "loss": 1.3333, + "theoretical_loss": 3.6419430091809275, + "tokens_seen": 1109983232 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033518555667001005, + "loss": 1.4121, + "theoretical_loss": 3.641923095972507, + "tokens_seen": 1110048768 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033517552657973923, + "loss": 1.2964, + "theoretical_loss": 3.641903184268865, + "tokens_seen": 1110114304 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003351654964894684, + "loss": 1.2453, + "theoretical_loss": 3.6418832740698, + "tokens_seen": 1110179840 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003351554663991976, + "loss": 1.3235, + "theoretical_loss": 3.6418633653751087, + "tokens_seen": 1110245376 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003351454363089268, + "loss": 1.2798, + "theoretical_loss": 3.6418434581845887, + "tokens_seen": 1110310912 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033513540621865596, + "loss": 1.4222, + "theoretical_loss": 3.641823552498037, + "tokens_seen": 1110376448 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003351253761283852, + "loss": 1.3947, + "theoretical_loss": 3.6418036483152525, + "tokens_seen": 1110441984 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003351153460381143, + "loss": 1.4223, + "theoretical_loss": 3.641783745636032, + "tokens_seen": 1110507520 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033510531594784356, + "loss": 1.4262, + "theoretical_loss": 3.641763844460173, + "tokens_seen": 1110573056 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003350952858575727, + "loss": 1.4153, + "theoretical_loss": 3.641743944787474, + "tokens_seen": 1110638592 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003350852557673019, + "loss": 1.3848, + "theoretical_loss": 3.6417240466177327, + "tokens_seen": 1110704128 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003350752256770311, + "loss": 1.2687, + "theoretical_loss": 3.641704149950746, + "tokens_seen": 1110769664 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 372441, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3595235347747803, + "objective/train/theoretical_loss": 3.6416842547863126, + "objective/train/tokens_used": 1110835200, + "theoretical_loss": 3.6416842547863126, + "tokens_seen": 1110835200 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003350651955867603, + "loss": 1.3761, + "theoretical_loss": 3.6416842547863126, + "tokens_seen": 1110835200 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033505516549648946, + "loss": 1.4456, + "theoretical_loss": 3.6416643611242305, + "tokens_seen": 1110900736 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003350451354062187, + "loss": 1.3761, + "theoretical_loss": 3.641644468964297, + "tokens_seen": 1110966272 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003350351053159478, + "loss": 1.3896, + "theoretical_loss": 3.6416245783063106, + "tokens_seen": 1111031808 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033502507522567706, + "loss": 1.2112, + "theoretical_loss": 3.6416046891500686, + "tokens_seen": 1111097344 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003350150451354062, + "loss": 1.3831, + "theoretical_loss": 3.64158480149537, + "tokens_seen": 1111162880 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003350050150451354, + "loss": 1.2866, + "theoretical_loss": 3.641564915342012, + "tokens_seen": 1111228416 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003349949849548646, + "loss": 1.2955, + "theoretical_loss": 3.6415450306897936, + "tokens_seen": 1111293952 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003349849548645938, + "loss": 1.3538, + "theoretical_loss": 3.6415251475385126, + "tokens_seen": 1111359488 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033497492477432296, + "loss": 1.2752, + "theoretical_loss": 3.6415052658879667, + "tokens_seen": 1111425024 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033496489468405215, + "loss": 1.3708, + "theoretical_loss": 3.6414853857379548, + "tokens_seen": 1111490560 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003349548645937813, + "loss": 1.3961, + "theoretical_loss": 3.641465507088275, + "tokens_seen": 1111556096 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033494483450351056, + "loss": 1.3253, + "theoretical_loss": 3.6414456299387257, + "tokens_seen": 1111621632 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003349348044132397, + "loss": 1.3921, + "theoretical_loss": 3.641425754289105, + "tokens_seen": 1111687168 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003349247743229689, + "loss": 1.3817, + "theoretical_loss": 3.641405880139211, + "tokens_seen": 1111752704 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033491474423269805, + "loss": 1.3469, + "theoretical_loss": 3.641386007488843, + "tokens_seen": 1111818240 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003349047141424273, + "loss": 1.3311, + "theoretical_loss": 3.6413661363377994, + "tokens_seen": 1111883776 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033489468405215647, + "loss": 1.3845, + "theoretical_loss": 3.6413462666858782, + "tokens_seen": 1111949312 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033488465396188565, + "loss": 1.3571, + "theoretical_loss": 3.641326398532878, + "tokens_seen": 1112014848 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033487462387161483, + "loss": 1.2879, + "theoretical_loss": 3.6413065318785978, + "tokens_seen": 1112080384 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033486459378134406, + "loss": 1.3352, + "theoretical_loss": 3.6412866667228356, + "tokens_seen": 1112145920 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003348545636910732, + "loss": 1.2997, + "theoretical_loss": 3.641266803065391, + "tokens_seen": 1112211456 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003348445336008024, + "loss": 1.3767, + "theoretical_loss": 3.6412469409060617, + "tokens_seen": 1112276992 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003348345035105316, + "loss": 1.3296, + "theoretical_loss": 3.641227080244647, + "tokens_seen": 1112342528 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003348244734202608, + "loss": 1.3132, + "theoretical_loss": 3.6412072210809456, + "tokens_seen": 1112408064 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033481444332999, + "loss": 1.3401, + "theoretical_loss": 3.6411873634147565, + "tokens_seen": 1112473600 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033480441323971915, + "loss": 1.3381, + "theoretical_loss": 3.6411675072458785, + "tokens_seen": 1112539136 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003347943831494484, + "loss": 1.3693, + "theoretical_loss": 3.64114765257411, + "tokens_seen": 1112604672 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003347843530591775, + "loss": 1.4209, + "theoretical_loss": 3.641127799399251, + "tokens_seen": 1112670208 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033477432296890675, + "loss": 1.3159, + "theoretical_loss": 3.641107947721099, + "tokens_seen": 1112735744 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033476429287863593, + "loss": 1.3261, + "theoretical_loss": 3.641088097539454, + "tokens_seen": 1112801280 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003347542627883651, + "loss": 1.3298, + "theoretical_loss": 3.6410682488541157, + "tokens_seen": 1112866816 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003347442326980943, + "loss": 1.4173, + "theoretical_loss": 3.641048401664882, + "tokens_seen": 1112932352 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033473420260782353, + "loss": 1.3703, + "theoretical_loss": 3.6410285559715523, + "tokens_seen": 1112997888 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033472417251755265, + "loss": 1.4407, + "theoretical_loss": 3.6410087117739263, + "tokens_seen": 1113063424 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003347141424272819, + "loss": 1.3431, + "theoretical_loss": 3.6409888690718026, + "tokens_seen": 1113128960 + }, + { + "epoch": 0.34, + "learning_rate": 0.000334704112337011, + "loss": 1.272, + "theoretical_loss": 3.640969027864981, + "tokens_seen": 1113194496 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033469408224674025, + "loss": 1.2847, + "theoretical_loss": 3.6409491881532605, + "tokens_seen": 1113260032 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033468405215646943, + "loss": 1.3402, + "theoretical_loss": 3.64092934993644, + "tokens_seen": 1113325568 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003346740220661986, + "loss": 1.3248, + "theoretical_loss": 3.64090951321432, + "tokens_seen": 1113391104 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003346639919759278, + "loss": 1.3531, + "theoretical_loss": 3.6408896779866993, + "tokens_seen": 1113456640 + }, + { + "epoch": 0.34, + "learning_rate": 0.000334653961885657, + "loss": 1.3331, + "theoretical_loss": 3.640869844253377, + "tokens_seen": 1113522176 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033464393179538616, + "loss": 1.3721, + "theoretical_loss": 3.6408500120141527, + "tokens_seen": 1113587712 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003346339017051154, + "loss": 1.3244, + "theoretical_loss": 3.640830181268826, + "tokens_seen": 1113653248 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003346238716148445, + "loss": 1.343, + "theoretical_loss": 3.6408103520171973, + "tokens_seen": 1113718784 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033461384152457376, + "loss": 1.4437, + "theoretical_loss": 3.6407905242590646, + "tokens_seen": 1113784320 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003346038114343029, + "loss": 1.3958, + "theoretical_loss": 3.6407706979942294, + "tokens_seen": 1113849856 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003345937813440321, + "loss": 1.3872, + "theoretical_loss": 3.6407508732224896, + "tokens_seen": 1113915392 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003345837512537613, + "loss": 1.4011, + "theoretical_loss": 3.6407310499436463, + "tokens_seen": 1113980928 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003345737211634905, + "loss": 1.3168, + "theoretical_loss": 3.6407112281574987, + "tokens_seen": 1114046464 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 373698, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3815072774887085, + "objective/train/theoretical_loss": 3.640691407863846, + "objective/train/tokens_used": 1114112000, + "theoretical_loss": 3.640691407863846, + "tokens_seen": 1114112000 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033456369107321966, + "loss": 1.3353, + "theoretical_loss": 3.640691407863846, + "tokens_seen": 1114112000 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003345536609829489, + "loss": 1.4585, + "theoretical_loss": 3.6406715890624897, + "tokens_seen": 1114177536 + }, + { + "epoch": 0.34, + "learning_rate": 0.000334543630892678, + "loss": 1.291, + "theoretical_loss": 3.6406517717532276, + "tokens_seen": 1114243072 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033453360080240726, + "loss": 1.4391, + "theoretical_loss": 3.640631955935861, + "tokens_seen": 1114308608 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003345235707121364, + "loss": 1.3805, + "theoretical_loss": 3.6406121416101893, + "tokens_seen": 1114374144 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003345135406218656, + "loss": 1.3992, + "theoretical_loss": 3.6405923287760134, + "tokens_seen": 1114439680 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003345035105315948, + "loss": 1.45, + "theoretical_loss": 3.6405725174331316, + "tokens_seen": 1114505216 + }, + { + "epoch": 0.34, + "learning_rate": 0.000334493480441324, + "loss": 1.3933, + "theoretical_loss": 3.640552707581346, + "tokens_seen": 1114570752 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033448345035105316, + "loss": 1.4084, + "theoretical_loss": 3.640532899220455, + "tokens_seen": 1114636288 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033447342026078235, + "loss": 1.3029, + "theoretical_loss": 3.6405130923502593, + "tokens_seen": 1114701824 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003344633901705115, + "loss": 1.3445, + "theoretical_loss": 3.6404932869705595, + "tokens_seen": 1114767360 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033445336008024076, + "loss": 1.2695, + "theoretical_loss": 3.6404734830811556, + "tokens_seen": 1114832896 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003344433299899699, + "loss": 1.4372, + "theoretical_loss": 3.6404536806818477, + "tokens_seen": 1114898432 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003344332998996991, + "loss": 1.4207, + "theoretical_loss": 3.640433879772436, + "tokens_seen": 1114963968 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033442326980942825, + "loss": 1.3388, + "theoretical_loss": 3.6404140803527216, + "tokens_seen": 1115029504 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003344132397191575, + "loss": 1.2536, + "theoretical_loss": 3.640394282422504, + "tokens_seen": 1115095040 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033440320962888667, + "loss": 1.4004, + "theoretical_loss": 3.6403744859815843, + "tokens_seen": 1115160576 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033439317953861585, + "loss": 1.3688, + "theoretical_loss": 3.6403546910297617, + "tokens_seen": 1115226112 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033438314944834503, + "loss": 1.3595, + "theoretical_loss": 3.6403348975668384, + "tokens_seen": 1115291648 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033437311935807427, + "loss": 1.3178, + "theoretical_loss": 3.640315105592614, + "tokens_seen": 1115357184 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003343630892678034, + "loss": 1.3071, + "theoretical_loss": 3.640295315106889, + "tokens_seen": 1115422720 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033435305917753263, + "loss": 1.543, + "theoretical_loss": 3.640275526109464, + "tokens_seen": 1115488256 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033434302908726175, + "loss": 1.362, + "theoretical_loss": 3.64025573860014, + "tokens_seen": 1115553792 + }, + { + "epoch": 0.34, + "learning_rate": 0.000334332998996991, + "loss": 1.4304, + "theoretical_loss": 3.640235952578718, + "tokens_seen": 1115619328 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033432296890672017, + "loss": 1.4729, + "theoretical_loss": 3.640216168044997, + "tokens_seen": 1115684864 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033431293881644935, + "loss": 1.321, + "theoretical_loss": 3.64019638499878, + "tokens_seen": 1115750400 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033430290872617853, + "loss": 1.2997, + "theoretical_loss": 3.640176603439867, + "tokens_seen": 1115815936 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003342928786359077, + "loss": 1.3681, + "theoretical_loss": 3.640156823368058, + "tokens_seen": 1115881472 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003342828485456369, + "loss": 1.3695, + "theoretical_loss": 3.6401370447831547, + "tokens_seen": 1115947008 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033427281845536613, + "loss": 1.3551, + "theoretical_loss": 3.640117267684958, + "tokens_seen": 1116012544 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033426278836509526, + "loss": 1.4196, + "theoretical_loss": 3.6400974920732683, + "tokens_seen": 1116078080 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003342527582748245, + "loss": 1.3454, + "theoretical_loss": 3.6400777179478876, + "tokens_seen": 1116143616 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003342427281845536, + "loss": 1.315, + "theoretical_loss": 3.640057945308616, + "tokens_seen": 1116209152 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033423269809428285, + "loss": 1.3494, + "theoretical_loss": 3.6400381741552548, + "tokens_seen": 1116274688 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033422266800401204, + "loss": 1.3555, + "theoretical_loss": 3.6400184044876056, + "tokens_seen": 1116340224 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003342126379137412, + "loss": 1.3569, + "theoretical_loss": 3.6399986363054686, + "tokens_seen": 1116405760 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003342026078234704, + "loss": 1.3786, + "theoretical_loss": 3.6399788696086457, + "tokens_seen": 1116471296 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033419257773319963, + "loss": 1.2612, + "theoretical_loss": 3.639959104396938, + "tokens_seen": 1116536832 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033418254764292876, + "loss": 1.3503, + "theoretical_loss": 3.6399393406701464, + "tokens_seen": 1116602368 + }, + { + "epoch": 0.34, + "learning_rate": 0.000334172517552658, + "loss": 1.3639, + "theoretical_loss": 3.639919578428073, + "tokens_seen": 1116667904 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003341624874623871, + "loss": 1.267, + "theoretical_loss": 3.639899817670518, + "tokens_seen": 1116733440 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033415245737211636, + "loss": 1.2902, + "theoretical_loss": 3.639880058397284, + "tokens_seen": 1116798976 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033414242728184554, + "loss": 1.2823, + "theoretical_loss": 3.639860300608172, + "tokens_seen": 1116864512 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003341323971915747, + "loss": 1.312, + "theoretical_loss": 3.6398405443029827, + "tokens_seen": 1116930048 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003341223671013039, + "loss": 1.3547, + "theoretical_loss": 3.6398207894815178, + "tokens_seen": 1116995584 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003341123370110331, + "loss": 1.3292, + "theoretical_loss": 3.63980103614358, + "tokens_seen": 1117061120 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033410230692076226, + "loss": 1.261, + "theoretical_loss": 3.6397812842889694, + "tokens_seen": 1117126656 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003340922768304915, + "loss": 1.3927, + "theoretical_loss": 3.639761533917489, + "tokens_seen": 1117192192 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003340822467402207, + "loss": 1.5071, + "theoretical_loss": 3.639741785028939, + "tokens_seen": 1117257728 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033407221664994986, + "loss": 1.3619, + "theoretical_loss": 3.6397220376231223, + "tokens_seen": 1117323264 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 374907, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2493081092834473, + "objective/train/theoretical_loss": 3.63970229169984, + "objective/train/tokens_used": 1117388800, + "theoretical_loss": 3.63970229169984, + "tokens_seen": 1117388800 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003340621865596791, + "loss": 1.292, + "theoretical_loss": 3.63970229169984, + "tokens_seen": 1117388800 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003340521564694082, + "loss": 1.2627, + "theoretical_loss": 3.639682547258894, + "tokens_seen": 1117454336 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033404212637913746, + "loss": 1.3792, + "theoretical_loss": 3.6396628043000856, + "tokens_seen": 1117519872 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003340320962888666, + "loss": 1.3184, + "theoretical_loss": 3.6396430628232173, + "tokens_seen": 1117585408 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003340220661985958, + "loss": 1.451, + "theoretical_loss": 3.6396233228280908, + "tokens_seen": 1117650944 + }, + { + "epoch": 0.34, + "learning_rate": 0.000334012036108325, + "loss": 1.3256, + "theoretical_loss": 3.639603584314508, + "tokens_seen": 1117716480 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003340020060180542, + "loss": 1.2867, + "theoretical_loss": 3.639583847282271, + "tokens_seen": 1117782016 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033399197592778336, + "loss": 1.3224, + "theoretical_loss": 3.6395641117311817, + "tokens_seen": 1117847552 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033398194583751255, + "loss": 1.4552, + "theoretical_loss": 3.6395443776610414, + "tokens_seen": 1117913088 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003339719157472417, + "loss": 1.225, + "theoretical_loss": 3.6395246450716527, + "tokens_seen": 1117978624 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033396188565697096, + "loss": 1.3624, + "theoretical_loss": 3.6395049139628184, + "tokens_seen": 1118044160 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003339518555667001, + "loss": 1.3057, + "theoretical_loss": 3.63948518433434, + "tokens_seen": 1118109696 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003339418254764293, + "loss": 1.2317, + "theoretical_loss": 3.639465456186019, + "tokens_seen": 1118175232 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033393179538615845, + "loss": 1.3742, + "theoretical_loss": 3.639445729517659, + "tokens_seen": 1118240768 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003339217652958877, + "loss": 1.3551, + "theoretical_loss": 3.6394260043290614, + "tokens_seen": 1118306304 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033391173520561687, + "loss": 1.4075, + "theoretical_loss": 3.6394062806200287, + "tokens_seen": 1118371840 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033390170511534605, + "loss": 1.3351, + "theoretical_loss": 3.639386558390363, + "tokens_seen": 1118437376 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033389167502507523, + "loss": 1.3781, + "theoretical_loss": 3.6393668376398667, + "tokens_seen": 1118502912 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033388164493480447, + "loss": 1.2847, + "theoretical_loss": 3.6393471183683426, + "tokens_seen": 1118568448 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003338716148445336, + "loss": 1.3488, + "theoretical_loss": 3.639327400575593, + "tokens_seen": 1118633984 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033386158475426283, + "loss": 1.3342, + "theoretical_loss": 3.6393076842614196, + "tokens_seen": 1118699520 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033385155466399195, + "loss": 1.275, + "theoretical_loss": 3.6392879694256264, + "tokens_seen": 1118765056 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003338415245737212, + "loss": 1.3804, + "theoretical_loss": 3.6392682560680143, + "tokens_seen": 1118830592 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033383149448345037, + "loss": 1.3989, + "theoretical_loss": 3.6392485441883875, + "tokens_seen": 1118896128 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033382146439317955, + "loss": 1.2883, + "theoretical_loss": 3.6392288337865475, + "tokens_seen": 1118961664 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033382146439317955, + "loss": 1.5716, + "theoretical_loss": 3.6392091248622975, + "tokens_seen": 1119027200 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033381143430290873, + "loss": 1.3162, + "theoretical_loss": 3.6391894174154396, + "tokens_seen": 1119092736 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003338014042126379, + "loss": 1.3181, + "theoretical_loss": 3.6391697114457773, + "tokens_seen": 1119158272 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003337913741223671, + "loss": 1.3369, + "theoretical_loss": 3.639150006953113, + "tokens_seen": 1119223808 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033378134403209633, + "loss": 1.4258, + "theoretical_loss": 3.63913030393725, + "tokens_seen": 1119289344 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033377131394182546, + "loss": 1.461, + "theoretical_loss": 3.6391106023979902, + "tokens_seen": 1119354880 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003337612838515547, + "loss": 1.3616, + "theoretical_loss": 3.6390909023351368, + "tokens_seen": 1119420416 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003337612838515547, + "loss": 1.2862, + "theoretical_loss": 3.639071203748494, + "tokens_seen": 1119485952 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003337512537612838, + "loss": 1.3179, + "theoretical_loss": 3.6390515066378626, + "tokens_seen": 1119551488 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033374122367101306, + "loss": 1.389, + "theoretical_loss": 3.6390318110030475, + "tokens_seen": 1119617024 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033373119358074224, + "loss": 1.4435, + "theoretical_loss": 3.6390121168438507, + "tokens_seen": 1119682560 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003337211634904714, + "loss": 1.3197, + "theoretical_loss": 3.638992424160076, + "tokens_seen": 1119748096 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003337111334002006, + "loss": 1.2947, + "theoretical_loss": 3.6389727329515256, + "tokens_seen": 1119813632 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033370110330992983, + "loss": 1.3932, + "theoretical_loss": 3.6389530432180033, + "tokens_seen": 1119879168 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033369107321965896, + "loss": 1.4446, + "theoretical_loss": 3.6389333549593124, + "tokens_seen": 1119944704 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003336810431293882, + "loss": 1.32, + "theoretical_loss": 3.638913668175255, + "tokens_seen": 1120010240 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003336710130391173, + "loss": 1.5033, + "theoretical_loss": 3.6388939828656364, + "tokens_seen": 1120075776 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033366098294884656, + "loss": 1.4082, + "theoretical_loss": 3.6388742990302587, + "tokens_seen": 1120141312 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033365095285857574, + "loss": 1.279, + "theoretical_loss": 3.638854616668925, + "tokens_seen": 1120206848 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003336409227683049, + "loss": 1.3494, + "theoretical_loss": 3.638834935781439, + "tokens_seen": 1120272384 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003336308926780341, + "loss": 1.3344, + "theoretical_loss": 3.6388152563676037, + "tokens_seen": 1120337920 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003336208625877633, + "loss": 1.3869, + "theoretical_loss": 3.638795578427224, + "tokens_seen": 1120403456 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033361083249749246, + "loss": 1.316, + "theoretical_loss": 3.638775901960102, + "tokens_seen": 1120468992 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003336008024072217, + "loss": 1.3146, + "theoretical_loss": 3.6387562269660414, + "tokens_seen": 1120534528 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003335907723169508, + "loss": 1.3978, + "theoretical_loss": 3.638736553444846, + "tokens_seen": 1120600064 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 375797, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4197529554367065, + "objective/train/theoretical_loss": 3.63871688139632, + "objective/train/tokens_used": 1120665600, + "theoretical_loss": 3.63871688139632, + "tokens_seen": 1120665600 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033358074222668006, + "loss": 1.3913, + "theoretical_loss": 3.63871688139632, + "tokens_seen": 1120665600 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033357071213640924, + "loss": 1.3444, + "theoretical_loss": 3.638697210820266, + "tokens_seen": 1120731136 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003335606820461384, + "loss": 1.3146, + "theoretical_loss": 3.6386775417164885, + "tokens_seen": 1120796672 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003335506519558676, + "loss": 1.319, + "theoretical_loss": 3.638657874084791, + "tokens_seen": 1120862208 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003335406218655968, + "loss": 1.3283, + "theoretical_loss": 3.638638207924977, + "tokens_seen": 1120927744 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033353059177532597, + "loss": 1.4001, + "theoretical_loss": 3.6386185432368503, + "tokens_seen": 1120993280 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003335205616850552, + "loss": 1.3016, + "theoretical_loss": 3.638598880020215, + "tokens_seen": 1121058816 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033351053159478433, + "loss": 1.4238, + "theoretical_loss": 3.638579218274875, + "tokens_seen": 1121124352 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033350050150451356, + "loss": 1.4085, + "theoretical_loss": 3.638559558000634, + "tokens_seen": 1121189888 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003334904714142427, + "loss": 1.2924, + "theoretical_loss": 3.6385398991972964, + "tokens_seen": 1121255424 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003334804413239719, + "loss": 1.3552, + "theoretical_loss": 3.638520241864666, + "tokens_seen": 1121320960 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003334704112337011, + "loss": 1.3645, + "theoretical_loss": 3.6385005860025457, + "tokens_seen": 1121386496 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003334603811434303, + "loss": 1.3136, + "theoretical_loss": 3.6384809316107414, + "tokens_seen": 1121452032 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033345035105315947, + "loss": 1.4346, + "theoretical_loss": 3.638461278689056, + "tokens_seen": 1121517568 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033344032096288865, + "loss": 1.3882, + "theoretical_loss": 3.6384416272372944, + "tokens_seen": 1121583104 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033343029087261783, + "loss": 1.3406, + "theoretical_loss": 3.63842197725526, + "tokens_seen": 1121648640 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033342026078234707, + "loss": 1.3971, + "theoretical_loss": 3.6384023287427576, + "tokens_seen": 1121714176 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003334102306920762, + "loss": 1.3115, + "theoretical_loss": 3.638382681699591, + "tokens_seen": 1121779712 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033340020060180543, + "loss": 1.3515, + "theoretical_loss": 3.6383630361255648, + "tokens_seen": 1121845248 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003333901705115346, + "loss": 1.4567, + "theoretical_loss": 3.638343392020483, + "tokens_seen": 1121910784 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003333801404212638, + "loss": 1.3551, + "theoretical_loss": 3.638323749384151, + "tokens_seen": 1121976320 + }, + { + "epoch": 0.34, + "learning_rate": 0.000333370110330993, + "loss": 1.291, + "theoretical_loss": 3.6383041082163716, + "tokens_seen": 1122041856 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033336008024072215, + "loss": 1.2786, + "theoretical_loss": 3.63828446851695, + "tokens_seen": 1122107392 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033335005015045134, + "loss": 1.4398, + "theoretical_loss": 3.6382648302856917, + "tokens_seen": 1122172928 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033334002006018057, + "loss": 1.3445, + "theoretical_loss": 3.6382451935223994, + "tokens_seen": 1122238464 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033332998996990975, + "loss": 1.3544, + "theoretical_loss": 3.638225558226879, + "tokens_seen": 1122304000 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033331995987963893, + "loss": 1.3693, + "theoretical_loss": 3.6382059243989344, + "tokens_seen": 1122369536 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003333099297893681, + "loss": 1.3746, + "theoretical_loss": 3.6381862920383705, + "tokens_seen": 1122435072 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003332998996990973, + "loss": 1.3637, + "theoretical_loss": 3.6381666611449917, + "tokens_seen": 1122500608 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033328986960882653, + "loss": 1.3863, + "theoretical_loss": 3.638147031718603, + "tokens_seen": 1122566144 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033327983951855566, + "loss": 1.4103, + "theoretical_loss": 3.6381274037590092, + "tokens_seen": 1122631680 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003332698094282849, + "loss": 1.4044, + "theoretical_loss": 3.638107777266015, + "tokens_seen": 1122697216 + }, + { + "epoch": 0.34, + "learning_rate": 0.000333259779338014, + "loss": 1.5289, + "theoretical_loss": 3.6380881522394244, + "tokens_seen": 1122762752 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033324974924774326, + "loss": 1.3053, + "theoretical_loss": 3.6380685286790437, + "tokens_seen": 1122828288 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033323971915747244, + "loss": 1.3112, + "theoretical_loss": 3.638048906584677, + "tokens_seen": 1122893824 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003332296890672016, + "loss": 1.422, + "theoretical_loss": 3.6380292859561294, + "tokens_seen": 1122959360 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003332196589769308, + "loss": 1.2993, + "theoretical_loss": 3.6380096667932054, + "tokens_seen": 1123024896 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033320962888666003, + "loss": 1.349, + "theoretical_loss": 3.63799004909571, + "tokens_seen": 1123090432 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033319959879638916, + "loss": 1.2655, + "theoretical_loss": 3.6379704328634492, + "tokens_seen": 1123155968 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003331895687061184, + "loss": 1.2834, + "theoretical_loss": 3.6379508180962272, + "tokens_seen": 1123221504 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003331795386158475, + "loss": 1.2542, + "theoretical_loss": 3.63793120479385, + "tokens_seen": 1123287040 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033316950852557676, + "loss": 1.3242, + "theoretical_loss": 3.6379115929561214, + "tokens_seen": 1123352576 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033315947843530594, + "loss": 1.433, + "theoretical_loss": 3.6378919825828477, + "tokens_seen": 1123418112 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003331494483450351, + "loss": 1.3752, + "theoretical_loss": 3.637872373673834, + "tokens_seen": 1123483648 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003331394182547643, + "loss": 1.4052, + "theoretical_loss": 3.637852766228885, + "tokens_seen": 1123549184 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003331293881644935, + "loss": 1.4025, + "theoretical_loss": 3.637833160247806, + "tokens_seen": 1123614720 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033311935807422266, + "loss": 1.3757, + "theoretical_loss": 3.637813555730403, + "tokens_seen": 1123680256 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003331093279839519, + "loss": 1.3297, + "theoretical_loss": 3.637793952676481, + "tokens_seen": 1123745792 + }, + { + "epoch": 0.34, + "learning_rate": 0.000333099297893681, + "loss": 1.3332, + "theoretical_loss": 3.637774351085845, + "tokens_seen": 1123811328 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033308926780341026, + "loss": 1.3956, + "theoretical_loss": 3.637754750958302, + "tokens_seen": 1123876864 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 377038, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3510358333587646, + "objective/train/theoretical_loss": 3.637735152293655, + "objective/train/tokens_used": 1123942400, + "theoretical_loss": 3.637735152293655, + "tokens_seen": 1123942400 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033307923771313944, + "loss": 1.404, + "theoretical_loss": 3.637735152293655, + "tokens_seen": 1123942400 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003330692076228686, + "loss": 1.3957, + "theoretical_loss": 3.637715555091712, + "tokens_seen": 1124007936 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003330591775325978, + "loss": 1.4376, + "theoretical_loss": 3.6376959593522766, + "tokens_seen": 1124073472 + }, + { + "epoch": 0.34, + "learning_rate": 0.000333049147442327, + "loss": 1.3442, + "theoretical_loss": 3.637676365075156, + "tokens_seen": 1124139008 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033303911735205617, + "loss": 1.313, + "theoretical_loss": 3.6376567722601547, + "tokens_seen": 1124204544 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003330290872617854, + "loss": 1.3202, + "theoretical_loss": 3.637637180907079, + "tokens_seen": 1124270080 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033301905717151453, + "loss": 1.33, + "theoretical_loss": 3.637617591015734, + "tokens_seen": 1124335616 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033300902708124376, + "loss": 1.2423, + "theoretical_loss": 3.637598002585926, + "tokens_seen": 1124401152 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003329989969909729, + "loss": 1.2652, + "theoretical_loss": 3.6375784156174604, + "tokens_seen": 1124466688 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003329889669007021, + "loss": 1.3488, + "theoretical_loss": 3.6375588301101436, + "tokens_seen": 1124532224 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003329789368104313, + "loss": 1.3582, + "theoretical_loss": 3.6375392460637808, + "tokens_seen": 1124597760 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003329689067201605, + "loss": 1.3547, + "theoretical_loss": 3.6375196634781783, + "tokens_seen": 1124663296 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033295887662988967, + "loss": 1.3302, + "theoretical_loss": 3.637500082353142, + "tokens_seen": 1124728832 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033294884653961885, + "loss": 1.3606, + "theoretical_loss": 3.6374805026884776, + "tokens_seen": 1124794368 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033293881644934803, + "loss": 1.4117, + "theoretical_loss": 3.637460924483991, + "tokens_seen": 1124859904 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033292878635907727, + "loss": 1.2721, + "theoretical_loss": 3.6374413477394887, + "tokens_seen": 1124925440 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003329187562688064, + "loss": 1.2985, + "theoretical_loss": 3.637421772454777, + "tokens_seen": 1124990976 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033290872617853563, + "loss": 1.4395, + "theoretical_loss": 3.637402198629661, + "tokens_seen": 1125056512 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003328986960882648, + "loss": 1.2673, + "theoretical_loss": 3.637382626263948, + "tokens_seen": 1125122048 + }, + { + "epoch": 0.34, + "learning_rate": 0.000332888665997994, + "loss": 1.3993, + "theoretical_loss": 3.637363055357443, + "tokens_seen": 1125187584 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003328786359077232, + "loss": 1.342, + "theoretical_loss": 3.6373434859099536, + "tokens_seen": 1125253120 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033286860581745235, + "loss": 1.384, + "theoretical_loss": 3.637323917921285, + "tokens_seen": 1125318656 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033285857572718154, + "loss": 1.2281, + "theoretical_loss": 3.637304351391244, + "tokens_seen": 1125384192 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033284854563691077, + "loss": 1.3885, + "theoretical_loss": 3.6372847863196363, + "tokens_seen": 1125449728 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003328385155466399, + "loss": 1.2886, + "theoretical_loss": 3.6372652227062696, + "tokens_seen": 1125515264 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033282848545636913, + "loss": 1.278, + "theoretical_loss": 3.637245660550949, + "tokens_seen": 1125580800 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033281845536609826, + "loss": 1.4387, + "theoretical_loss": 3.6372260998534816, + "tokens_seen": 1125646336 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003328084252758275, + "loss": 1.2377, + "theoretical_loss": 3.6372065406136733, + "tokens_seen": 1125711872 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003327983951855567, + "loss": 1.3916, + "theoretical_loss": 3.6371869828313317, + "tokens_seen": 1125777408 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033278836509528586, + "loss": 1.4099, + "theoretical_loss": 3.637167426506262, + "tokens_seen": 1125842944 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033277833500501504, + "loss": 1.3542, + "theoretical_loss": 3.637147871638272, + "tokens_seen": 1125908480 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003327683049147442, + "loss": 1.3698, + "theoretical_loss": 3.637128318227168, + "tokens_seen": 1125974016 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003327582748244734, + "loss": 1.3841, + "theoretical_loss": 3.637108766272756, + "tokens_seen": 1126039552 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033274824473420264, + "loss": 1.2721, + "theoretical_loss": 3.637089215774843, + "tokens_seen": 1126105088 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033273821464393176, + "loss": 1.296, + "theoretical_loss": 3.6370696667332365, + "tokens_seen": 1126170624 + }, + { + "epoch": 0.34, + "learning_rate": 0.000332728184553661, + "loss": 1.3586, + "theoretical_loss": 3.6370501191477427, + "tokens_seen": 1126236160 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003327181544633902, + "loss": 1.426, + "theoretical_loss": 3.6370305730181682, + "tokens_seen": 1126301696 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033270812437311936, + "loss": 1.289, + "theoretical_loss": 3.6370110283443204, + "tokens_seen": 1126367232 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033269809428284854, + "loss": 1.3275, + "theoretical_loss": 3.6369914851260057, + "tokens_seen": 1126432768 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003326880641925777, + "loss": 1.3064, + "theoretical_loss": 3.636971943363031, + "tokens_seen": 1126498304 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003326780341023069, + "loss": 1.3562, + "theoretical_loss": 3.6369524030552043, + "tokens_seen": 1126563840 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033266800401203614, + "loss": 1.3371, + "theoretical_loss": 3.6369328642023313, + "tokens_seen": 1126629376 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033265797392176527, + "loss": 1.3876, + "theoretical_loss": 3.6369133268042195, + "tokens_seen": 1126694912 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003326479438314945, + "loss": 1.3966, + "theoretical_loss": 3.636893790860676, + "tokens_seen": 1126760448 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033263791374122363, + "loss": 1.3876, + "theoretical_loss": 3.6368742563715077, + "tokens_seen": 1126825984 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033262788365095286, + "loss": 1.3334, + "theoretical_loss": 3.6368547233365227, + "tokens_seen": 1126891520 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033261785356068205, + "loss": 1.4626, + "theoretical_loss": 3.6368351917555275, + "tokens_seen": 1126957056 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003326078234704112, + "loss": 1.3642, + "theoretical_loss": 3.6368156616283285, + "tokens_seen": 1127022592 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003325977933801404, + "loss": 1.2555, + "theoretical_loss": 3.6367961329547343, + "tokens_seen": 1127088128 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033258776328986964, + "loss": 1.3876, + "theoretical_loss": 3.6367766057345516, + "tokens_seen": 1127153664 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 377930, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3540847301483154, + "objective/train/theoretical_loss": 3.6367570799675875, + "objective/train/tokens_used": 1127219200, + "theoretical_loss": 3.6367570799675875, + "tokens_seen": 1127219200 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003325777331995988, + "loss": 1.3733, + "theoretical_loss": 3.6367570799675875, + "tokens_seen": 1127219200 + }, + { + "epoch": 0.34, + "learning_rate": 0.000332567703109328, + "loss": 1.2587, + "theoretical_loss": 3.6367375556536503, + "tokens_seen": 1127284736 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003325576730190572, + "loss": 1.2666, + "theoretical_loss": 3.636718032792546, + "tokens_seen": 1127350272 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033254764292878637, + "loss": 1.3103, + "theoretical_loss": 3.6366985113840835, + "tokens_seen": 1127415808 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003325376128385156, + "loss": 1.2991, + "theoretical_loss": 3.6366789914280693, + "tokens_seen": 1127481344 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033252758274824473, + "loss": 1.282, + "theoretical_loss": 3.636659472924311, + "tokens_seen": 1127546880 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033251755265797396, + "loss": 1.4379, + "theoretical_loss": 3.636639955872617, + "tokens_seen": 1127612416 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003325075225677031, + "loss": 1.3571, + "theoretical_loss": 3.636620440272794, + "tokens_seen": 1127677952 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033249749247743233, + "loss": 1.4018, + "theoretical_loss": 3.6366009261246495, + "tokens_seen": 1127743488 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003324874623871615, + "loss": 1.3263, + "theoretical_loss": 3.636581413427992, + "tokens_seen": 1127809024 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003324774322968907, + "loss": 1.3622, + "theoretical_loss": 3.6365619021826285, + "tokens_seen": 1127874560 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033246740220661987, + "loss": 1.3409, + "theoretical_loss": 3.6365423923883666, + "tokens_seen": 1127940096 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033245737211634905, + "loss": 1.377, + "theoretical_loss": 3.6365228840450152, + "tokens_seen": 1128005632 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033244734202607823, + "loss": 1.2967, + "theoretical_loss": 3.636503377152381, + "tokens_seen": 1128071168 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033243731193580747, + "loss": 1.3078, + "theoretical_loss": 3.6364838717102725, + "tokens_seen": 1128136704 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003324272818455366, + "loss": 1.3445, + "theoretical_loss": 3.636464367718497, + "tokens_seen": 1128202240 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033241725175526583, + "loss": 1.3423, + "theoretical_loss": 3.636444865176863, + "tokens_seen": 1128267776 + }, + { + "epoch": 0.34, + "learning_rate": 0.000332407221664995, + "loss": 1.3157, + "theoretical_loss": 3.636425364085178, + "tokens_seen": 1128333312 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003323971915747242, + "loss": 1.3183, + "theoretical_loss": 3.63640586444325, + "tokens_seen": 1128398848 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003323871614844534, + "loss": 1.3703, + "theoretical_loss": 3.636386366250888, + "tokens_seen": 1128464384 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033237713139418255, + "loss": 1.4176, + "theoretical_loss": 3.636366869507899, + "tokens_seen": 1128529920 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033236710130391174, + "loss": 1.295, + "theoretical_loss": 3.6363473742140906, + "tokens_seen": 1128595456 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033235707121364097, + "loss": 1.3184, + "theoretical_loss": 3.6363278803692727, + "tokens_seen": 1128660992 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003323470411233701, + "loss": 1.2522, + "theoretical_loss": 3.6363083879732523, + "tokens_seen": 1128726528 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033233701103309933, + "loss": 1.2585, + "theoretical_loss": 3.6362888970258376, + "tokens_seen": 1128792064 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033232698094282846, + "loss": 1.4135, + "theoretical_loss": 3.6362694075268367, + "tokens_seen": 1128857600 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003323169508525577, + "loss": 1.3413, + "theoretical_loss": 3.636249919476059, + "tokens_seen": 1128923136 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003323069207622869, + "loss": 1.3734, + "theoretical_loss": 3.6362304328733117, + "tokens_seen": 1128988672 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033229689067201606, + "loss": 1.4389, + "theoretical_loss": 3.6362109477184035, + "tokens_seen": 1129054208 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033228686058174524, + "loss": 1.4065, + "theoretical_loss": 3.6361914640111435, + "tokens_seen": 1129119744 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003322768304914744, + "loss": 1.2476, + "theoretical_loss": 3.6361719817513385, + "tokens_seen": 1129185280 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003322668004012036, + "loss": 1.4183, + "theoretical_loss": 3.6361525009387985, + "tokens_seen": 1129250816 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033225677031093284, + "loss": 1.3651, + "theoretical_loss": 3.6361330215733316, + "tokens_seen": 1129316352 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033224674022066196, + "loss": 1.3184, + "theoretical_loss": 3.6361135436547456, + "tokens_seen": 1129381888 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003322367101303912, + "loss": 1.283, + "theoretical_loss": 3.63609406718285, + "tokens_seen": 1129447424 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003322266800401204, + "loss": 1.295, + "theoretical_loss": 3.6360745921574527, + "tokens_seen": 1129512960 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033221664994984956, + "loss": 1.3254, + "theoretical_loss": 3.6360551185783634, + "tokens_seen": 1129578496 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033220661985957874, + "loss": 1.3496, + "theoretical_loss": 3.63603564644539, + "tokens_seen": 1129644032 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003321965897693079, + "loss": 1.2779, + "theoretical_loss": 3.636016175758341, + "tokens_seen": 1129709568 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003321865596790371, + "loss": 1.4266, + "theoretical_loss": 3.635996706517026, + "tokens_seen": 1129775104 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033217652958876634, + "loss": 1.3489, + "theoretical_loss": 3.6359772387212526, + "tokens_seen": 1129840640 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033216649949849547, + "loss": 1.2769, + "theoretical_loss": 3.6359577723708307, + "tokens_seen": 1129906176 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003321564694082247, + "loss": 1.3621, + "theoretical_loss": 3.635938307465569, + "tokens_seen": 1129971712 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033214643931795383, + "loss": 1.3475, + "theoretical_loss": 3.6359188440052757, + "tokens_seen": 1130037248 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033213640922768306, + "loss": 1.3758, + "theoretical_loss": 3.6358993819897605, + "tokens_seen": 1130102784 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033212637913741225, + "loss": 1.2813, + "theoretical_loss": 3.635879921418832, + "tokens_seen": 1130168320 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003321163490471414, + "loss": 1.3646, + "theoretical_loss": 3.6358604622922996, + "tokens_seen": 1130233856 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003321063189568706, + "loss": 1.3169, + "theoretical_loss": 3.635841004609972, + "tokens_seen": 1130299392 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033209628886659984, + "loss": 1.3387, + "theoretical_loss": 3.6358215483716587, + "tokens_seen": 1130364928 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033208625877632897, + "loss": 1.3395, + "theoretical_loss": 3.635802093577168, + "tokens_seen": 1130430464 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 379157, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3223717212677002, + "objective/train/theoretical_loss": 3.63578264022631, + "objective/train/tokens_used": 1130496000, + "theoretical_loss": 3.63578264022631, + "tokens_seen": 1130496000 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003320762286860582, + "loss": 1.2817, + "theoretical_loss": 3.63578264022631, + "tokens_seen": 1130496000 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033206619859578733, + "loss": 1.3442, + "theoretical_loss": 3.6357631883188932, + "tokens_seen": 1130561536 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033205616850551657, + "loss": 1.3184, + "theoretical_loss": 3.6357437378547273, + "tokens_seen": 1130627072 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033204613841524575, + "loss": 1.3421, + "theoretical_loss": 3.635724288833621, + "tokens_seen": 1130692608 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033203610832497493, + "loss": 1.2905, + "theoretical_loss": 3.635704841255384, + "tokens_seen": 1130758144 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003320260782347041, + "loss": 1.2936, + "theoretical_loss": 3.6356853951198262, + "tokens_seen": 1130823680 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003320160481444333, + "loss": 1.3473, + "theoretical_loss": 3.6356659504267563, + "tokens_seen": 1130889216 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003320060180541625, + "loss": 1.2989, + "theoretical_loss": 3.6356465071759834, + "tokens_seen": 1130954752 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003319959879638917, + "loss": 1.3231, + "theoretical_loss": 3.6356270653673173, + "tokens_seen": 1131020288 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033198595787362084, + "loss": 1.4203, + "theoretical_loss": 3.635607625000568, + "tokens_seen": 1131085824 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033197592778335007, + "loss": 1.3429, + "theoretical_loss": 3.6355881860755437, + "tokens_seen": 1131151360 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003319658976930792, + "loss": 1.314, + "theoretical_loss": 3.635568748592056, + "tokens_seen": 1131216896 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033195586760280843, + "loss": 1.3083, + "theoretical_loss": 3.6355493125499123, + "tokens_seen": 1131282432 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003319458375125376, + "loss": 1.3933, + "theoretical_loss": 3.6355298779489242, + "tokens_seen": 1131347968 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003319358074222668, + "loss": 1.346, + "theoretical_loss": 3.6355104447889, + "tokens_seen": 1131413504 + }, + { + "epoch": 0.34, + "learning_rate": 0.000331925777331996, + "loss": 1.292, + "theoretical_loss": 3.6354910130696494, + "tokens_seen": 1131479040 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003319157472417252, + "loss": 1.3622, + "theoretical_loss": 3.635471582790983, + "tokens_seen": 1131544576 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033190571715145434, + "loss": 1.2897, + "theoretical_loss": 3.63545215395271, + "tokens_seen": 1131610112 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003318956870611836, + "loss": 1.3218, + "theoretical_loss": 3.6354327265546402, + "tokens_seen": 1131675648 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003318856569709127, + "loss": 1.4306, + "theoretical_loss": 3.635413300596584, + "tokens_seen": 1131741184 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033187562688064194, + "loss": 1.3035, + "theoretical_loss": 3.635393876078351, + "tokens_seen": 1131806720 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003318655967903711, + "loss": 1.2907, + "theoretical_loss": 3.6353744529997503, + "tokens_seen": 1131872256 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003318555667001003, + "loss": 1.2609, + "theoretical_loss": 3.635355031360593, + "tokens_seen": 1131937792 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003318455366098295, + "loss": 1.3425, + "theoretical_loss": 3.6353356111606887, + "tokens_seen": 1132003328 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033183550651955866, + "loss": 1.2588, + "theoretical_loss": 3.6353161923998476, + "tokens_seen": 1132068864 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003318254764292879, + "loss": 1.3129, + "theoretical_loss": 3.6352967750778786, + "tokens_seen": 1132134400 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003318154463390171, + "loss": 1.3285, + "theoretical_loss": 3.6352773591945935, + "tokens_seen": 1132199936 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033180541624874626, + "loss": 1.3794, + "theoretical_loss": 3.6352579447498017, + "tokens_seen": 1132265472 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033179538615847544, + "loss": 1.3211, + "theoretical_loss": 3.6352385317433136, + "tokens_seen": 1132331008 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003317853560682046, + "loss": 1.3627, + "theoretical_loss": 3.6352191201749386, + "tokens_seen": 1132396544 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003317753259779338, + "loss": 1.3456, + "theoretical_loss": 3.6351997100444873, + "tokens_seen": 1132462080 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033176529588766304, + "loss": 1.3614, + "theoretical_loss": 3.6351803013517707, + "tokens_seen": 1132527616 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033175526579739216, + "loss": 1.2447, + "theoretical_loss": 3.6351608940965985, + "tokens_seen": 1132593152 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003317452357071214, + "loss": 1.3, + "theoretical_loss": 3.6351414882787814, + "tokens_seen": 1132658688 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003317352056168506, + "loss": 1.3048, + "theoretical_loss": 3.6351220838981293, + "tokens_seen": 1132724224 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033172517552657976, + "loss": 1.3749, + "theoretical_loss": 3.6351026809544527, + "tokens_seen": 1132789760 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033171514543630894, + "loss": 1.2411, + "theoretical_loss": 3.6350832794475627, + "tokens_seen": 1132855296 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003317051153460381, + "loss": 1.4289, + "theoretical_loss": 3.635063879377269, + "tokens_seen": 1132920832 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003316950852557673, + "loss": 1.2681, + "theoretical_loss": 3.6350444807433826, + "tokens_seen": 1132986368 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033168505516549654, + "loss": 1.2489, + "theoretical_loss": 3.6350250835457136, + "tokens_seen": 1133051904 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033167502507522567, + "loss": 1.2894, + "theoretical_loss": 3.6350056877840733, + "tokens_seen": 1133117440 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003316649949849549, + "loss": 1.346, + "theoretical_loss": 3.6349862934582715, + "tokens_seen": 1133182976 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033165496489468403, + "loss": 1.1925, + "theoretical_loss": 3.63496690056812, + "tokens_seen": 1133248512 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033164493480441326, + "loss": 1.3698, + "theoretical_loss": 3.6349475091134287, + "tokens_seen": 1133314048 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033163490471414245, + "loss": 1.3207, + "theoretical_loss": 3.6349281190940084, + "tokens_seen": 1133379584 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003316248746238716, + "loss": 1.3352, + "theoretical_loss": 3.63490873050967, + "tokens_seen": 1133445120 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003316148445336008, + "loss": 1.3024, + "theoretical_loss": 3.6348893433602245, + "tokens_seen": 1133510656 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033160481444333004, + "loss": 1.2914, + "theoretical_loss": 3.634869957645482, + "tokens_seen": 1133576192 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033159478435305917, + "loss": 1.3062, + "theoretical_loss": 3.634850573365255, + "tokens_seen": 1133641728 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003315847542627884, + "loss": 1.2364, + "theoretical_loss": 3.6348311905193524, + "tokens_seen": 1133707264 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 380162, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3779149055480957, + "objective/train/theoretical_loss": 3.6348118091075867, + "objective/train/tokens_used": 1133772800, + "theoretical_loss": 3.6348118091075867, + "tokens_seen": 1133772800 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033157472417251753, + "loss": 1.3621, + "theoretical_loss": 3.6348118091075867, + "tokens_seen": 1133772800 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033156469408224677, + "loss": 1.275, + "theoretical_loss": 3.634792429129768, + "tokens_seen": 1133838336 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033155466399197595, + "loss": 1.3236, + "theoretical_loss": 3.634773050585708, + "tokens_seen": 1133903872 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033154463390170513, + "loss": 1.3103, + "theoretical_loss": 3.6347536734752177, + "tokens_seen": 1133969408 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003315346038114343, + "loss": 1.2767, + "theoretical_loss": 3.6347342977981074, + "tokens_seen": 1134034944 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003315245737211635, + "loss": 1.4201, + "theoretical_loss": 3.6347149235541893, + "tokens_seen": 1134100480 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003315145436308927, + "loss": 1.3667, + "theoretical_loss": 3.634695550743274, + "tokens_seen": 1134166016 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003315045135406219, + "loss": 1.2838, + "theoretical_loss": 3.6346761793651723, + "tokens_seen": 1134231552 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033149448345035104, + "loss": 1.3726, + "theoretical_loss": 3.6346568094196963, + "tokens_seen": 1134297088 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033148445336008027, + "loss": 1.3484, + "theoretical_loss": 3.634637440906657, + "tokens_seen": 1134362624 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003314744232698094, + "loss": 1.3297, + "theoretical_loss": 3.634618073825866, + "tokens_seen": 1134428160 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033146439317953863, + "loss": 1.2652, + "theoretical_loss": 3.634598708177134, + "tokens_seen": 1134493696 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003314543630892678, + "loss": 1.2549, + "theoretical_loss": 3.6345793439602723, + "tokens_seen": 1134559232 + }, + { + "epoch": 0.34, + "learning_rate": 0.000331444332998997, + "loss": 1.4342, + "theoretical_loss": 3.6345599811750935, + "tokens_seen": 1134624768 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003314343029087262, + "loss": 1.2835, + "theoretical_loss": 3.634540619821408, + "tokens_seen": 1134690304 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003314242728184554, + "loss": 1.2732, + "theoretical_loss": 3.634521259899027, + "tokens_seen": 1134755840 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033141424272818454, + "loss": 1.3809, + "theoretical_loss": 3.6345019014077637, + "tokens_seen": 1134821376 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003314042126379138, + "loss": 1.2523, + "theoretical_loss": 3.634482544347428, + "tokens_seen": 1134886912 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003313941825476429, + "loss": 1.3768, + "theoretical_loss": 3.634463188717832, + "tokens_seen": 1134952448 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033138415245737214, + "loss": 1.2666, + "theoretical_loss": 3.634443834518788, + "tokens_seen": 1135017984 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003313741223671013, + "loss": 1.2506, + "theoretical_loss": 3.6344244817501066, + "tokens_seen": 1135083520 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003313640922768305, + "loss": 1.1947, + "theoretical_loss": 3.6344051304116007, + "tokens_seen": 1135149056 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003313540621865597, + "loss": 1.5073, + "theoretical_loss": 3.634385780503081, + "tokens_seen": 1135214592 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033134403209628886, + "loss": 1.187, + "theoretical_loss": 3.6343664320243594, + "tokens_seen": 1135280128 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033133400200601804, + "loss": 1.3088, + "theoretical_loss": 3.6343470849752486, + "tokens_seen": 1135345664 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003313239719157473, + "loss": 1.3276, + "theoretical_loss": 3.6343277393555597, + "tokens_seen": 1135411200 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003313139418254764, + "loss": 1.3643, + "theoretical_loss": 3.634308395165105, + "tokens_seen": 1135476736 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033130391173520564, + "loss": 1.3285, + "theoretical_loss": 3.6342890524036955, + "tokens_seen": 1135542272 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033129388164493477, + "loss": 1.3584, + "theoretical_loss": 3.6342697110711444, + "tokens_seen": 1135607808 + }, + { + "epoch": 0.34, + "learning_rate": 0.000331283851554664, + "loss": 1.4033, + "theoretical_loss": 3.634250371167263, + "tokens_seen": 1135673344 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003312738214643932, + "loss": 1.3022, + "theoretical_loss": 3.634231032691863, + "tokens_seen": 1135738880 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033126379137412236, + "loss": 1.2794, + "theoretical_loss": 3.634211695644758, + "tokens_seen": 1135804416 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033125376128385155, + "loss": 1.2271, + "theoretical_loss": 3.634192360025758, + "tokens_seen": 1135869952 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003312437311935808, + "loss": 1.3658, + "theoretical_loss": 3.634173025834677, + "tokens_seen": 1135935488 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003312337011033099, + "loss": 1.298, + "theoretical_loss": 3.6341536930713265, + "tokens_seen": 1136001024 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033122367101303914, + "loss": 1.3095, + "theoretical_loss": 3.6341343617355184, + "tokens_seen": 1136066560 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033121364092276827, + "loss": 1.448, + "theoretical_loss": 3.634115031827065, + "tokens_seen": 1136132096 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003312036108324975, + "loss": 1.2965, + "theoretical_loss": 3.6340957033457784, + "tokens_seen": 1136197632 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003311935807422267, + "loss": 1.1765, + "theoretical_loss": 3.6340763762914716, + "tokens_seen": 1136263168 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033118355065195587, + "loss": 1.2499, + "theoretical_loss": 3.634057050663957, + "tokens_seen": 1136328704 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033117352056168505, + "loss": 1.3198, + "theoretical_loss": 3.634037726463046, + "tokens_seen": 1136394240 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033116349047141423, + "loss": 1.3006, + "theoretical_loss": 3.634018403688552, + "tokens_seen": 1136459776 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003311534603811434, + "loss": 1.2018, + "theoretical_loss": 3.633999082340287, + "tokens_seen": 1136525312 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033114343029087265, + "loss": 1.2701, + "theoretical_loss": 3.633979762418063, + "tokens_seen": 1136590848 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033113340020060177, + "loss": 1.3114, + "theoretical_loss": 3.6339604439216937, + "tokens_seen": 1136656384 + }, + { + "epoch": 0.34, + "learning_rate": 0.000331123370110331, + "loss": 1.3938, + "theoretical_loss": 3.633941126850991, + "tokens_seen": 1136721920 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033111334002006013, + "loss": 1.2764, + "theoretical_loss": 3.6339218112057674, + "tokens_seen": 1136787456 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033110330992978937, + "loss": 1.2857, + "theoretical_loss": 3.633902496985836, + "tokens_seen": 1136852992 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033109327983951855, + "loss": 1.2962, + "theoretical_loss": 3.633883184191009, + "tokens_seen": 1136918528 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033108324974924773, + "loss": 1.2293, + "theoretical_loss": 3.6338638728210997, + "tokens_seen": 1136984064 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 381093, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3194715976715088, + "objective/train/theoretical_loss": 3.63384456287592, + "objective/train/tokens_used": 1137049600, + "theoretical_loss": 3.63384456287592, + "tokens_seen": 1137049600 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033107321965897697, + "loss": 1.3098, + "theoretical_loss": 3.63384456287592, + "tokens_seen": 1137049600 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033106318956870615, + "loss": 1.3432, + "theoretical_loss": 3.6338252543552834, + "tokens_seen": 1137115136 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033105315947843533, + "loss": 1.3139, + "theoretical_loss": 3.6338059472590025, + "tokens_seen": 1137180672 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003310431293881645, + "loss": 1.2605, + "theoretical_loss": 3.63378664158689, + "tokens_seen": 1137246208 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003310330992978937, + "loss": 1.2499, + "theoretical_loss": 3.6337673373387585, + "tokens_seen": 1137311744 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003310230692076229, + "loss": 1.2444, + "theoretical_loss": 3.6337480345144217, + "tokens_seen": 1137377280 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003310130391173521, + "loss": 1.3575, + "theoretical_loss": 3.633728733113692, + "tokens_seen": 1137442816 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033100300902708124, + "loss": 1.2687, + "theoretical_loss": 3.6337094331363824, + "tokens_seen": 1137508352 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033099297893681047, + "loss": 1.2936, + "theoretical_loss": 3.6336901345823067, + "tokens_seen": 1137573888 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003309829488465396, + "loss": 1.1643, + "theoretical_loss": 3.633670837451277, + "tokens_seen": 1137639424 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033097291875626883, + "loss": 1.296, + "theoretical_loss": 3.633651541743107, + "tokens_seen": 1137704960 + }, + { + "epoch": 0.34, + "learning_rate": 0.000330962888665998, + "loss": 1.3058, + "theoretical_loss": 3.633632247457609, + "tokens_seen": 1137770496 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003309528585757272, + "loss": 1.3026, + "theoretical_loss": 3.633612954594597, + "tokens_seen": 1137836032 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003309428284854564, + "loss": 1.255, + "theoretical_loss": 3.633593663153884, + "tokens_seen": 1137901568 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003309327983951856, + "loss": 1.3463, + "theoretical_loss": 3.6335743731352834, + "tokens_seen": 1137967104 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033092276830491474, + "loss": 1.2476, + "theoretical_loss": 3.633555084538608, + "tokens_seen": 1138032640 + }, + { + "epoch": 0.34, + "learning_rate": 0.000330912738214644, + "loss": 1.2937, + "theoretical_loss": 3.6335357973636713, + "tokens_seen": 1138098176 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003309027081243731, + "loss": 1.3061, + "theoretical_loss": 3.6335165116102868, + "tokens_seen": 1138163712 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033089267803410234, + "loss": 1.4047, + "theoretical_loss": 3.633497227278268, + "tokens_seen": 1138229248 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003308826479438315, + "loss": 1.2276, + "theoretical_loss": 3.633477944367428, + "tokens_seen": 1138294784 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003308726178535607, + "loss": 1.239, + "theoretical_loss": 3.6334586628775805, + "tokens_seen": 1138360320 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003308625877632899, + "loss": 1.2566, + "theoretical_loss": 3.6334393828085387, + "tokens_seen": 1138425856 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033085255767301906, + "loss": 1.2739, + "theoretical_loss": 3.633420104160116, + "tokens_seen": 1138491392 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033084252758274824, + "loss": 1.2123, + "theoretical_loss": 3.6334008269321263, + "tokens_seen": 1138556928 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003308324974924775, + "loss": 1.2612, + "theoretical_loss": 3.6333815511243834, + "tokens_seen": 1138622464 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003308224674022066, + "loss": 1.319, + "theoretical_loss": 3.633362276736701, + "tokens_seen": 1138688000 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033081243731193584, + "loss": 1.3169, + "theoretical_loss": 3.633343003768892, + "tokens_seen": 1138753536 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033080240722166497, + "loss": 1.3537, + "theoretical_loss": 3.63332373222077, + "tokens_seen": 1138819072 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003307923771313942, + "loss": 1.2624, + "theoretical_loss": 3.63330446209215, + "tokens_seen": 1138884608 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003307823470411234, + "loss": 1.267, + "theoretical_loss": 3.6332851933828447, + "tokens_seen": 1138950144 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033077231695085256, + "loss": 1.3246, + "theoretical_loss": 3.6332659260926685, + "tokens_seen": 1139015680 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033076228686058175, + "loss": 1.339, + "theoretical_loss": 3.6332466602214346, + "tokens_seen": 1139081216 + }, + { + "epoch": 0.35, + "learning_rate": 0.000330752256770311, + "loss": 1.3217, + "theoretical_loss": 3.6332273957689574, + "tokens_seen": 1139146752 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003307422266800401, + "loss": 1.2961, + "theoretical_loss": 3.6332081327350503, + "tokens_seen": 1139212288 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033073219658976934, + "loss": 1.3812, + "theoretical_loss": 3.633188871119528, + "tokens_seen": 1139277824 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033072216649949847, + "loss": 1.3301, + "theoretical_loss": 3.633169610922204, + "tokens_seen": 1139343360 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003307121364092277, + "loss": 1.4105, + "theoretical_loss": 3.6331503521428923, + "tokens_seen": 1139408896 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003307021063189569, + "loss": 1.3305, + "theoretical_loss": 3.6331310947814073, + "tokens_seen": 1139474432 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033069207622868607, + "loss": 1.2796, + "theoretical_loss": 3.6331118388375625, + "tokens_seen": 1139539968 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033068204613841525, + "loss": 1.4174, + "theoretical_loss": 3.6330925843111723, + "tokens_seen": 1139605504 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033067201604814443, + "loss": 1.3065, + "theoretical_loss": 3.633073331202051, + "tokens_seen": 1139671040 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003306619859578736, + "loss": 1.288, + "theoretical_loss": 3.6330540795100124, + "tokens_seen": 1139736576 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033065195586760285, + "loss": 1.3697, + "theoretical_loss": 3.6330348292348713, + "tokens_seen": 1139802112 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033064192577733197, + "loss": 1.1704, + "theoretical_loss": 3.6330155803764415, + "tokens_seen": 1139867648 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003306318956870612, + "loss": 1.2962, + "theoretical_loss": 3.632996332934537, + "tokens_seen": 1139933184 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033062186559679034, + "loss": 1.3919, + "theoretical_loss": 3.6329770869089737, + "tokens_seen": 1139998720 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033061183550651957, + "loss": 1.2954, + "theoretical_loss": 3.6329578422995636, + "tokens_seen": 1140064256 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033060180541624875, + "loss": 1.2753, + "theoretical_loss": 3.632938599106123, + "tokens_seen": 1140129792 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033059177532597793, + "loss": 1.2509, + "theoretical_loss": 3.6329193573284657, + "tokens_seen": 1140195328 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003305817452357071, + "loss": 1.2348, + "theoretical_loss": 3.6329001169664057, + "tokens_seen": 1140260864 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 382191, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.349395513534546, + "objective/train/theoretical_loss": 3.6328808780197583, + "objective/train/tokens_used": 1140326400, + "theoretical_loss": 3.6328808780197583, + "tokens_seen": 1140326400 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033057171514543635, + "loss": 1.2754, + "theoretical_loss": 3.6328808780197583, + "tokens_seen": 1140326400 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003305616850551655, + "loss": 1.2032, + "theoretical_loss": 3.632861640488338, + "tokens_seen": 1140391936 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003305516549648947, + "loss": 1.3156, + "theoretical_loss": 3.6328424043719587, + "tokens_seen": 1140457472 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033054162487462384, + "loss": 1.3886, + "theoretical_loss": 3.632823169670435, + "tokens_seen": 1140523008 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003305315947843531, + "loss": 1.3207, + "theoretical_loss": 3.6328039363835822, + "tokens_seen": 1140588544 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033052156469408225, + "loss": 1.2461, + "theoretical_loss": 3.632784704511215, + "tokens_seen": 1140654080 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033051153460381144, + "loss": 1.2702, + "theoretical_loss": 3.632765474053148, + "tokens_seen": 1140719616 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003305015045135406, + "loss": 1.3082, + "theoretical_loss": 3.632746245009195, + "tokens_seen": 1140785152 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003304914744232698, + "loss": 1.2379, + "theoretical_loss": 3.632727017379172, + "tokens_seen": 1140850688 + }, + { + "epoch": 0.35, + "learning_rate": 0.000330481444332999, + "loss": 1.3651, + "theoretical_loss": 3.632707791162893, + "tokens_seen": 1140916224 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003304714142427282, + "loss": 1.3474, + "theoretical_loss": 3.6326885663601733, + "tokens_seen": 1140981760 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033046138415245734, + "loss": 1.362, + "theoretical_loss": 3.6326693429708277, + "tokens_seen": 1141047296 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003304513540621866, + "loss": 1.2992, + "theoretical_loss": 3.632650120994671, + "tokens_seen": 1141112832 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003304413239719157, + "loss": 1.3236, + "theoretical_loss": 3.6326309004315185, + "tokens_seen": 1141178368 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033043129388164494, + "loss": 1.3326, + "theoretical_loss": 3.632611681281185, + "tokens_seen": 1141243904 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003304212637913741, + "loss": 1.3276, + "theoretical_loss": 3.632592463543485, + "tokens_seen": 1141309440 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003304112337011033, + "loss": 1.2908, + "theoretical_loss": 3.632573247218235, + "tokens_seen": 1141374976 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003304012036108325, + "loss": 1.2427, + "theoretical_loss": 3.6325540323052485, + "tokens_seen": 1141440512 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003303911735205617, + "loss": 1.3281, + "theoretical_loss": 3.6325348188043414, + "tokens_seen": 1141506048 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033038114343029084, + "loss": 1.2708, + "theoretical_loss": 3.632515606715329, + "tokens_seen": 1141571584 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003303711133400201, + "loss": 1.3913, + "theoretical_loss": 3.6324963960380257, + "tokens_seen": 1141637120 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003303610832497492, + "loss": 1.2543, + "theoretical_loss": 3.632477186772248, + "tokens_seen": 1141702656 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033035105315947844, + "loss": 1.4086, + "theoretical_loss": 3.6324579789178095, + "tokens_seen": 1141768192 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003303410230692076, + "loss": 1.3623, + "theoretical_loss": 3.6324387724745275, + "tokens_seen": 1141833728 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003303309929789368, + "loss": 1.3772, + "theoretical_loss": 3.6324195674422155, + "tokens_seen": 1141899264 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033032096288866604, + "loss": 1.2307, + "theoretical_loss": 3.63240036382069, + "tokens_seen": 1141964800 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033031093279839517, + "loss": 1.3248, + "theoretical_loss": 3.632381161609766, + "tokens_seen": 1142030336 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003303009027081244, + "loss": 1.2489, + "theoretical_loss": 3.632361960809259, + "tokens_seen": 1142095872 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003302908726178536, + "loss": 1.289, + "theoretical_loss": 3.632342761418985, + "tokens_seen": 1142161408 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033028084252758276, + "loss": 1.2372, + "theoretical_loss": 3.6323235634387587, + "tokens_seen": 1142226944 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033027081243731195, + "loss": 1.2782, + "theoretical_loss": 3.632304366868396, + "tokens_seen": 1142292480 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003302607823470412, + "loss": 1.4103, + "theoretical_loss": 3.6322851717077125, + "tokens_seen": 1142358016 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003302507522567703, + "loss": 1.3175, + "theoretical_loss": 3.632265977956524, + "tokens_seen": 1142423552 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033024072216649954, + "loss": 1.3702, + "theoretical_loss": 3.6322467856146456, + "tokens_seen": 1142489088 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033023069207622867, + "loss": 1.3261, + "theoretical_loss": 3.6322275946818934, + "tokens_seen": 1142554624 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003302206619859579, + "loss": 1.2709, + "theoretical_loss": 3.632208405158083, + "tokens_seen": 1142620160 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003302106318956871, + "loss": 1.3508, + "theoretical_loss": 3.63218921704303, + "tokens_seen": 1142685696 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033020060180541627, + "loss": 1.3333, + "theoretical_loss": 3.632170030336551, + "tokens_seen": 1142751232 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033019057171514545, + "loss": 1.2065, + "theoretical_loss": 3.6321508450384608, + "tokens_seen": 1142816768 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033018054162487463, + "loss": 1.3608, + "theoretical_loss": 3.6321316611485757, + "tokens_seen": 1142882304 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003301705115346038, + "loss": 1.402, + "theoretical_loss": 3.632112478666712, + "tokens_seen": 1142947840 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033016048144433305, + "loss": 1.3553, + "theoretical_loss": 3.6320932975926845, + "tokens_seen": 1143013376 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003301504513540622, + "loss": 1.3298, + "theoretical_loss": 3.6320741179263103, + "tokens_seen": 1143078912 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003301404212637914, + "loss": 1.4013, + "theoretical_loss": 3.632054939667405, + "tokens_seen": 1143144448 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033013039117352054, + "loss": 1.3251, + "theoretical_loss": 3.6320357628157844, + "tokens_seen": 1143209984 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033012036108324977, + "loss": 1.3402, + "theoretical_loss": 3.6320165873712646, + "tokens_seen": 1143275520 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033011033099297895, + "loss": 1.3867, + "theoretical_loss": 3.6319974133336625, + "tokens_seen": 1143341056 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033010030090270813, + "loss": 1.3164, + "theoretical_loss": 3.6319782407027934, + "tokens_seen": 1143406592 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003300902708124373, + "loss": 1.2554, + "theoretical_loss": 3.631959069478474, + "tokens_seen": 1143472128 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033008024072216655, + "loss": 1.3469, + "theoretical_loss": 3.6319398996605194, + "tokens_seen": 1143537664 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 383129, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2316079139709473, + "objective/train/theoretical_loss": 3.631920731248747, + "objective/train/tokens_used": 1143603200, + "theoretical_loss": 3.631920731248747, + "tokens_seen": 1143603200 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003300702106318957, + "loss": 1.2534, + "theoretical_loss": 3.631920731248747, + "tokens_seen": 1143603200 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003300601805416249, + "loss": 1.2933, + "theoretical_loss": 3.6319015642429733, + "tokens_seen": 1143668736 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033005015045135404, + "loss": 1.3583, + "theoretical_loss": 3.631882398643013, + "tokens_seen": 1143734272 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003300401203610833, + "loss": 1.32, + "theoretical_loss": 3.6318632344486845, + "tokens_seen": 1143799808 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033003009027081245, + "loss": 1.4014, + "theoretical_loss": 3.6318440716598026, + "tokens_seen": 1143865344 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033002006018054164, + "loss": 1.2482, + "theoretical_loss": 3.631824910276184, + "tokens_seen": 1143930880 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003300100300902708, + "loss": 1.3174, + "theoretical_loss": 3.631805750297646, + "tokens_seen": 1143996416 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033, + "loss": 1.3263, + "theoretical_loss": 3.631786591724004, + "tokens_seen": 1144061952 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003299899699097292, + "loss": 1.2306, + "theoretical_loss": 3.6317674345550754, + "tokens_seen": 1144127488 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003299799398194584, + "loss": 1.3849, + "theoretical_loss": 3.6317482787906763, + "tokens_seen": 1144193024 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032996990972918754, + "loss": 1.306, + "theoretical_loss": 3.6317291244306236, + "tokens_seen": 1144258560 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003299598796389168, + "loss": 1.2026, + "theoretical_loss": 3.6317099714747334, + "tokens_seen": 1144324096 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003299498495486459, + "loss": 1.1899, + "theoretical_loss": 3.631690819922823, + "tokens_seen": 1144389632 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032993981945837514, + "loss": 1.3185, + "theoretical_loss": 3.631671669774708, + "tokens_seen": 1144455168 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003299297893681043, + "loss": 1.3279, + "theoretical_loss": 3.6316525210302064, + "tokens_seen": 1144520704 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003299197592778335, + "loss": 1.3251, + "theoretical_loss": 3.6316333736891337, + "tokens_seen": 1144586240 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003299097291875627, + "loss": 1.3407, + "theoretical_loss": 3.6316142277513084, + "tokens_seen": 1144651776 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003298996990972919, + "loss": 1.3156, + "theoretical_loss": 3.6315950832165456, + "tokens_seen": 1144717312 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032988966900702104, + "loss": 1.3663, + "theoretical_loss": 3.6315759400846632, + "tokens_seen": 1144782848 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003298796389167503, + "loss": 1.3061, + "theoretical_loss": 3.631556798355478, + "tokens_seen": 1144848384 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003298696088264794, + "loss": 1.3865, + "theoretical_loss": 3.6315376580288063, + "tokens_seen": 1144913920 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032985957873620864, + "loss": 1.2582, + "theoretical_loss": 3.631518519104466, + "tokens_seen": 1144979456 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003298495486459378, + "loss": 1.3194, + "theoretical_loss": 3.631499381582273, + "tokens_seen": 1145044992 + }, + { + "epoch": 0.35, + "learning_rate": 0.000329839518555667, + "loss": 1.3961, + "theoretical_loss": 3.6314802454620456, + "tokens_seen": 1145110528 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003298294884653962, + "loss": 1.3283, + "theoretical_loss": 3.6314611107436, + "tokens_seen": 1145176064 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032981945837512537, + "loss": 1.3342, + "theoretical_loss": 3.6314419774267535, + "tokens_seen": 1145241600 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032980942828485455, + "loss": 1.2995, + "theoretical_loss": 3.631422845511323, + "tokens_seen": 1145307136 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003297993981945838, + "loss": 1.2794, + "theoretical_loss": 3.6314037149971257, + "tokens_seen": 1145372672 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003297893681043129, + "loss": 1.3657, + "theoretical_loss": 3.6313845858839793, + "tokens_seen": 1145438208 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032977933801404215, + "loss": 1.3607, + "theoretical_loss": 3.6313654581717008, + "tokens_seen": 1145503744 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003297693079237713, + "loss": 1.2842, + "theoretical_loss": 3.631346331860107, + "tokens_seen": 1145569280 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003297592778335005, + "loss": 1.2933, + "theoretical_loss": 3.631327206949016, + "tokens_seen": 1145634816 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003297492477432297, + "loss": 1.3139, + "theoretical_loss": 3.6313080834382445, + "tokens_seen": 1145700352 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032973921765295887, + "loss": 1.3271, + "theoretical_loss": 3.631288961327611, + "tokens_seen": 1145765888 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032972918756268805, + "loss": 1.2109, + "theoretical_loss": 3.6312698406169313, + "tokens_seen": 1145831424 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003297191574724173, + "loss": 1.2885, + "theoretical_loss": 3.6312507213060234, + "tokens_seen": 1145896960 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003297091273821464, + "loss": 1.3517, + "theoretical_loss": 3.6312316033947054, + "tokens_seen": 1145962496 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032969909729187565, + "loss": 1.3185, + "theoretical_loss": 3.6312124868827946, + "tokens_seen": 1146028032 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003296890672016048, + "loss": 1.4179, + "theoretical_loss": 3.631193371770108, + "tokens_seen": 1146093568 + }, + { + "epoch": 0.35, + "learning_rate": 0.000329679037111334, + "loss": 1.2476, + "theoretical_loss": 3.631174258056464, + "tokens_seen": 1146159104 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003296690070210632, + "loss": 1.3513, + "theoretical_loss": 3.6311551457416793, + "tokens_seen": 1146224640 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003296589769307924, + "loss": 1.3242, + "theoretical_loss": 3.6311360348255723, + "tokens_seen": 1146290176 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032964894684052155, + "loss": 1.2407, + "theoretical_loss": 3.63111692530796, + "tokens_seen": 1146355712 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032963891675025074, + "loss": 1.2359, + "theoretical_loss": 3.6310978171886608, + "tokens_seen": 1146421248 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003296288866599799, + "loss": 1.1952, + "theoretical_loss": 3.631078710467492, + "tokens_seen": 1146486784 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032961885656970915, + "loss": 1.2022, + "theoretical_loss": 3.6310596051442716, + "tokens_seen": 1146552320 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003296088264794383, + "loss": 1.3222, + "theoretical_loss": 3.6310405012188176, + "tokens_seen": 1146617856 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003295987963891675, + "loss": 1.225, + "theoretical_loss": 3.6310213986909474, + "tokens_seen": 1146683392 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003295887662988967, + "loss": 1.3632, + "theoretical_loss": 3.631002297560479, + "tokens_seen": 1146748928 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003295787362086259, + "loss": 1.2482, + "theoretical_loss": 3.6309831978272307, + "tokens_seen": 1146814464 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 384318, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2261451482772827, + "objective/train/theoretical_loss": 3.63096409949102, + "objective/train/tokens_used": 1146880000, + "theoretical_loss": 3.63096409949102, + "tokens_seen": 1146880000 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003295687061183551, + "loss": 1.2458, + "theoretical_loss": 3.63096409949102, + "tokens_seen": 1146880000 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032955867602808424, + "loss": 1.3252, + "theoretical_loss": 3.630945002551665, + "tokens_seen": 1146945536 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003295486459378135, + "loss": 1.235, + "theoretical_loss": 3.630925907008984, + "tokens_seen": 1147011072 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032953861584754265, + "loss": 1.2662, + "theoretical_loss": 3.6309068128627953, + "tokens_seen": 1147076608 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032952858575727184, + "loss": 1.3616, + "theoretical_loss": 3.630887720112916, + "tokens_seen": 1147142144 + }, + { + "epoch": 0.35, + "learning_rate": 0.000329518555667001, + "loss": 1.3043, + "theoretical_loss": 3.630868628759165, + "tokens_seen": 1147207680 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003295085255767302, + "loss": 1.3161, + "theoretical_loss": 3.6308495388013604, + "tokens_seen": 1147273216 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003294984954864594, + "loss": 1.4151, + "theoretical_loss": 3.6308304502393196, + "tokens_seen": 1147338752 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003294884653961886, + "loss": 1.3344, + "theoretical_loss": 3.630811363072862, + "tokens_seen": 1147404288 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032947843530591774, + "loss": 1.3325, + "theoretical_loss": 3.6307922773018055, + "tokens_seen": 1147469824 + }, + { + "epoch": 0.35, + "learning_rate": 0.000329468405215647, + "loss": 1.2495, + "theoretical_loss": 3.630773192925968, + "tokens_seen": 1147535360 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003294583751253761, + "loss": 1.3133, + "theoretical_loss": 3.630754109945168, + "tokens_seen": 1147600896 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032944834503510534, + "loss": 1.3491, + "theoretical_loss": 3.6307350283592243, + "tokens_seen": 1147666432 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003294383149448345, + "loss": 1.2927, + "theoretical_loss": 3.6307159481679547, + "tokens_seen": 1147731968 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003294282848545637, + "loss": 1.3039, + "theoretical_loss": 3.630696869371178, + "tokens_seen": 1147797504 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003294182547642929, + "loss": 1.3681, + "theoretical_loss": 3.6306777919687128, + "tokens_seen": 1147863040 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003294082246740221, + "loss": 1.3154, + "theoretical_loss": 3.630658715960377, + "tokens_seen": 1147928576 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032939819458375124, + "loss": 1.2523, + "theoretical_loss": 3.63063964134599, + "tokens_seen": 1147994112 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003293881644934805, + "loss": 1.3055, + "theoretical_loss": 3.6306205681253694, + "tokens_seen": 1148059648 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003293781344032096, + "loss": 1.1829, + "theoretical_loss": 3.6306014962983344, + "tokens_seen": 1148125184 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032936810431293884, + "loss": 1.3197, + "theoretical_loss": 3.6305824258647035, + "tokens_seen": 1148190720 + }, + { + "epoch": 0.35, + "learning_rate": 0.000329358074222668, + "loss": 1.3105, + "theoretical_loss": 3.6305633568242954, + "tokens_seen": 1148256256 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003293480441323972, + "loss": 1.2766, + "theoretical_loss": 3.630544289176929, + "tokens_seen": 1148321792 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003293380140421264, + "loss": 1.3318, + "theoretical_loss": 3.6305252229224223, + "tokens_seen": 1148387328 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032932798395185557, + "loss": 1.435, + "theoretical_loss": 3.6305061580605953, + "tokens_seen": 1148452864 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032931795386158475, + "loss": 1.3616, + "theoretical_loss": 3.630487094591266, + "tokens_seen": 1148518400 + }, + { + "epoch": 0.35, + "learning_rate": 0.000329307923771314, + "loss": 1.2581, + "theoretical_loss": 3.630468032514253, + "tokens_seen": 1148583936 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003292978936810431, + "loss": 1.341, + "theoretical_loss": 3.6304489718293755, + "tokens_seen": 1148649472 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032928786359077235, + "loss": 1.4271, + "theoretical_loss": 3.630429912536453, + "tokens_seen": 1148715008 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003292778335005015, + "loss": 1.3245, + "theoretical_loss": 3.6304108546353033, + "tokens_seen": 1148780544 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003292678034102307, + "loss": 1.3042, + "theoretical_loss": 3.6303917981257467, + "tokens_seen": 1148846080 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003292577733199599, + "loss": 1.2603, + "theoretical_loss": 3.6303727430076007, + "tokens_seen": 1148911616 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032924774322968907, + "loss": 1.173, + "theoretical_loss": 3.630353689280686, + "tokens_seen": 1148977152 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032923771313941825, + "loss": 1.3626, + "theoretical_loss": 3.63033463694482, + "tokens_seen": 1149042688 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003292276830491475, + "loss": 1.3003, + "theoretical_loss": 3.6303155859998233, + "tokens_seen": 1149108224 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003292176529588766, + "loss": 1.3221, + "theoretical_loss": 3.630296536445514, + "tokens_seen": 1149173760 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032920762286860585, + "loss": 1.3282, + "theoretical_loss": 3.630277488281712, + "tokens_seen": 1149239296 + }, + { + "epoch": 0.35, + "learning_rate": 0.000329197592778335, + "loss": 1.3491, + "theoretical_loss": 3.6302584415082357, + "tokens_seen": 1149304832 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003291875626880642, + "loss": 1.3945, + "theoretical_loss": 3.630239396124905, + "tokens_seen": 1149370368 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003291775325977934, + "loss": 1.321, + "theoretical_loss": 3.630220352131539, + "tokens_seen": 1149435904 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003291675025075226, + "loss": 1.327, + "theoretical_loss": 3.6302013095279575, + "tokens_seen": 1149501440 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032915747241725175, + "loss": 1.3225, + "theoretical_loss": 3.6301822683139786, + "tokens_seen": 1149566976 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032914744232698094, + "loss": 1.3051, + "theoretical_loss": 3.6301632284894225, + "tokens_seen": 1149632512 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003291374122367101, + "loss": 1.4156, + "theoretical_loss": 3.630144190054109, + "tokens_seen": 1149698048 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032912738214643935, + "loss": 1.3408, + "theoretical_loss": 3.6301251530078575, + "tokens_seen": 1149763584 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003291173520561685, + "loss": 1.1908, + "theoretical_loss": 3.6301061173504863, + "tokens_seen": 1149829120 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003291073219658977, + "loss": 1.2892, + "theoretical_loss": 3.6300870830818166, + "tokens_seen": 1149894656 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003290972918756269, + "loss": 1.3297, + "theoretical_loss": 3.6300680502016665, + "tokens_seen": 1149960192 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003290872617853561, + "loss": 1.4119, + "theoretical_loss": 3.630049018709856, + "tokens_seen": 1150025728 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032907723169508526, + "loss": 1.2954, + "theoretical_loss": 3.6300299886062053, + "tokens_seen": 1150091264 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 385229, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.201535940170288, + "objective/train/theoretical_loss": 3.6300109598905332, + "objective/train/tokens_used": 1150156800, + "theoretical_loss": 3.6300109598905332, + "tokens_seen": 1150156800 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032906720160481444, + "loss": 1.2651, + "theoretical_loss": 3.6300109598905332, + "tokens_seen": 1150156800 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003290571715145436, + "loss": 1.2591, + "theoretical_loss": 3.6299919325626604, + "tokens_seen": 1150222336 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032904714142427286, + "loss": 1.2407, + "theoretical_loss": 3.629972906622406, + "tokens_seen": 1150287872 + }, + { + "epoch": 0.35, + "learning_rate": 0.000329037111334002, + "loss": 1.3549, + "theoretical_loss": 3.6299538820695894, + "tokens_seen": 1150353408 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003290270812437312, + "loss": 1.239, + "theoretical_loss": 3.629934858904031, + "tokens_seen": 1150418944 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032901705115346034, + "loss": 1.331, + "theoretical_loss": 3.6299158371255507, + "tokens_seen": 1150484480 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003290070210631896, + "loss": 1.3631, + "theoretical_loss": 3.629896816733967, + "tokens_seen": 1150550016 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032899699097291876, + "loss": 1.3843, + "theoretical_loss": 3.6298777977291024, + "tokens_seen": 1150615552 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032898696088264794, + "loss": 1.3514, + "theoretical_loss": 3.6298587801107747, + "tokens_seen": 1150681088 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003289769307923771, + "loss": 1.3789, + "theoretical_loss": 3.6298397638788042, + "tokens_seen": 1150746624 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003289669007021063, + "loss": 1.2569, + "theoretical_loss": 3.629820749033011, + "tokens_seen": 1150812160 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003289568706118355, + "loss": 1.3343, + "theoretical_loss": 3.6298017355732157, + "tokens_seen": 1150877696 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003289468405215647, + "loss": 1.3631, + "theoretical_loss": 3.629782723499238, + "tokens_seen": 1150943232 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032893681043129385, + "loss": 1.2898, + "theoretical_loss": 3.629763712810898, + "tokens_seen": 1151008768 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003289267803410231, + "loss": 1.3436, + "theoretical_loss": 3.629744703508015, + "tokens_seen": 1151074304 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032891675025075226, + "loss": 1.2649, + "theoretical_loss": 3.6297256955904107, + "tokens_seen": 1151139840 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032890672016048145, + "loss": 1.222, + "theoretical_loss": 3.6297066890579037, + "tokens_seen": 1151205376 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003288966900702106, + "loss": 1.3333, + "theoretical_loss": 3.6296876839103156, + "tokens_seen": 1151270912 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003288866599799398, + "loss": 1.2532, + "theoretical_loss": 3.629668680147466, + "tokens_seen": 1151336448 + }, + { + "epoch": 0.35, + "learning_rate": 0.000328876629889669, + "loss": 1.4233, + "theoretical_loss": 3.6296496777691756, + "tokens_seen": 1151401984 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003288665997993982, + "loss": 1.2443, + "theoretical_loss": 3.629630676775264, + "tokens_seen": 1151467520 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032885656970912735, + "loss": 1.2394, + "theoretical_loss": 3.6296116771655518, + "tokens_seen": 1151533056 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003288465396188566, + "loss": 1.4107, + "theoretical_loss": 3.62959267893986, + "tokens_seen": 1151598592 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003288365095285857, + "loss": 1.4144, + "theoretical_loss": 3.6295736820980085, + "tokens_seen": 1151664128 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032882647943831495, + "loss": 1.2691, + "theoretical_loss": 3.6295546866398176, + "tokens_seen": 1151729664 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003288164493480442, + "loss": 1.2579, + "theoretical_loss": 3.629535692565108, + "tokens_seen": 1151795200 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003288064192577733, + "loss": 1.3445, + "theoretical_loss": 3.6295166998737005, + "tokens_seen": 1151860736 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032879638916750255, + "loss": 1.2367, + "theoretical_loss": 3.6294977085654154, + "tokens_seen": 1151926272 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003287863590772317, + "loss": 1.3294, + "theoretical_loss": 3.6294787186400734, + "tokens_seen": 1151991808 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003287763289869609, + "loss": 1.3205, + "theoretical_loss": 3.629459730097495, + "tokens_seen": 1152057344 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003287662988966901, + "loss": 1.2594, + "theoretical_loss": 3.6294407429375006, + "tokens_seen": 1152122880 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032875626880641927, + "loss": 1.2063, + "theoretical_loss": 3.6294217571599114, + "tokens_seen": 1152188416 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032874623871614845, + "loss": 1.4139, + "theoretical_loss": 3.629402772764548, + "tokens_seen": 1152253952 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003287362086258777, + "loss": 1.2639, + "theoretical_loss": 3.6293837897512313, + "tokens_seen": 1152319488 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003287261785356068, + "loss": 1.3313, + "theoretical_loss": 3.6293648081197816, + "tokens_seen": 1152385024 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032871614844533605, + "loss": 1.3323, + "theoretical_loss": 3.6293458278700204, + "tokens_seen": 1152450560 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003287061183550652, + "loss": 1.3445, + "theoretical_loss": 3.6293268490017674, + "tokens_seen": 1152516096 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003286960882647944, + "loss": 1.352, + "theoretical_loss": 3.629307871514845, + "tokens_seen": 1152581632 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003286860581745236, + "loss": 1.2139, + "theoretical_loss": 3.6292888954090734, + "tokens_seen": 1152647168 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003286760280842528, + "loss": 1.2408, + "theoretical_loss": 3.629269920684273, + "tokens_seen": 1152712704 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032866599799398195, + "loss": 1.1888, + "theoretical_loss": 3.6292509473402657, + "tokens_seen": 1152778240 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032865596790371114, + "loss": 1.3784, + "theoretical_loss": 3.6292319753768725, + "tokens_seen": 1152843776 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003286459378134403, + "loss": 1.4314, + "theoretical_loss": 3.6292130047939137, + "tokens_seen": 1152909312 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032863590772316955, + "loss": 1.3399, + "theoretical_loss": 3.6291940355912113, + "tokens_seen": 1152974848 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003286258776328987, + "loss": 1.4331, + "theoretical_loss": 3.6291750677685854, + "tokens_seen": 1153040384 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003286158475426279, + "loss": 1.4296, + "theoretical_loss": 3.6291561013258584, + "tokens_seen": 1153105920 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003286058174523571, + "loss": 1.2454, + "theoretical_loss": 3.629137136262851, + "tokens_seen": 1153171456 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003285957873620863, + "loss": 1.366, + "theoretical_loss": 3.6291181725793833, + "tokens_seen": 1153236992 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032858575727181546, + "loss": 1.3476, + "theoretical_loss": 3.629099210275278, + "tokens_seen": 1153302528 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032857572718154464, + "loss": 1.2834, + "theoretical_loss": 3.6290802493503564, + "tokens_seen": 1153368064 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 386389, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3444721698760986, + "objective/train/theoretical_loss": 3.6290612898044388, + "objective/train/tokens_used": 1153433600, + "theoretical_loss": 3.6290612898044388, + "tokens_seen": 1153433600 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003285656970912738, + "loss": 1.3599, + "theoretical_loss": 3.6290612898044388, + "tokens_seen": 1153433600 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032855566700100306, + "loss": 1.3234, + "theoretical_loss": 3.6290423316373475, + "tokens_seen": 1153499136 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003285456369107322, + "loss": 1.2952, + "theoretical_loss": 3.629023374848903, + "tokens_seen": 1153564672 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003285356068204614, + "loss": 1.2854, + "theoretical_loss": 3.6290044194389277, + "tokens_seen": 1153630208 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032852557673019054, + "loss": 1.288, + "theoretical_loss": 3.6289854654072426, + "tokens_seen": 1153695744 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003285155466399198, + "loss": 1.2822, + "theoretical_loss": 3.628966512753669, + "tokens_seen": 1153761280 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032850551654964896, + "loss": 1.3805, + "theoretical_loss": 3.6289475614780287, + "tokens_seen": 1153826816 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032849548645937814, + "loss": 1.2338, + "theoretical_loss": 3.628928611580143, + "tokens_seen": 1153892352 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003284854563691073, + "loss": 1.2726, + "theoretical_loss": 3.628909663059834, + "tokens_seen": 1153957888 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003284754262788365, + "loss": 1.3166, + "theoretical_loss": 3.6288907159169232, + "tokens_seen": 1154023424 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003284653961885657, + "loss": 1.3049, + "theoretical_loss": 3.628871770151232, + "tokens_seen": 1154088960 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003284553660982949, + "loss": 1.2976, + "theoretical_loss": 3.628852825762582, + "tokens_seen": 1154154496 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032844533600802405, + "loss": 1.346, + "theoretical_loss": 3.6288338827507953, + "tokens_seen": 1154220032 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003284353059177533, + "loss": 1.3137, + "theoretical_loss": 3.628814941115693, + "tokens_seen": 1154285568 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032842527582748246, + "loss": 1.3396, + "theoretical_loss": 3.628796000857098, + "tokens_seen": 1154351104 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032841524573721165, + "loss": 1.3145, + "theoretical_loss": 3.628777061974831, + "tokens_seen": 1154416640 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003284052156469408, + "loss": 1.3154, + "theoretical_loss": 3.628758124468715, + "tokens_seen": 1154482176 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032839518555667, + "loss": 1.2784, + "theoretical_loss": 3.6287391883385705, + "tokens_seen": 1154547712 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003283851554663992, + "loss": 1.2543, + "theoretical_loss": 3.6287202535842207, + "tokens_seen": 1154613248 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003283751253761284, + "loss": 1.4478, + "theoretical_loss": 3.6287013202054865, + "tokens_seen": 1154678784 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032836509528585755, + "loss": 1.3018, + "theoretical_loss": 3.628682388202191, + "tokens_seen": 1154744320 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003283550651955868, + "loss": 1.3422, + "theoretical_loss": 3.628663457574155, + "tokens_seen": 1154809856 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003283450351053159, + "loss": 1.1978, + "theoretical_loss": 3.628644528321202, + "tokens_seen": 1154875392 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032833500501504515, + "loss": 1.3003, + "theoretical_loss": 3.628625600443153, + "tokens_seen": 1154940928 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032832497492477433, + "loss": 1.4102, + "theoretical_loss": 3.6286066739398306, + "tokens_seen": 1155006464 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003283149448345035, + "loss": 1.2584, + "theoretical_loss": 3.6285877488110563, + "tokens_seen": 1155072000 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003283049147442327, + "loss": 1.3344, + "theoretical_loss": 3.6285688250566532, + "tokens_seen": 1155137536 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003282948846539619, + "loss": 1.1573, + "theoretical_loss": 3.6285499026764434, + "tokens_seen": 1155203072 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032828485456369105, + "loss": 1.4001, + "theoretical_loss": 3.6285309816702482, + "tokens_seen": 1155268608 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003282748244734203, + "loss": 1.3039, + "theoretical_loss": 3.628512062037891, + "tokens_seen": 1155334144 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003282647943831494, + "loss": 1.3127, + "theoretical_loss": 3.6284931437791936, + "tokens_seen": 1155399680 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032825476429287865, + "loss": 1.2437, + "theoretical_loss": 3.628474226893978, + "tokens_seen": 1155465216 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032824473420260783, + "loss": 1.3464, + "theoretical_loss": 3.628455311382068, + "tokens_seen": 1155530752 + }, + { + "epoch": 0.35, + "learning_rate": 0.000328234704112337, + "loss": 1.4179, + "theoretical_loss": 3.6284363972432843, + "tokens_seen": 1155596288 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003282246740220662, + "loss": 1.3036, + "theoretical_loss": 3.628417484477451, + "tokens_seen": 1155661824 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003282146439317954, + "loss": 1.3049, + "theoretical_loss": 3.6283985730843886, + "tokens_seen": 1155727360 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032820461384152456, + "loss": 1.3432, + "theoretical_loss": 3.628379663063921, + "tokens_seen": 1155792896 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003281945837512538, + "loss": 1.3305, + "theoretical_loss": 3.628360754415871, + "tokens_seen": 1155858432 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003281845536609829, + "loss": 1.3335, + "theoretical_loss": 3.62834184714006, + "tokens_seen": 1155923968 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032817452357071215, + "loss": 1.3372, + "theoretical_loss": 3.628322941236312, + "tokens_seen": 1155989504 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003281644934804413, + "loss": 1.2572, + "theoretical_loss": 3.6283040367044483, + "tokens_seen": 1156055040 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003281544633901705, + "loss": 1.2652, + "theoretical_loss": 3.628285133544293, + "tokens_seen": 1156120576 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003281444332998997, + "loss": 1.2269, + "theoretical_loss": 3.6282662317556675, + "tokens_seen": 1156186112 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003281344032096289, + "loss": 1.3406, + "theoretical_loss": 3.6282473313383954, + "tokens_seen": 1156251648 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032812437311935806, + "loss": 1.3491, + "theoretical_loss": 3.628228432292299, + "tokens_seen": 1156317184 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003281143430290873, + "loss": 1.3067, + "theoretical_loss": 3.628209534617201, + "tokens_seen": 1156382720 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003281043129388164, + "loss": 1.2603, + "theoretical_loss": 3.628190638312925, + "tokens_seen": 1156448256 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032809428284854566, + "loss": 1.3757, + "theoretical_loss": 3.628171743379294, + "tokens_seen": 1156513792 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003280842527582748, + "loss": 1.407, + "theoretical_loss": 3.62815284981613, + "tokens_seen": 1156579328 + }, + { + "epoch": 0.35, + "learning_rate": 0.000328074222668004, + "loss": 1.3361, + "theoretical_loss": 3.6281339576232563, + "tokens_seen": 1156644864 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 387574, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.331512212753296, + "objective/train/theoretical_loss": 3.628115066800496, + "objective/train/tokens_used": 1156710400, + "theoretical_loss": 3.628115066800496, + "tokens_seen": 1156710400 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032806419257773326, + "loss": 1.3915, + "theoretical_loss": 3.628115066800496, + "tokens_seen": 1156710400 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003280541624874624, + "loss": 1.3947, + "theoretical_loss": 3.628096177347672, + "tokens_seen": 1156775936 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003280441323971916, + "loss": 1.2334, + "theoretical_loss": 3.6280772892646067, + "tokens_seen": 1156841472 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032803410230692074, + "loss": 1.4595, + "theoretical_loss": 3.6280584025511247, + "tokens_seen": 1156907008 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032802407221665, + "loss": 1.2934, + "theoretical_loss": 3.628039517207048, + "tokens_seen": 1156972544 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032801404212637916, + "loss": 1.3235, + "theoretical_loss": 3.6280206332322, + "tokens_seen": 1157038080 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032800401203610834, + "loss": 1.3808, + "theoretical_loss": 3.628001750626404, + "tokens_seen": 1157103616 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003279939819458375, + "loss": 1.2989, + "theoretical_loss": 3.627982869389483, + "tokens_seen": 1157169152 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003279839518555667, + "loss": 1.3641, + "theoretical_loss": 3.6279639895212608, + "tokens_seen": 1157234688 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003279739217652959, + "loss": 1.3106, + "theoretical_loss": 3.6279451110215595, + "tokens_seen": 1157300224 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003279638916750251, + "loss": 1.2354, + "theoretical_loss": 3.6279262338902036, + "tokens_seen": 1157365760 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032795386158475425, + "loss": 1.2103, + "theoretical_loss": 3.6279073581270165, + "tokens_seen": 1157431296 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003279438314944835, + "loss": 1.3089, + "theoretical_loss": 3.6278884837318204, + "tokens_seen": 1157496832 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032793380140421266, + "loss": 1.2739, + "theoretical_loss": 3.6278696107044395, + "tokens_seen": 1157562368 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032792377131394185, + "loss": 1.3018, + "theoretical_loss": 3.6278507390446975, + "tokens_seen": 1157627904 + }, + { + "epoch": 0.35, + "learning_rate": 0.000327913741223671, + "loss": 1.3531, + "theoretical_loss": 3.627831868752417, + "tokens_seen": 1157693440 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003279037111334002, + "loss": 1.3263, + "theoretical_loss": 3.6278129998274227, + "tokens_seen": 1157758976 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003278936810431294, + "loss": 1.2693, + "theoretical_loss": 3.6277941322695373, + "tokens_seen": 1157824512 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003278836509528586, + "loss": 1.2884, + "theoretical_loss": 3.627775266078584, + "tokens_seen": 1157890048 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032787362086258775, + "loss": 1.3909, + "theoretical_loss": 3.6277564012543877, + "tokens_seen": 1157955584 + }, + { + "epoch": 0.35, + "learning_rate": 0.000327863590772317, + "loss": 1.2708, + "theoretical_loss": 3.6277375377967704, + "tokens_seen": 1158021120 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003278535606820461, + "loss": 1.345, + "theoretical_loss": 3.627718675705558, + "tokens_seen": 1158086656 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032784353059177535, + "loss": 1.3341, + "theoretical_loss": 3.6276998149805717, + "tokens_seen": 1158152192 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032783350050150453, + "loss": 1.3972, + "theoretical_loss": 3.627680955621637, + "tokens_seen": 1158217728 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003278234704112337, + "loss": 1.3151, + "theoretical_loss": 3.627662097628577, + "tokens_seen": 1158283264 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003278134403209629, + "loss": 1.3494, + "theoretical_loss": 3.6276432410012154, + "tokens_seen": 1158348800 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032780341023069213, + "loss": 1.3725, + "theoretical_loss": 3.6276243857393764, + "tokens_seen": 1158414336 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032779338014042125, + "loss": 1.3489, + "theoretical_loss": 3.627605531842884, + "tokens_seen": 1158479872 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003277833500501505, + "loss": 1.2079, + "theoretical_loss": 3.6275866793115616, + "tokens_seen": 1158545408 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003277733199598796, + "loss": 1.2609, + "theoretical_loss": 3.6275678281452333, + "tokens_seen": 1158610944 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032776328986960885, + "loss": 1.3863, + "theoretical_loss": 3.627548978343723, + "tokens_seen": 1158676480 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032775325977933803, + "loss": 1.3746, + "theoretical_loss": 3.627530129906855, + "tokens_seen": 1158742016 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003277432296890672, + "loss": 1.2633, + "theoretical_loss": 3.627511282834453, + "tokens_seen": 1158807552 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003277331995987964, + "loss": 1.2995, + "theoretical_loss": 3.6274924371263415, + "tokens_seen": 1158873088 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003277231695085256, + "loss": 1.3432, + "theoretical_loss": 3.627473592782344, + "tokens_seen": 1158938624 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032771313941825476, + "loss": 1.3892, + "theoretical_loss": 3.627454749802285, + "tokens_seen": 1159004160 + }, + { + "epoch": 0.35, + "learning_rate": 0.000327703109327984, + "loss": 1.365, + "theoretical_loss": 3.6274359081859884, + "tokens_seen": 1159069696 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003276930792377131, + "loss": 1.2757, + "theoretical_loss": 3.627417067933279, + "tokens_seen": 1159135232 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032768304914744235, + "loss": 1.4015, + "theoretical_loss": 3.6273982290439806, + "tokens_seen": 1159200768 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003276730190571715, + "loss": 1.3194, + "theoretical_loss": 3.627379391517917, + "tokens_seen": 1159266304 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003276629889669007, + "loss": 1.249, + "theoretical_loss": 3.6273605553549135, + "tokens_seen": 1159331840 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003276529588766299, + "loss": 1.3167, + "theoretical_loss": 3.6273417205547935, + "tokens_seen": 1159397376 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003276429287863591, + "loss": 1.2452, + "theoretical_loss": 3.627322887117382, + "tokens_seen": 1159462912 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032763289869608826, + "loss": 1.3783, + "theoretical_loss": 3.6273040550425035, + "tokens_seen": 1159528448 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003276228686058175, + "loss": 1.3153, + "theoretical_loss": 3.627285224329982, + "tokens_seen": 1159593984 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003276128385155466, + "loss": 1.2514, + "theoretical_loss": 3.6272663949796415, + "tokens_seen": 1159659520 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032760280842527586, + "loss": 1.3222, + "theoretical_loss": 3.6272475669913073, + "tokens_seen": 1159725056 + }, + { + "epoch": 0.35, + "learning_rate": 0.000327592778335005, + "loss": 1.2112, + "theoretical_loss": 3.627228740364804, + "tokens_seen": 1159790592 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003275827482447342, + "loss": 1.4033, + "theoretical_loss": 3.6272099150999555, + "tokens_seen": 1159856128 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003275727181544634, + "loss": 1.4595, + "theoretical_loss": 3.627191091196586, + "tokens_seen": 1159921664 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 388554, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4284337759017944, + "objective/train/theoretical_loss": 3.6271722686545216, + "objective/train/tokens_used": 1159987200, + "theoretical_loss": 3.6271722686545216, + "tokens_seen": 1159987200 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003275626880641926, + "loss": 1.3859, + "theoretical_loss": 3.6271722686545216, + "tokens_seen": 1159987200 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032755265797392176, + "loss": 1.283, + "theoretical_loss": 3.6271534474735865, + "tokens_seen": 1160052736 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032754262788365094, + "loss": 1.2882, + "theoretical_loss": 3.6271346276536045, + "tokens_seen": 1160118272 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003275325977933801, + "loss": 1.299, + "theoretical_loss": 3.627115809194401, + "tokens_seen": 1160183808 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032752256770310936, + "loss": 1.3337, + "theoretical_loss": 3.627096992095801, + "tokens_seen": 1160249344 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003275125376128385, + "loss": 1.2427, + "theoretical_loss": 3.627078176357628, + "tokens_seen": 1160314880 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003275025075225677, + "loss": 1.2024, + "theoretical_loss": 3.6270593619797085, + "tokens_seen": 1160380416 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032749247743229685, + "loss": 1.365, + "theoretical_loss": 3.6270405489618662, + "tokens_seen": 1160445952 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003274824473420261, + "loss": 1.2611, + "theoretical_loss": 3.6270217373039264, + "tokens_seen": 1160511488 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032747241725175527, + "loss": 1.3137, + "theoretical_loss": 3.627002927005714, + "tokens_seen": 1160577024 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032746238716148445, + "loss": 1.3236, + "theoretical_loss": 3.626984118067054, + "tokens_seen": 1160642560 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032745235707121363, + "loss": 1.3493, + "theoretical_loss": 3.6269653104877713, + "tokens_seen": 1160708096 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032744232698094286, + "loss": 1.2743, + "theoretical_loss": 3.626946504267691, + "tokens_seen": 1160773632 + }, + { + "epoch": 0.35, + "learning_rate": 0.000327432296890672, + "loss": 1.3066, + "theoretical_loss": 3.6269276994066377, + "tokens_seen": 1160839168 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003274222668004012, + "loss": 1.371, + "theoretical_loss": 3.6269088959044367, + "tokens_seen": 1160904704 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032741223671013035, + "loss": 1.2435, + "theoretical_loss": 3.626890093760913, + "tokens_seen": 1160970240 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003274022066198596, + "loss": 1.3042, + "theoretical_loss": 3.626871292975893, + "tokens_seen": 1161035776 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032739217652958877, + "loss": 1.2481, + "theoretical_loss": 3.6268524935491997, + "tokens_seen": 1161101312 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032738214643931795, + "loss": 1.2081, + "theoretical_loss": 3.62683369548066, + "tokens_seen": 1161166848 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032737211634904713, + "loss": 1.2685, + "theoretical_loss": 3.6268148987700983, + "tokens_seen": 1161232384 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003273620862587763, + "loss": 1.4109, + "theoretical_loss": 3.62679610341734, + "tokens_seen": 1161297920 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003273520561685055, + "loss": 1.2081, + "theoretical_loss": 3.626777309422211, + "tokens_seen": 1161363456 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032734202607823473, + "loss": 1.3068, + "theoretical_loss": 3.6267585167845358, + "tokens_seen": 1161428992 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032733199598796386, + "loss": 1.2745, + "theoretical_loss": 3.62673972550414, + "tokens_seen": 1161494528 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003273219658976931, + "loss": 1.3212, + "theoretical_loss": 3.6267209355808494, + "tokens_seen": 1161560064 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032731193580742233, + "loss": 1.3449, + "theoretical_loss": 3.6267021470144893, + "tokens_seen": 1161625600 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032730190571715145, + "loss": 1.4296, + "theoretical_loss": 3.6266833598048844, + "tokens_seen": 1161691136 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003272918756268807, + "loss": 1.32, + "theoretical_loss": 3.626664573951861, + "tokens_seen": 1161756672 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003272818455366098, + "loss": 1.2791, + "theoretical_loss": 3.626645789455245, + "tokens_seen": 1161822208 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032727181544633905, + "loss": 1.3029, + "theoretical_loss": 3.6266270063148607, + "tokens_seen": 1161887744 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032726178535606823, + "loss": 1.2567, + "theoretical_loss": 3.6266082245305347, + "tokens_seen": 1161953280 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003272517552657974, + "loss": 1.2881, + "theoretical_loss": 3.626589444102092, + "tokens_seen": 1162018816 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003272417251755266, + "loss": 1.2016, + "theoretical_loss": 3.6265706650293588, + "tokens_seen": 1162084352 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003272316950852558, + "loss": 1.3027, + "theoretical_loss": 3.6265518873121607, + "tokens_seen": 1162149888 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032722166499498496, + "loss": 1.3406, + "theoretical_loss": 3.626533110950323, + "tokens_seen": 1162215424 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003272116349047142, + "loss": 1.3584, + "theoretical_loss": 3.6265143359436722, + "tokens_seen": 1162280960 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003272016048144433, + "loss": 1.3214, + "theoretical_loss": 3.6264955622920327, + "tokens_seen": 1162346496 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032719157472417256, + "loss": 1.3012, + "theoretical_loss": 3.626476789995232, + "tokens_seen": 1162412032 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003271815446339017, + "loss": 1.1616, + "theoretical_loss": 3.6264580190530946, + "tokens_seen": 1162477568 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003271715145436309, + "loss": 1.3465, + "theoretical_loss": 3.626439249465448, + "tokens_seen": 1162543104 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003271614844533601, + "loss": 1.3401, + "theoretical_loss": 3.626420481232116, + "tokens_seen": 1162608640 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003271514543630893, + "loss": 1.2309, + "theoretical_loss": 3.6264017143529257, + "tokens_seen": 1162674176 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032714142427281846, + "loss": 1.3174, + "theoretical_loss": 3.626382948827703, + "tokens_seen": 1162739712 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003271313941825477, + "loss": 1.38, + "theoretical_loss": 3.626364184656274, + "tokens_seen": 1162805248 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003271213640922768, + "loss": 1.3377, + "theoretical_loss": 3.6263454218384648, + "tokens_seen": 1162870784 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032711133400200606, + "loss": 1.2984, + "theoretical_loss": 3.626326660374101, + "tokens_seen": 1162936320 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003271013039117352, + "loss": 1.3566, + "theoretical_loss": 3.6263079002630096, + "tokens_seen": 1163001856 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003270912738214644, + "loss": 1.2994, + "theoretical_loss": 3.6262891415050156, + "tokens_seen": 1163067392 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003270812437311936, + "loss": 1.3074, + "theoretical_loss": 3.6262703840999455, + "tokens_seen": 1163132928 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003270712136409228, + "loss": 1.2978, + "theoretical_loss": 3.6262516280476262, + "tokens_seen": 1163198464 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 389752, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3062615394592285, + "objective/train/theoretical_loss": 3.6262328733478837, + "objective/train/tokens_used": 1163264000, + "theoretical_loss": 3.6262328733478837, + "tokens_seen": 1163264000 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032706118355065196, + "loss": 1.2964, + "theoretical_loss": 3.6262328733478837, + "tokens_seen": 1163264000 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032705115346038114, + "loss": 1.3128, + "theoretical_loss": 3.6262141200005433, + "tokens_seen": 1163329536 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003270411233701103, + "loss": 1.2805, + "theoretical_loss": 3.6261953680054324, + "tokens_seen": 1163395072 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032703109327983956, + "loss": 1.2853, + "theoretical_loss": 3.6261766173623773, + "tokens_seen": 1163460608 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003270210631895687, + "loss": 1.2369, + "theoretical_loss": 3.6261578680712034, + "tokens_seen": 1163526144 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003270110330992979, + "loss": 1.4275, + "theoretical_loss": 3.6261391201317377, + "tokens_seen": 1163591680 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032700100300902705, + "loss": 1.2616, + "theoretical_loss": 3.6261203735438072, + "tokens_seen": 1163657216 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003269909729187563, + "loss": 1.2895, + "theoretical_loss": 3.6261016283072376, + "tokens_seen": 1163722752 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032698094282848547, + "loss": 1.3255, + "theoretical_loss": 3.626082884421855, + "tokens_seen": 1163788288 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032697091273821465, + "loss": 1.4737, + "theoretical_loss": 3.6260641418874875, + "tokens_seen": 1163853824 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032696088264794383, + "loss": 1.3365, + "theoretical_loss": 3.62604540070396, + "tokens_seen": 1163919360 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032695085255767306, + "loss": 1.163, + "theoretical_loss": 3.6260266608711005, + "tokens_seen": 1163984896 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003269408224674022, + "loss": 1.3232, + "theoretical_loss": 3.626007922388734, + "tokens_seen": 1164050432 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003269307923771314, + "loss": 1.3378, + "theoretical_loss": 3.6259891852566883, + "tokens_seen": 1164115968 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032692076228686055, + "loss": 1.3246, + "theoretical_loss": 3.62597044947479, + "tokens_seen": 1164181504 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003269107321965898, + "loss": 1.3735, + "theoretical_loss": 3.6259517150428655, + "tokens_seen": 1164247040 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032690070210631897, + "loss": 1.2837, + "theoretical_loss": 3.6259329819607418, + "tokens_seen": 1164312576 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032689067201604815, + "loss": 1.3443, + "theoretical_loss": 3.625914250228245, + "tokens_seen": 1164378112 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032688064192577733, + "loss": 1.3896, + "theoretical_loss": 3.625895519845203, + "tokens_seen": 1164443648 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003268706118355065, + "loss": 1.3353, + "theoretical_loss": 3.625876790811442, + "tokens_seen": 1164509184 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003268605817452357, + "loss": 1.3295, + "theoretical_loss": 3.625858063126789, + "tokens_seen": 1164574720 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032685055165496493, + "loss": 1.3264, + "theoretical_loss": 3.625839336791071, + "tokens_seen": 1164640256 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032684052156469406, + "loss": 1.2246, + "theoretical_loss": 3.625820611804115, + "tokens_seen": 1164705792 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003268304914744233, + "loss": 1.3518, + "theoretical_loss": 3.6258018881657472, + "tokens_seen": 1164771328 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003268204613841524, + "loss": 1.304, + "theoretical_loss": 3.6257831658757955, + "tokens_seen": 1164836864 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032681043129388165, + "loss": 1.1497, + "theoretical_loss": 3.6257644449340867, + "tokens_seen": 1164902400 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032680040120361084, + "loss": 1.3207, + "theoretical_loss": 3.6257457253404475, + "tokens_seen": 1164967936 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032679037111334, + "loss": 1.2876, + "theoretical_loss": 3.6257270070947056, + "tokens_seen": 1165033472 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003267803410230692, + "loss": 1.3117, + "theoretical_loss": 3.6257082901966884, + "tokens_seen": 1165099008 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032677031093279843, + "loss": 1.3155, + "theoretical_loss": 3.6256895746462217, + "tokens_seen": 1165164544 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032676028084252756, + "loss": 1.2568, + "theoretical_loss": 3.625670860443134, + "tokens_seen": 1165230080 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003267502507522568, + "loss": 1.3763, + "theoretical_loss": 3.6256521475872514, + "tokens_seen": 1165295616 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003267402206619859, + "loss": 1.2687, + "theoretical_loss": 3.6256334360784024, + "tokens_seen": 1165361152 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032673019057171516, + "loss": 1.4326, + "theoretical_loss": 3.625614725916413, + "tokens_seen": 1165426688 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032672016048144434, + "loss": 1.2143, + "theoretical_loss": 3.6255960171011115, + "tokens_seen": 1165492224 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003267101303911735, + "loss": 1.3294, + "theoretical_loss": 3.6255773096323254, + "tokens_seen": 1165557760 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003267001003009027, + "loss": 1.3122, + "theoretical_loss": 3.625558603509881, + "tokens_seen": 1165623296 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003266900702106319, + "loss": 1.3188, + "theoretical_loss": 3.6255398987336065, + "tokens_seen": 1165688832 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032668004012036106, + "loss": 1.1803, + "theoretical_loss": 3.6255211953033295, + "tokens_seen": 1165754368 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003266700100300903, + "loss": 1.3467, + "theoretical_loss": 3.6255024932188773, + "tokens_seen": 1165819904 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003266599799398194, + "loss": 1.4138, + "theoretical_loss": 3.625483792480077, + "tokens_seen": 1165885440 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032664994984954866, + "loss": 1.347, + "theoretical_loss": 3.6254650930867562, + "tokens_seen": 1165950976 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032663991975927784, + "loss": 1.2407, + "theoretical_loss": 3.625446395038743, + "tokens_seen": 1166016512 + }, + { + "epoch": 0.35, + "learning_rate": 0.000326629889669007, + "loss": 1.3439, + "theoretical_loss": 3.625427698335865, + "tokens_seen": 1166082048 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003266198595787362, + "loss": 1.3165, + "theoretical_loss": 3.625409002977949, + "tokens_seen": 1166147584 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003266098294884654, + "loss": 1.3556, + "theoretical_loss": 3.6253903089648234, + "tokens_seen": 1166213120 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032659979939819457, + "loss": 1.3254, + "theoretical_loss": 3.6253716162963157, + "tokens_seen": 1166278656 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003265897693079238, + "loss": 1.3494, + "theoretical_loss": 3.625352924972254, + "tokens_seen": 1166344192 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032657973921765293, + "loss": 1.3034, + "theoretical_loss": 3.625334234992465, + "tokens_seen": 1166409728 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032656970912738216, + "loss": 1.3312, + "theoretical_loss": 3.625315546356778, + "tokens_seen": 1166475264 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 390671, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2493693828582764, + "objective/train/theoretical_loss": 3.62529685906502, + "objective/train/tokens_used": 1166540800, + "theoretical_loss": 3.62529685906502, + "tokens_seen": 1166540800 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032655967903711135, + "loss": 1.3308, + "theoretical_loss": 3.62529685906502, + "tokens_seen": 1166540800 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003265496489468405, + "loss": 1.3749, + "theoretical_loss": 3.625278173117019, + "tokens_seen": 1166606336 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032653961885656976, + "loss": 1.3172, + "theoretical_loss": 3.6252594885126026, + "tokens_seen": 1166671872 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003265295887662989, + "loss": 1.3954, + "theoretical_loss": 3.625240805251599, + "tokens_seen": 1166737408 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003265195586760281, + "loss": 1.409, + "theoretical_loss": 3.625222123333836, + "tokens_seen": 1166802944 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032650952858575725, + "loss": 1.3653, + "theoretical_loss": 3.6252034427591417, + "tokens_seen": 1166868480 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003264994984954865, + "loss": 1.292, + "theoretical_loss": 3.6251847635273444, + "tokens_seen": 1166934016 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032648946840521567, + "loss": 1.346, + "theoretical_loss": 3.6251660856382717, + "tokens_seen": 1166999552 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032647943831494485, + "loss": 1.3325, + "theoretical_loss": 3.6251474090917517, + "tokens_seen": 1167065088 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032646940822467403, + "loss": 1.4175, + "theoretical_loss": 3.625128733887613, + "tokens_seen": 1167130624 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032645937813440326, + "loss": 1.3435, + "theoretical_loss": 3.6251100600256834, + "tokens_seen": 1167196160 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003264493480441324, + "loss": 1.33, + "theoretical_loss": 3.6250913875057904, + "tokens_seen": 1167261696 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003264393179538616, + "loss": 1.29, + "theoretical_loss": 3.6250727163277636, + "tokens_seen": 1167327232 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032642928786359075, + "loss": 1.46, + "theoretical_loss": 3.6250540464914307, + "tokens_seen": 1167392768 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032641925777332, + "loss": 1.2592, + "theoretical_loss": 3.6250353779966193, + "tokens_seen": 1167458304 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032640922768304917, + "loss": 1.2812, + "theoretical_loss": 3.625016710843159, + "tokens_seen": 1167523840 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032639919759277835, + "loss": 1.3206, + "theoretical_loss": 3.6249980450308765, + "tokens_seen": 1167589376 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032638916750250753, + "loss": 1.3854, + "theoretical_loss": 3.6249793805596013, + "tokens_seen": 1167654912 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003263791374122367, + "loss": 1.3319, + "theoretical_loss": 3.624960717429162, + "tokens_seen": 1167720448 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003263691073219659, + "loss": 1.3953, + "theoretical_loss": 3.6249420556393863, + "tokens_seen": 1167785984 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032635907723169513, + "loss": 1.2785, + "theoretical_loss": 3.6249233951901028, + "tokens_seen": 1167851520 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032634904714142426, + "loss": 1.3564, + "theoretical_loss": 3.62490473608114, + "tokens_seen": 1167917056 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003263390170511535, + "loss": 1.2969, + "theoretical_loss": 3.624886078312327, + "tokens_seen": 1167982592 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003263289869608826, + "loss": 1.3319, + "theoretical_loss": 3.6248674218834918, + "tokens_seen": 1168048128 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032631895687061185, + "loss": 1.2684, + "theoretical_loss": 3.6248487667944627, + "tokens_seen": 1168113664 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032630892678034104, + "loss": 1.3516, + "theoretical_loss": 3.624830113045069, + "tokens_seen": 1168179200 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003262988966900702, + "loss": 1.2846, + "theoretical_loss": 3.624811460635139, + "tokens_seen": 1168244736 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003262888665997994, + "loss": 1.3068, + "theoretical_loss": 3.624792809564502, + "tokens_seen": 1168310272 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032627883650952863, + "loss": 1.4431, + "theoretical_loss": 3.6247741598329855, + "tokens_seen": 1168375808 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032626880641925776, + "loss": 1.2929, + "theoretical_loss": 3.6247555114404193, + "tokens_seen": 1168441344 + }, + { + "epoch": 0.35, + "learning_rate": 0.000326258776328987, + "loss": 1.4786, + "theoretical_loss": 3.624736864386632, + "tokens_seen": 1168506880 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003262487462387161, + "loss": 1.4406, + "theoretical_loss": 3.6247182186714517, + "tokens_seen": 1168572416 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032623871614844536, + "loss": 1.2726, + "theoretical_loss": 3.6246995742947083, + "tokens_seen": 1168637952 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032622868605817454, + "loss": 1.3536, + "theoretical_loss": 3.6246809312562296, + "tokens_seen": 1168703488 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003262186559679037, + "loss": 1.3986, + "theoretical_loss": 3.6246622895558454, + "tokens_seen": 1168769024 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003262086258776329, + "loss": 1.3098, + "theoretical_loss": 3.624643649193384, + "tokens_seen": 1168834560 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003261985957873621, + "loss": 1.255, + "theoretical_loss": 3.6246250101686748, + "tokens_seen": 1168900096 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032618856569709126, + "loss": 1.3554, + "theoretical_loss": 3.624606372481547, + "tokens_seen": 1168965632 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003261785356068205, + "loss": 1.3366, + "theoretical_loss": 3.624587736131829, + "tokens_seen": 1169031168 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003261685055165496, + "loss": 1.3067, + "theoretical_loss": 3.62456910111935, + "tokens_seen": 1169096704 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032615847542627886, + "loss": 1.3334, + "theoretical_loss": 3.6245504674439393, + "tokens_seen": 1169162240 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032614844533600804, + "loss": 1.3728, + "theoretical_loss": 3.624531835105426, + "tokens_seen": 1169227776 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003261384152457372, + "loss": 1.3314, + "theoretical_loss": 3.624513204103639, + "tokens_seen": 1169293312 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003261283851554664, + "loss": 1.329, + "theoretical_loss": 3.6244945744384083, + "tokens_seen": 1169358848 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003261183550651956, + "loss": 1.2898, + "theoretical_loss": 3.6244759461095617, + "tokens_seen": 1169424384 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032610832497492477, + "loss": 1.2918, + "theoretical_loss": 3.62445731911693, + "tokens_seen": 1169489920 + }, + { + "epoch": 0.35, + "learning_rate": 0.000326098294884654, + "loss": 1.2719, + "theoretical_loss": 3.6244386934603416, + "tokens_seen": 1169555456 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032608826479438313, + "loss": 1.1761, + "theoretical_loss": 3.6244200691396258, + "tokens_seen": 1169620992 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032607823470411236, + "loss": 1.2592, + "theoretical_loss": 3.624401446154612, + "tokens_seen": 1169686528 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003260682046138415, + "loss": 1.2526, + "theoretical_loss": 3.6243828245051297, + "tokens_seen": 1169752064 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 392038, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.412144660949707, + "objective/train/theoretical_loss": 3.624364204191009, + "objective/train/tokens_used": 1169817600, + "theoretical_loss": 3.624364204191009, + "tokens_seen": 1169817600 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003260581745235707, + "loss": 1.3025, + "theoretical_loss": 3.624364204191009, + "tokens_seen": 1169817600 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003260481444332999, + "loss": 1.3188, + "theoretical_loss": 3.624345585212078, + "tokens_seen": 1169883136 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003260381143430291, + "loss": 1.3749, + "theoretical_loss": 3.624326967568167, + "tokens_seen": 1169948672 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032602808425275827, + "loss": 1.2579, + "theoretical_loss": 3.624308351259105, + "tokens_seen": 1170014208 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032601805416248745, + "loss": 1.3408, + "theoretical_loss": 3.624289736284722, + "tokens_seen": 1170079744 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032600802407221663, + "loss": 1.3393, + "theoretical_loss": 3.6242711226448474, + "tokens_seen": 1170145280 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032599799398194587, + "loss": 1.3446, + "theoretical_loss": 3.624252510339311, + "tokens_seen": 1170210816 + }, + { + "epoch": 0.35, + "learning_rate": 0.000325987963891675, + "loss": 1.3051, + "theoretical_loss": 3.624233899367942, + "tokens_seen": 1170276352 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032597793380140423, + "loss": 1.314, + "theoretical_loss": 3.6242152897305706, + "tokens_seen": 1170341888 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003259679037111334, + "loss": 1.29, + "theoretical_loss": 3.624196681427026, + "tokens_seen": 1170407424 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003259578736208626, + "loss": 1.3033, + "theoretical_loss": 3.624178074457138, + "tokens_seen": 1170472960 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003259478435305918, + "loss": 1.3171, + "theoretical_loss": 3.6241594688207366, + "tokens_seen": 1170538496 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032593781344032095, + "loss": 1.3465, + "theoretical_loss": 3.6241408645176514, + "tokens_seen": 1170604032 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032592778335005014, + "loss": 1.3102, + "theoretical_loss": 3.624122261547712, + "tokens_seen": 1170669568 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032591775325977937, + "loss": 1.3804, + "theoretical_loss": 3.624103659910749, + "tokens_seen": 1170735104 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003259077231695085, + "loss": 1.2433, + "theoretical_loss": 3.6240850596065917, + "tokens_seen": 1170800640 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032589769307923773, + "loss": 1.3036, + "theoretical_loss": 3.6240664606350697, + "tokens_seen": 1170866176 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032588766298896686, + "loss": 1.2709, + "theoretical_loss": 3.6240478629960133, + "tokens_seen": 1170931712 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003258776328986961, + "loss": 1.3638, + "theoretical_loss": 3.624029266689253, + "tokens_seen": 1170997248 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003258676028084253, + "loss": 1.363, + "theoretical_loss": 3.6240106717146174, + "tokens_seen": 1171062784 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032585757271815446, + "loss": 1.3057, + "theoretical_loss": 3.6239920780719386, + "tokens_seen": 1171128320 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032584754262788364, + "loss": 1.3736, + "theoretical_loss": 3.6239734857610446, + "tokens_seen": 1171193856 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003258375125376128, + "loss": 1.3032, + "theoretical_loss": 3.623954894781767, + "tokens_seen": 1171259392 + }, + { + "epoch": 0.35, + "learning_rate": 0.000325827482447342, + "loss": 1.2961, + "theoretical_loss": 3.6239363051339346, + "tokens_seen": 1171324928 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032581745235707124, + "loss": 1.282, + "theoretical_loss": 3.623917716817379, + "tokens_seen": 1171390464 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003258074222668004, + "loss": 1.3511, + "theoretical_loss": 3.623899129831929, + "tokens_seen": 1171456000 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003257973921765296, + "loss": 1.4036, + "theoretical_loss": 3.623880544177416, + "tokens_seen": 1171521536 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032578736208625883, + "loss": 1.335, + "theoretical_loss": 3.6238619598536697, + "tokens_seen": 1171587072 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032577733199598796, + "loss": 1.3759, + "theoretical_loss": 3.6238433768605205, + "tokens_seen": 1171652608 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003257673019057172, + "loss": 1.324, + "theoretical_loss": 3.623824795197798, + "tokens_seen": 1171718144 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003257572718154463, + "loss": 1.2969, + "theoretical_loss": 3.6238062148653336, + "tokens_seen": 1171783680 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032574724172517556, + "loss": 1.3915, + "theoretical_loss": 3.623787635862957, + "tokens_seen": 1171849216 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032573721163490474, + "loss": 1.4361, + "theoretical_loss": 3.6237690581904993, + "tokens_seen": 1171914752 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003257271815446339, + "loss": 1.2427, + "theoretical_loss": 3.6237504818477904, + "tokens_seen": 1171980288 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003257171514543631, + "loss": 1.3073, + "theoretical_loss": 3.6237319068346605, + "tokens_seen": 1172045824 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003257071213640923, + "loss": 1.3664, + "theoretical_loss": 3.6237133331509406, + "tokens_seen": 1172111360 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032569709127382146, + "loss": 1.3195, + "theoretical_loss": 3.623694760796461, + "tokens_seen": 1172176896 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003256870611835507, + "loss": 1.285, + "theoretical_loss": 3.6236761897710528, + "tokens_seen": 1172242432 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003256770310932798, + "loss": 1.2343, + "theoretical_loss": 3.623657620074546, + "tokens_seen": 1172307968 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032566700100300906, + "loss": 1.2731, + "theoretical_loss": 3.623639051706771, + "tokens_seen": 1172373504 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032565697091273824, + "loss": 1.3905, + "theoretical_loss": 3.6236204846675593, + "tokens_seen": 1172439040 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003256469408224674, + "loss": 1.2455, + "theoretical_loss": 3.6236019189567408, + "tokens_seen": 1172504576 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003256369107321966, + "loss": 1.3156, + "theoretical_loss": 3.623583354574147, + "tokens_seen": 1172570112 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003256268806419258, + "loss": 1.3033, + "theoretical_loss": 3.623564791519607, + "tokens_seen": 1172635648 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032561685055165497, + "loss": 1.2937, + "theoretical_loss": 3.623546229792954, + "tokens_seen": 1172701184 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003256068204613842, + "loss": 1.4611, + "theoretical_loss": 3.6235276693940173, + "tokens_seen": 1172766720 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032559679037111333, + "loss": 1.3604, + "theoretical_loss": 3.623509110322628, + "tokens_seen": 1172832256 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032558676028084256, + "loss": 1.299, + "theoretical_loss": 3.623490552578617, + "tokens_seen": 1172897792 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003255767301905717, + "loss": 1.3046, + "theoretical_loss": 3.6234719961618147, + "tokens_seen": 1172963328 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003255667001003009, + "loss": 1.338, + "theoretical_loss": 3.623453441072053, + "tokens_seen": 1173028864 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 392946, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2796860933303833, + "objective/train/theoretical_loss": 3.623434887309162, + "objective/train/tokens_used": 1173094400, + "theoretical_loss": 3.623434887309162, + "tokens_seen": 1173094400 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003255566700100301, + "loss": 1.2927, + "theoretical_loss": 3.623434887309162, + "tokens_seen": 1173094400 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003255466399197593, + "loss": 1.337, + "theoretical_loss": 3.6234163348729735, + "tokens_seen": 1173159936 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032553660982948847, + "loss": 1.2268, + "theoretical_loss": 3.623397783763318, + "tokens_seen": 1173225472 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032552657973921765, + "loss": 1.4407, + "theoretical_loss": 3.623379233980027, + "tokens_seen": 1173291008 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032551654964894683, + "loss": 1.3059, + "theoretical_loss": 3.623360685522931, + "tokens_seen": 1173356544 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032550651955867607, + "loss": 1.2573, + "theoretical_loss": 3.6233421383918607, + "tokens_seen": 1173422080 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003254964894684052, + "loss": 1.311, + "theoretical_loss": 3.623323592586649, + "tokens_seen": 1173487616 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032548645937813443, + "loss": 1.2992, + "theoretical_loss": 3.6233050481071256, + "tokens_seen": 1173553152 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003254764292878636, + "loss": 1.3884, + "theoretical_loss": 3.623286504953122, + "tokens_seen": 1173618688 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003254663991975928, + "loss": 1.3129, + "theoretical_loss": 3.6232679631244697, + "tokens_seen": 1173684224 + }, + { + "epoch": 0.36, + "learning_rate": 0.000325456369107322, + "loss": 1.2601, + "theoretical_loss": 3.623249422621, + "tokens_seen": 1173749760 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032544633901705115, + "loss": 1.2696, + "theoretical_loss": 3.6232308834425435, + "tokens_seen": 1173815296 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032543630892678034, + "loss": 1.3468, + "theoretical_loss": 3.623212345588933, + "tokens_seen": 1173880832 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032542627883650957, + "loss": 1.2495, + "theoretical_loss": 3.623193809059998, + "tokens_seen": 1173946368 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003254162487462387, + "loss": 1.3274, + "theoretical_loss": 3.623175273855572, + "tokens_seen": 1174011904 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032540621865596793, + "loss": 1.3024, + "theoretical_loss": 3.6231567399754843, + "tokens_seen": 1174077440 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032539618856569706, + "loss": 1.3191, + "theoretical_loss": 3.6231382074195677, + "tokens_seen": 1174142976 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003253861584754263, + "loss": 1.2192, + "theoretical_loss": 3.6231196761876534, + "tokens_seen": 1174208512 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003253761283851555, + "loss": 1.2409, + "theoretical_loss": 3.6231011462795726, + "tokens_seen": 1174274048 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032536609829488466, + "loss": 1.2653, + "theoretical_loss": 3.6230826176951574, + "tokens_seen": 1174339584 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032535606820461384, + "loss": 1.2863, + "theoretical_loss": 3.623064090434239, + "tokens_seen": 1174405120 + }, + { + "epoch": 0.36, + "learning_rate": 0.000325346038114343, + "loss": 1.2489, + "theoretical_loss": 3.623045564496649, + "tokens_seen": 1174470656 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003253360080240722, + "loss": 1.3671, + "theoretical_loss": 3.6230270398822197, + "tokens_seen": 1174536192 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032532597793380144, + "loss": 1.4391, + "theoretical_loss": 3.6230085165907813, + "tokens_seen": 1174601728 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032531594784353056, + "loss": 1.3583, + "theoretical_loss": 3.6229899946221673, + "tokens_seen": 1174667264 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003253059177532598, + "loss": 1.2951, + "theoretical_loss": 3.6229714739762082, + "tokens_seen": 1174732800 + }, + { + "epoch": 0.36, + "learning_rate": 0.000325295887662989, + "loss": 1.3528, + "theoretical_loss": 3.622952954652736, + "tokens_seen": 1174798336 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032528585757271816, + "loss": 1.2389, + "theoretical_loss": 3.622934436651583, + "tokens_seen": 1174863872 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032527582748244734, + "loss": 1.2557, + "theoretical_loss": 3.6229159199725807, + "tokens_seen": 1174929408 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003252657973921765, + "loss": 1.4274, + "theoretical_loss": 3.6228974046155606, + "tokens_seen": 1174994944 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003252557673019057, + "loss": 1.3854, + "theoretical_loss": 3.6228788905803553, + "tokens_seen": 1175060480 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032524573721163494, + "loss": 1.317, + "theoretical_loss": 3.6228603778667967, + "tokens_seen": 1175126016 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032523570712136407, + "loss": 1.3332, + "theoretical_loss": 3.6228418664747157, + "tokens_seen": 1175191552 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003252256770310933, + "loss": 1.2571, + "theoretical_loss": 3.622823356403945, + "tokens_seen": 1175257088 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032521564694082243, + "loss": 1.289, + "theoretical_loss": 3.622804847654317, + "tokens_seen": 1175322624 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032520561685055166, + "loss": 1.2763, + "theoretical_loss": 3.6227863402256633, + "tokens_seen": 1175388160 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032519558676028084, + "loss": 1.2469, + "theoretical_loss": 3.6227678341178158, + "tokens_seen": 1175453696 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032518555667001, + "loss": 1.3507, + "theoretical_loss": 3.622749329330607, + "tokens_seen": 1175519232 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003251755265797392, + "loss": 1.2963, + "theoretical_loss": 3.6227308258638686, + "tokens_seen": 1175584768 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032516549648946844, + "loss": 1.2692, + "theoretical_loss": 3.6227123237174332, + "tokens_seen": 1175650304 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032515546639919757, + "loss": 1.4596, + "theoretical_loss": 3.622693822891133, + "tokens_seen": 1175715840 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003251454363089268, + "loss": 1.3307, + "theoretical_loss": 3.6226753233848, + "tokens_seen": 1175781376 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032513540621865593, + "loss": 1.3086, + "theoretical_loss": 3.6226568251982667, + "tokens_seen": 1175846912 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032512537612838517, + "loss": 1.2644, + "theoretical_loss": 3.622638328331365, + "tokens_seen": 1175912448 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032511534603811435, + "loss": 1.2109, + "theoretical_loss": 3.6226198327839274, + "tokens_seen": 1175977984 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032510531594784353, + "loss": 1.3316, + "theoretical_loss": 3.6226013385557865, + "tokens_seen": 1176043520 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003250952858575727, + "loss": 1.2058, + "theoretical_loss": 3.6225828456467744, + "tokens_seen": 1176109056 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003250852557673019, + "loss": 1.3531, + "theoretical_loss": 3.6225643540567236, + "tokens_seen": 1176174592 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032507522567703107, + "loss": 1.2796, + "theoretical_loss": 3.622545863785467, + "tokens_seen": 1176240128 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003250651955867603, + "loss": 1.3303, + "theoretical_loss": 3.6225273748328357, + "tokens_seen": 1176305664 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 394137, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2862677574157715, + "objective/train/theoretical_loss": 3.6225088871986637, + "objective/train/tokens_used": 1176371200, + "theoretical_loss": 3.6225088871986637, + "tokens_seen": 1176371200 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003250551654964895, + "loss": 1.3156, + "theoretical_loss": 3.6225088871986637, + "tokens_seen": 1176371200 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032504513540621867, + "loss": 1.293, + "theoretical_loss": 3.622490400882783, + "tokens_seen": 1176436736 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032503510531594785, + "loss": 1.3718, + "theoretical_loss": 3.6224719158850256, + "tokens_seen": 1176502272 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032502507522567703, + "loss": 1.2745, + "theoretical_loss": 3.622453432205225, + "tokens_seen": 1176567808 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032501504513540627, + "loss": 1.3196, + "theoretical_loss": 3.622434949843213, + "tokens_seen": 1176633344 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003250050150451354, + "loss": 1.4111, + "theoretical_loss": 3.622416468798823, + "tokens_seen": 1176698880 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032499498495486463, + "loss": 1.1909, + "theoretical_loss": 3.622397989071888, + "tokens_seen": 1176764416 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003249849548645938, + "loss": 1.386, + "theoretical_loss": 3.6223795106622396, + "tokens_seen": 1176829952 + }, + { + "epoch": 0.36, + "learning_rate": 0.000324974924774323, + "loss": 1.3507, + "theoretical_loss": 3.622361033569711, + "tokens_seen": 1176895488 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003249648946840522, + "loss": 1.3429, + "theoretical_loss": 3.622342557794135, + "tokens_seen": 1176961024 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032495486459378135, + "loss": 1.3766, + "theoretical_loss": 3.6223240833353447, + "tokens_seen": 1177026560 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032494483450351054, + "loss": 1.3281, + "theoretical_loss": 3.6223056101931728, + "tokens_seen": 1177092096 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032493480441323977, + "loss": 1.3061, + "theoretical_loss": 3.6222871383674518, + "tokens_seen": 1177157632 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003249247743229689, + "loss": 1.2278, + "theoretical_loss": 3.6222686678580147, + "tokens_seen": 1177223168 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032491474423269813, + "loss": 1.2528, + "theoretical_loss": 3.6222501986646947, + "tokens_seen": 1177288704 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032490471414242726, + "loss": 1.3531, + "theoretical_loss": 3.622231730787325, + "tokens_seen": 1177354240 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003248946840521565, + "loss": 1.2948, + "theoretical_loss": 3.622213264225738, + "tokens_seen": 1177419776 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003248846539618857, + "loss": 1.3711, + "theoretical_loss": 3.622194798979767, + "tokens_seen": 1177485312 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032487462387161486, + "loss": 1.3089, + "theoretical_loss": 3.6221763350492457, + "tokens_seen": 1177550848 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032486459378134404, + "loss": 1.3437, + "theoretical_loss": 3.6221578724340056, + "tokens_seen": 1177616384 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003248545636910732, + "loss": 1.3175, + "theoretical_loss": 3.622139411133881, + "tokens_seen": 1177681920 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003248445336008024, + "loss": 1.2921, + "theoretical_loss": 3.622120951148705, + "tokens_seen": 1177747456 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032483450351053164, + "loss": 1.2659, + "theoretical_loss": 3.6221024924783105, + "tokens_seen": 1177812992 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032482447342026076, + "loss": 1.2144, + "theoretical_loss": 3.6220840351225307, + "tokens_seen": 1177878528 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032481444332999, + "loss": 1.3492, + "theoretical_loss": 3.6220655790811986, + "tokens_seen": 1177944064 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003248044132397192, + "loss": 1.2857, + "theoretical_loss": 3.622047124354148, + "tokens_seen": 1178009600 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032479438314944836, + "loss": 1.3068, + "theoretical_loss": 3.622028670941212, + "tokens_seen": 1178075136 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032478435305917754, + "loss": 1.2651, + "theoretical_loss": 3.6220102188422243, + "tokens_seen": 1178140672 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003247743229689067, + "loss": 1.3222, + "theoretical_loss": 3.6219917680570175, + "tokens_seen": 1178206208 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003247642928786359, + "loss": 1.2585, + "theoretical_loss": 3.6219733185854253, + "tokens_seen": 1178271744 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032475426278836514, + "loss": 1.3226, + "theoretical_loss": 3.621954870427282, + "tokens_seen": 1178337280 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032474423269809427, + "loss": 1.3121, + "theoretical_loss": 3.6219364235824187, + "tokens_seen": 1178402816 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003247342026078235, + "loss": 1.3163, + "theoretical_loss": 3.6219179780506714, + "tokens_seen": 1178468352 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032472417251755263, + "loss": 1.432, + "theoretical_loss": 3.621899533831873, + "tokens_seen": 1178533888 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032471414242728186, + "loss": 1.284, + "theoretical_loss": 3.621881090925856, + "tokens_seen": 1178599424 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032470411233701104, + "loss": 1.2983, + "theoretical_loss": 3.6218626493324546, + "tokens_seen": 1178664960 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003246940822467402, + "loss": 1.3492, + "theoretical_loss": 3.6218442090515026, + "tokens_seen": 1178730496 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003246840521564694, + "loss": 1.3196, + "theoretical_loss": 3.6218257700828334, + "tokens_seen": 1178796032 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032467402206619864, + "loss": 1.3451, + "theoretical_loss": 3.6218073324262807, + "tokens_seen": 1178861568 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032466399197592777, + "loss": 1.2799, + "theoretical_loss": 3.6217888960816786, + "tokens_seen": 1178927104 + }, + { + "epoch": 0.36, + "learning_rate": 0.000324653961885657, + "loss": 1.3029, + "theoretical_loss": 3.62177046104886, + "tokens_seen": 1178992640 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032464393179538613, + "loss": 1.3183, + "theoretical_loss": 3.6217520273276596, + "tokens_seen": 1179058176 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032463390170511537, + "loss": 1.2951, + "theoretical_loss": 3.6217335949179104, + "tokens_seen": 1179123712 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032462387161484455, + "loss": 1.3743, + "theoretical_loss": 3.6217151638194465, + "tokens_seen": 1179189248 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032461384152457373, + "loss": 1.2299, + "theoretical_loss": 3.621696734032102, + "tokens_seen": 1179254784 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003246038114343029, + "loss": 1.2517, + "theoretical_loss": 3.62167830555571, + "tokens_seen": 1179320320 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003245937813440321, + "loss": 1.2471, + "theoretical_loss": 3.6216598783901053, + "tokens_seen": 1179385856 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032458375125376127, + "loss": 1.2456, + "theoretical_loss": 3.6216414525351217, + "tokens_seen": 1179451392 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003245737211634905, + "loss": 1.2503, + "theoretical_loss": 3.6216230279905925, + "tokens_seen": 1179516928 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032456369107321963, + "loss": 1.3221, + "theoretical_loss": 3.621604604756352, + "tokens_seen": 1179582464 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 395236, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3821132183074951, + "objective/train/theoretical_loss": 3.621586182832235, + "objective/train/tokens_used": 1179648000, + "theoretical_loss": 3.621586182832235, + "tokens_seen": 1179648000 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032455366098294887, + "loss": 1.3876, + "theoretical_loss": 3.621586182832235, + "tokens_seen": 1179648000 + }, + { + "epoch": 0.36, + "learning_rate": 0.000324543630892678, + "loss": 1.4091, + "theoretical_loss": 3.621567762218074, + "tokens_seen": 1179713536 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032453360080240723, + "loss": 1.2637, + "theoretical_loss": 3.621549342913705, + "tokens_seen": 1179779072 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003245235707121364, + "loss": 1.2555, + "theoretical_loss": 3.621530924918961, + "tokens_seen": 1179844608 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003245135406218656, + "loss": 1.3228, + "theoretical_loss": 3.6215125082336757, + "tokens_seen": 1179910144 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003245035105315948, + "loss": 1.2493, + "theoretical_loss": 3.621494092857684, + "tokens_seen": 1179975680 + }, + { + "epoch": 0.36, + "learning_rate": 0.000324493480441324, + "loss": 1.2913, + "theoretical_loss": 3.62147567879082, + "tokens_seen": 1180041216 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032448345035105314, + "loss": 1.2819, + "theoretical_loss": 3.621457266032918, + "tokens_seen": 1180106752 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003244734202607824, + "loss": 1.4273, + "theoretical_loss": 3.621438854583812, + "tokens_seen": 1180172288 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003244633901705115, + "loss": 1.3732, + "theoretical_loss": 3.621420444443337, + "tokens_seen": 1180237824 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032445336008024074, + "loss": 1.2685, + "theoretical_loss": 3.621402035611326, + "tokens_seen": 1180303360 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003244433299899699, + "loss": 1.3286, + "theoretical_loss": 3.6213836280876146, + "tokens_seen": 1180368896 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003244332998996991, + "loss": 1.3586, + "theoretical_loss": 3.6213652218720367, + "tokens_seen": 1180434432 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003244232698094283, + "loss": 1.2061, + "theoretical_loss": 3.6213468169644267, + "tokens_seen": 1180499968 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032441323971915746, + "loss": 1.2338, + "theoretical_loss": 3.621328413364619, + "tokens_seen": 1180565504 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032440320962888664, + "loss": 1.3248, + "theoretical_loss": 3.621310011072448, + "tokens_seen": 1180631040 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003243931795386159, + "loss": 1.3631, + "theoretical_loss": 3.621291610087749, + "tokens_seen": 1180696576 + }, + { + "epoch": 0.36, + "learning_rate": 0.000324383149448345, + "loss": 1.2077, + "theoretical_loss": 3.6212732104103558, + "tokens_seen": 1180762112 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032437311935807424, + "loss": 1.3517, + "theoretical_loss": 3.621254812040103, + "tokens_seen": 1180827648 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032436308926780337, + "loss": 1.347, + "theoretical_loss": 3.621236414976825, + "tokens_seen": 1180893184 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003243530591775326, + "loss": 1.2942, + "theoretical_loss": 3.621218019220357, + "tokens_seen": 1180958720 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003243430290872618, + "loss": 1.3168, + "theoretical_loss": 3.621199624770534, + "tokens_seen": 1181024256 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032433299899699096, + "loss": 1.2492, + "theoretical_loss": 3.621181231627189, + "tokens_seen": 1181089792 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003243229689067202, + "loss": 1.3658, + "theoretical_loss": 3.6211628397901583, + "tokens_seen": 1181155328 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003243129388164494, + "loss": 1.2998, + "theoretical_loss": 3.621144449259276, + "tokens_seen": 1181220864 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032430290872617856, + "loss": 1.2001, + "theoretical_loss": 3.6211260600343773, + "tokens_seen": 1181286400 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032429287863590774, + "loss": 1.4373, + "theoretical_loss": 3.6211076721152966, + "tokens_seen": 1181351936 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003242828485456369, + "loss": 1.251, + "theoretical_loss": 3.621089285501869, + "tokens_seen": 1181417472 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003242728184553661, + "loss": 1.304, + "theoretical_loss": 3.621070900193929, + "tokens_seen": 1181483008 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032426278836509534, + "loss": 1.2641, + "theoretical_loss": 3.621052516191312, + "tokens_seen": 1181548544 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032425275827482447, + "loss": 1.2374, + "theoretical_loss": 3.6210341334938527, + "tokens_seen": 1181614080 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003242427281845537, + "loss": 1.3047, + "theoretical_loss": 3.621015752101386, + "tokens_seen": 1181679616 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032423269809428283, + "loss": 1.3391, + "theoretical_loss": 3.620997372013747, + "tokens_seen": 1181745152 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032422266800401206, + "loss": 1.2644, + "theoretical_loss": 3.6209789932307705, + "tokens_seen": 1181810688 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032421263791374125, + "loss": 1.451, + "theoretical_loss": 3.620960615752292, + "tokens_seen": 1181876224 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003242026078234704, + "loss": 1.392, + "theoretical_loss": 3.620942239578146, + "tokens_seen": 1181941760 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003241925777331996, + "loss": 1.2902, + "theoretical_loss": 3.620923864708168, + "tokens_seen": 1182007296 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032418254764292884, + "loss": 1.2566, + "theoretical_loss": 3.620905491142193, + "tokens_seen": 1182072832 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032417251755265797, + "loss": 1.3128, + "theoretical_loss": 3.6208871188800567, + "tokens_seen": 1182138368 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003241624874623872, + "loss": 1.3121, + "theoretical_loss": 3.620868747921593, + "tokens_seen": 1182203904 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032415245737211633, + "loss": 1.3371, + "theoretical_loss": 3.6208503782666384, + "tokens_seen": 1182269440 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032414242728184557, + "loss": 1.3199, + "theoretical_loss": 3.620832009915028, + "tokens_seen": 1182334976 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032413239719157475, + "loss": 1.2632, + "theoretical_loss": 3.6208136428665965, + "tokens_seen": 1182400512 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032412236710130393, + "loss": 1.3225, + "theoretical_loss": 3.6207952771211795, + "tokens_seen": 1182466048 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003241123370110331, + "loss": 1.2581, + "theoretical_loss": 3.620776912678612, + "tokens_seen": 1182531584 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003241023069207623, + "loss": 1.2897, + "theoretical_loss": 3.6207585495387304, + "tokens_seen": 1182597120 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032409227683049147, + "loss": 1.3133, + "theoretical_loss": 3.620740187701369, + "tokens_seen": 1182662656 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003240822467402207, + "loss": 1.3547, + "theoretical_loss": 3.6207218271663644, + "tokens_seen": 1182728192 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032407221664994984, + "loss": 1.352, + "theoretical_loss": 3.6207034679335504, + "tokens_seen": 1182793728 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032406218655967907, + "loss": 1.4303, + "theoretical_loss": 3.620685110002764, + "tokens_seen": 1182859264 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 396146, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.345963716506958, + "objective/train/theoretical_loss": 3.6206667533738406, + "objective/train/tokens_used": 1182924800, + "theoretical_loss": 3.6206667533738406, + "tokens_seen": 1182924800 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003240521564694082, + "loss": 1.3237, + "theoretical_loss": 3.6206667533738406, + "tokens_seen": 1182924800 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032404212637913743, + "loss": 1.3327, + "theoretical_loss": 3.6206483980466144, + "tokens_seen": 1182990336 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003240320962888666, + "loss": 1.328, + "theoretical_loss": 3.620630044020923, + "tokens_seen": 1183055872 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003240220661985958, + "loss": 1.3584, + "theoretical_loss": 3.6206116912966007, + "tokens_seen": 1183121408 + }, + { + "epoch": 0.36, + "learning_rate": 0.000324012036108325, + "loss": 1.3156, + "theoretical_loss": 3.620593339873483, + "tokens_seen": 1183186944 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003240020060180542, + "loss": 1.3133, + "theoretical_loss": 3.620574989751406, + "tokens_seen": 1183252480 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032399197592778334, + "loss": 1.3566, + "theoretical_loss": 3.620556640930206, + "tokens_seen": 1183318016 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003239819458375126, + "loss": 1.2875, + "theoretical_loss": 3.6205382934097177, + "tokens_seen": 1183383552 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003239719157472417, + "loss": 1.3878, + "theoretical_loss": 3.6205199471897775, + "tokens_seen": 1183449088 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032396188565697094, + "loss": 1.3817, + "theoretical_loss": 3.6205016022702208, + "tokens_seen": 1183514624 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003239518555667001, + "loss": 1.3843, + "theoretical_loss": 3.620483258650884, + "tokens_seen": 1183580160 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003239418254764293, + "loss": 1.271, + "theoretical_loss": 3.6204649163316023, + "tokens_seen": 1183645696 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003239317953861585, + "loss": 1.3295, + "theoretical_loss": 3.6204465753122124, + "tokens_seen": 1183711232 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032392176529588766, + "loss": 1.356, + "theoretical_loss": 3.620428235592549, + "tokens_seen": 1183776768 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032391173520561684, + "loss": 1.2657, + "theoretical_loss": 3.6204098971724488, + "tokens_seen": 1183842304 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003239017051153461, + "loss": 1.3453, + "theoretical_loss": 3.6203915600517482, + "tokens_seen": 1183907840 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003238916750250752, + "loss": 1.3371, + "theoretical_loss": 3.6203732242302826, + "tokens_seen": 1183973376 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032388164493480444, + "loss": 1.277, + "theoretical_loss": 3.6203548897078885, + "tokens_seen": 1184038912 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032387161484453357, + "loss": 1.3009, + "theoretical_loss": 3.6203365564844012, + "tokens_seen": 1184104448 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003238615847542628, + "loss": 1.3592, + "theoretical_loss": 3.6203182245596572, + "tokens_seen": 1184169984 + }, + { + "epoch": 0.36, + "learning_rate": 0.000323851554663992, + "loss": 1.3297, + "theoretical_loss": 3.6202998939334927, + "tokens_seen": 1184235520 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032384152457372116, + "loss": 1.3821, + "theoretical_loss": 3.6202815646057447, + "tokens_seen": 1184301056 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032383149448345034, + "loss": 1.3069, + "theoretical_loss": 3.6202632365762475, + "tokens_seen": 1184366592 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003238214643931796, + "loss": 1.378, + "theoretical_loss": 3.6202449098448386, + "tokens_seen": 1184432128 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003238114343029087, + "loss": 1.3302, + "theoretical_loss": 3.6202265844113537, + "tokens_seen": 1184497664 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032380140421263794, + "loss": 1.3487, + "theoretical_loss": 3.6202082602756303, + "tokens_seen": 1184563200 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032379137412236707, + "loss": 1.4046, + "theoretical_loss": 3.6201899374375026, + "tokens_seen": 1184628736 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003237813440320963, + "loss": 1.4087, + "theoretical_loss": 3.620171615896809, + "tokens_seen": 1184694272 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003237713139418255, + "loss": 1.3302, + "theoretical_loss": 3.6201532956533846, + "tokens_seen": 1184759808 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032376128385155467, + "loss": 1.2999, + "theoretical_loss": 3.6201349767070665, + "tokens_seen": 1184825344 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032375125376128385, + "loss": 1.3214, + "theoretical_loss": 3.62011665905769, + "tokens_seen": 1184890880 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032374122367101303, + "loss": 1.3349, + "theoretical_loss": 3.6200983427050932, + "tokens_seen": 1184956416 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003237311935807422, + "loss": 1.2523, + "theoretical_loss": 3.620080027649111, + "tokens_seen": 1185021952 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032372116349047145, + "loss": 1.2218, + "theoretical_loss": 3.620061713889581, + "tokens_seen": 1185087488 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032371113340020057, + "loss": 1.3138, + "theoretical_loss": 3.620043401426339, + "tokens_seen": 1185153024 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003237011033099298, + "loss": 1.3822, + "theoretical_loss": 3.6200250902592224, + "tokens_seen": 1185218560 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032369107321965893, + "loss": 1.3915, + "theoretical_loss": 3.620006780388067, + "tokens_seen": 1185284096 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032368104312938817, + "loss": 1.3724, + "theoretical_loss": 3.61998847181271, + "tokens_seen": 1185349632 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032367101303911735, + "loss": 1.3249, + "theoretical_loss": 3.619970164532988, + "tokens_seen": 1185415168 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032366098294884653, + "loss": 1.4143, + "theoretical_loss": 3.6199518585487374, + "tokens_seen": 1185480704 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003236509528585757, + "loss": 1.3454, + "theoretical_loss": 3.619933553859795, + "tokens_seen": 1185546240 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032364092276830495, + "loss": 1.3548, + "theoretical_loss": 3.6199152504659975, + "tokens_seen": 1185611776 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003236308926780341, + "loss": 1.3322, + "theoretical_loss": 3.619896948367182, + "tokens_seen": 1185677312 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003236208625877633, + "loss": 1.378, + "theoretical_loss": 3.619878647563185, + "tokens_seen": 1185742848 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032361083249749244, + "loss": 1.3747, + "theoretical_loss": 3.6198603480538427, + "tokens_seen": 1185808384 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003236008024072217, + "loss": 1.1974, + "theoretical_loss": 3.6198420498389936, + "tokens_seen": 1185873920 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032359077231695085, + "loss": 1.2901, + "theoretical_loss": 3.6198237529184736, + "tokens_seen": 1185939456 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032358074222668004, + "loss": 1.2927, + "theoretical_loss": 3.619805457292119, + "tokens_seen": 1186004992 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032357071213640927, + "loss": 1.3606, + "theoretical_loss": 3.6197871629597684, + "tokens_seen": 1186070528 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003235606820461384, + "loss": 1.3933, + "theoretical_loss": 3.6197688699212573, + "tokens_seen": 1186136064 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 397382, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2057826519012451, + "objective/train/theoretical_loss": 3.619750578176424, + "objective/train/tokens_used": 1186201600, + "theoretical_loss": 3.619750578176424, + "tokens_seen": 1186201600 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032355065195586763, + "loss": 1.2262, + "theoretical_loss": 3.619750578176424, + "tokens_seen": 1186201600 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003235406218655968, + "loss": 1.2949, + "theoretical_loss": 3.619732287725104, + "tokens_seen": 1186267136 + }, + { + "epoch": 0.36, + "learning_rate": 0.000323530591775326, + "loss": 1.3508, + "theoretical_loss": 3.619713998567135, + "tokens_seen": 1186332672 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003235205616850552, + "loss": 1.3025, + "theoretical_loss": 3.619695710702355, + "tokens_seen": 1186398208 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003235105315947844, + "loss": 1.2946, + "theoretical_loss": 3.6196774241306002, + "tokens_seen": 1186463744 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032350050150451354, + "loss": 1.258, + "theoretical_loss": 3.619659138851708, + "tokens_seen": 1186529280 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003234904714142428, + "loss": 1.2939, + "theoretical_loss": 3.6196408548655157, + "tokens_seen": 1186594816 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003234804413239719, + "loss": 1.3511, + "theoretical_loss": 3.61962257217186, + "tokens_seen": 1186660352 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032347041123370114, + "loss": 1.245, + "theoretical_loss": 3.619604290770579, + "tokens_seen": 1186725888 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003234603811434303, + "loss": 1.3975, + "theoretical_loss": 3.619586010661509, + "tokens_seen": 1186791424 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003234503510531595, + "loss": 1.3668, + "theoretical_loss": 3.6195677318444885, + "tokens_seen": 1186856960 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003234403209628887, + "loss": 1.3501, + "theoretical_loss": 3.619549454319354, + "tokens_seen": 1186922496 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032343029087261786, + "loss": 1.3287, + "theoretical_loss": 3.6195311780859427, + "tokens_seen": 1186988032 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032342026078234704, + "loss": 1.3728, + "theoretical_loss": 3.619512903144093, + "tokens_seen": 1187053568 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003234102306920763, + "loss": 1.4264, + "theoretical_loss": 3.619494629493641, + "tokens_seen": 1187119104 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003234002006018054, + "loss": 1.3373, + "theoretical_loss": 3.6194763571344257, + "tokens_seen": 1187184640 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032339017051153464, + "loss": 1.3054, + "theoretical_loss": 3.619458086066283, + "tokens_seen": 1187250176 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032338014042126377, + "loss": 1.4662, + "theoretical_loss": 3.6194398162890513, + "tokens_seen": 1187315712 + }, + { + "epoch": 0.36, + "learning_rate": 0.000323370110330993, + "loss": 1.3692, + "theoretical_loss": 3.6194215478025678, + "tokens_seen": 1187381248 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003233600802407222, + "loss": 1.3305, + "theoretical_loss": 3.6194032806066705, + "tokens_seen": 1187446784 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032335005015045136, + "loss": 1.406, + "theoretical_loss": 3.619385014701197, + "tokens_seen": 1187512320 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032334002006018054, + "loss": 1.411, + "theoretical_loss": 3.6193667500859847, + "tokens_seen": 1187577856 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003233299899699098, + "loss": 1.3074, + "theoretical_loss": 3.619348486760871, + "tokens_seen": 1187643392 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003233199598796389, + "loss": 1.2445, + "theoretical_loss": 3.619330224725694, + "tokens_seen": 1187708928 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032330992978936814, + "loss": 1.347, + "theoretical_loss": 3.6193119639802918, + "tokens_seen": 1187774464 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032329989969909727, + "loss": 1.286, + "theoretical_loss": 3.619293704524501, + "tokens_seen": 1187840000 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003232898696088265, + "loss": 1.3902, + "theoretical_loss": 3.6192754463581602, + "tokens_seen": 1187905536 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003232798395185557, + "loss": 1.3355, + "theoretical_loss": 3.619257189481107, + "tokens_seen": 1187971072 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032326980942828487, + "loss": 1.3032, + "theoretical_loss": 3.619238933893179, + "tokens_seen": 1188036608 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032325977933801405, + "loss": 1.3355, + "theoretical_loss": 3.619220679594215, + "tokens_seen": 1188102144 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032324974924774323, + "loss": 1.3181, + "theoretical_loss": 3.619202426584052, + "tokens_seen": 1188167680 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003232397191574724, + "loss": 1.3659, + "theoretical_loss": 3.6191841748625277, + "tokens_seen": 1188233216 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032322968906720165, + "loss": 1.236, + "theoretical_loss": 3.619165924429481, + "tokens_seen": 1188298752 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032321965897693077, + "loss": 1.236, + "theoretical_loss": 3.61914767528475, + "tokens_seen": 1188364288 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032320962888666, + "loss": 1.3983, + "theoretical_loss": 3.619129427428171, + "tokens_seen": 1188429824 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032319959879638913, + "loss": 1.2271, + "theoretical_loss": 3.6191111808595835, + "tokens_seen": 1188495360 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032318956870611837, + "loss": 1.3743, + "theoretical_loss": 3.6190929355788253, + "tokens_seen": 1188560896 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032317953861584755, + "loss": 1.3277, + "theoretical_loss": 3.619074691585735, + "tokens_seen": 1188626432 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032316950852557673, + "loss": 1.3633, + "theoretical_loss": 3.6190564488801495, + "tokens_seen": 1188691968 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003231594784353059, + "loss": 1.4431, + "theoretical_loss": 3.619038207461908, + "tokens_seen": 1188757504 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032314944834503515, + "loss": 1.2523, + "theoretical_loss": 3.6190199673308485, + "tokens_seen": 1188823040 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003231394182547643, + "loss": 1.2975, + "theoretical_loss": 3.6190017284868086, + "tokens_seen": 1188888576 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003231293881644935, + "loss": 1.3564, + "theoretical_loss": 3.6189834909296277, + "tokens_seen": 1188954112 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032311935807422264, + "loss": 1.2253, + "theoretical_loss": 3.618965254659143, + "tokens_seen": 1189019648 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003231093279839519, + "loss": 1.2432, + "theoretical_loss": 3.618947019675193, + "tokens_seen": 1189085184 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032309929789368105, + "loss": 1.2832, + "theoretical_loss": 3.6189287859776167, + "tokens_seen": 1189150720 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032308926780341024, + "loss": 1.3906, + "theoretical_loss": 3.6189105535662516, + "tokens_seen": 1189216256 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003230792377131394, + "loss": 1.3498, + "theoretical_loss": 3.618892322440937, + "tokens_seen": 1189281792 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003230692076228686, + "loss": 1.3829, + "theoretical_loss": 3.61887409260151, + "tokens_seen": 1189347328 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003230591775325978, + "loss": 1.263, + "theoretical_loss": 3.6188558640478106, + "tokens_seen": 1189412864 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 398479, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.322056531906128, + "objective/train/theoretical_loss": 3.618837636779676, + "objective/train/tokens_used": 1189478400, + "theoretical_loss": 3.618837636779676, + "tokens_seen": 1189478400 + }, + { + "epoch": 0.36, + "learning_rate": 0.000323049147442327, + "loss": 1.3633, + "theoretical_loss": 3.618837636779676, + "tokens_seen": 1189478400 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032303911735205614, + "loss": 1.4146, + "theoretical_loss": 3.618819410796946, + "tokens_seen": 1189543936 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003230290872617854, + "loss": 1.2322, + "theoretical_loss": 3.618801186099458, + "tokens_seen": 1189609472 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003230190571715145, + "loss": 1.282, + "theoretical_loss": 3.618782962687051, + "tokens_seen": 1189675008 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032300902708124374, + "loss": 1.2968, + "theoretical_loss": 3.6187647405595635, + "tokens_seen": 1189740544 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003229989969909729, + "loss": 1.3849, + "theoretical_loss": 3.6187465197168347, + "tokens_seen": 1189806080 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003229889669007021, + "loss": 1.3252, + "theoretical_loss": 3.6187283001587023, + "tokens_seen": 1189871616 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003229789368104313, + "loss": 1.3326, + "theoretical_loss": 3.6187100818850055, + "tokens_seen": 1189937152 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003229689067201605, + "loss": 1.3311, + "theoretical_loss": 3.6186918648955837, + "tokens_seen": 1190002688 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032295887662988964, + "loss": 1.3772, + "theoretical_loss": 3.6186736491902742, + "tokens_seen": 1190068224 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003229488465396189, + "loss": 1.2841, + "theoretical_loss": 3.6186554347689164, + "tokens_seen": 1190133760 + }, + { + "epoch": 0.36, + "learning_rate": 0.000322938816449348, + "loss": 1.3383, + "theoretical_loss": 3.6186372216313503, + "tokens_seen": 1190199296 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032292878635907724, + "loss": 1.3284, + "theoretical_loss": 3.6186190097774125, + "tokens_seen": 1190264832 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003229187562688064, + "loss": 1.2454, + "theoretical_loss": 3.618600799206943, + "tokens_seen": 1190330368 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003229087261785356, + "loss": 1.1964, + "theoretical_loss": 3.6185825899197814, + "tokens_seen": 1190395904 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003228986960882648, + "loss": 1.282, + "theoretical_loss": 3.618564381915766, + "tokens_seen": 1190461440 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032288866599799397, + "loss": 1.281, + "theoretical_loss": 3.6185461751947354, + "tokens_seen": 1190526976 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032287863590772315, + "loss": 1.3068, + "theoretical_loss": 3.6185279697565287, + "tokens_seen": 1190592512 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003228686058174524, + "loss": 1.295, + "theoretical_loss": 3.6185097656009853, + "tokens_seen": 1190658048 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003228585757271815, + "loss": 1.2617, + "theoretical_loss": 3.618491562727944, + "tokens_seen": 1190723584 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032284854563691074, + "loss": 1.3104, + "theoretical_loss": 3.6184733611372435, + "tokens_seen": 1190789120 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003228385155466399, + "loss": 1.3659, + "theoretical_loss": 3.6184551608287236, + "tokens_seen": 1190854656 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003228284854563691, + "loss": 1.3316, + "theoretical_loss": 3.6184369618022236, + "tokens_seen": 1190920192 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032281845536609834, + "loss": 1.2699, + "theoretical_loss": 3.618418764057581, + "tokens_seen": 1190985728 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032280842527582747, + "loss": 1.3299, + "theoretical_loss": 3.618400567594637, + "tokens_seen": 1191051264 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003227983951855567, + "loss": 1.3192, + "theoretical_loss": 3.6183823724132296, + "tokens_seen": 1191116800 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003227883650952859, + "loss": 1.3874, + "theoretical_loss": 3.6183641785131986, + "tokens_seen": 1191182336 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032277833500501507, + "loss": 1.3557, + "theoretical_loss": 3.6183459858943827, + "tokens_seen": 1191247872 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032276830491474425, + "loss": 1.3238, + "theoretical_loss": 3.618327794556622, + "tokens_seen": 1191313408 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032275827482447343, + "loss": 1.2632, + "theoretical_loss": 3.618309604499755, + "tokens_seen": 1191378944 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003227482447342026, + "loss": 1.3613, + "theoretical_loss": 3.618291415723622, + "tokens_seen": 1191444480 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032273821464393185, + "loss": 1.2852, + "theoretical_loss": 3.6182732282280607, + "tokens_seen": 1191510016 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032272818455366097, + "loss": 1.3743, + "theoretical_loss": 3.6182550420129127, + "tokens_seen": 1191575552 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003227181544633902, + "loss": 1.2657, + "theoretical_loss": 3.618236857078016, + "tokens_seen": 1191641088 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032270812437311933, + "loss": 1.2911, + "theoretical_loss": 3.6182186734232102, + "tokens_seen": 1191706624 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032269809428284857, + "loss": 1.2492, + "theoretical_loss": 3.6182004910483356, + "tokens_seen": 1191772160 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032268806419257775, + "loss": 1.4231, + "theoretical_loss": 3.618182309953231, + "tokens_seen": 1191837696 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032267803410230693, + "loss": 1.2878, + "theoretical_loss": 3.618164130137736, + "tokens_seen": 1191903232 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003226680040120361, + "loss": 1.2623, + "theoretical_loss": 3.61814595160169, + "tokens_seen": 1191968768 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032265797392176535, + "loss": 1.2934, + "theoretical_loss": 3.618127774344934, + "tokens_seen": 1192034304 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003226479438314945, + "loss": 1.3535, + "theoretical_loss": 3.6181095983673055, + "tokens_seen": 1192099840 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003226379137412237, + "loss": 1.4075, + "theoretical_loss": 3.6180914236686457, + "tokens_seen": 1192165376 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032262788365095284, + "loss": 1.416, + "theoretical_loss": 3.618073250248794, + "tokens_seen": 1192230912 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003226178535606821, + "loss": 1.2569, + "theoretical_loss": 3.6180550781075898, + "tokens_seen": 1192296448 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032260782347041125, + "loss": 1.2706, + "theoretical_loss": 3.618036907244873, + "tokens_seen": 1192361984 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032259779338014044, + "loss": 1.2862, + "theoretical_loss": 3.618018737660484, + "tokens_seen": 1192427520 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003225877632898696, + "loss": 1.3522, + "theoretical_loss": 3.6180005693542614, + "tokens_seen": 1192493056 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003225777331995988, + "loss": 1.354, + "theoretical_loss": 3.617982402326046, + "tokens_seen": 1192558592 + }, + { + "epoch": 0.36, + "learning_rate": 0.000322567703109328, + "loss": 1.2177, + "theoretical_loss": 3.617964236575678, + "tokens_seen": 1192624128 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003225576730190572, + "loss": 1.386, + "theoretical_loss": 3.6179460721029955, + "tokens_seen": 1192689664 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 399461, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4459493160247803, + "objective/train/theoretical_loss": 3.6179279089078404, + "objective/train/tokens_used": 1192755200, + "theoretical_loss": 3.6179279089078404, + "tokens_seen": 1192755200 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032254764292878634, + "loss": 1.4048, + "theoretical_loss": 3.6179279089078404, + "tokens_seen": 1192755200 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003225376128385156, + "loss": 1.4555, + "theoretical_loss": 3.617909746990052, + "tokens_seen": 1192820736 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003225275827482447, + "loss": 1.2894, + "theoretical_loss": 3.6178915863494696, + "tokens_seen": 1192886272 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032251755265797394, + "loss": 1.3266, + "theoretical_loss": 3.6178734269859345, + "tokens_seen": 1192951808 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003225075225677031, + "loss": 1.3242, + "theoretical_loss": 3.6178552688992855, + "tokens_seen": 1193017344 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003224974924774323, + "loss": 1.2848, + "theoretical_loss": 3.6178371120893638, + "tokens_seen": 1193082880 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003224874623871615, + "loss": 1.223, + "theoretical_loss": 3.6178189565560084, + "tokens_seen": 1193148416 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003224774322968907, + "loss": 1.2589, + "theoretical_loss": 3.617800802299061, + "tokens_seen": 1193213952 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032246740220661984, + "loss": 1.3282, + "theoretical_loss": 3.61778264931836, + "tokens_seen": 1193279488 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003224573721163491, + "loss": 1.2908, + "theoretical_loss": 3.617764497613746, + "tokens_seen": 1193345024 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003224473420260782, + "loss": 1.4126, + "theoretical_loss": 3.6177463471850597, + "tokens_seen": 1193410560 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032243731193580744, + "loss": 1.4185, + "theoretical_loss": 3.617728198032142, + "tokens_seen": 1193476096 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003224272818455366, + "loss": 1.3741, + "theoretical_loss": 3.617710050154832, + "tokens_seen": 1193541632 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003224172517552658, + "loss": 1.3208, + "theoretical_loss": 3.61769190355297, + "tokens_seen": 1193607168 + }, + { + "epoch": 0.36, + "learning_rate": 0.000322407221664995, + "loss": 1.3202, + "theoretical_loss": 3.617673758226397, + "tokens_seen": 1193672704 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032239719157472417, + "loss": 1.3133, + "theoretical_loss": 3.6176556141749536, + "tokens_seen": 1193738240 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032238716148445335, + "loss": 1.3213, + "theoretical_loss": 3.6176374713984796, + "tokens_seen": 1193803776 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003223771313941826, + "loss": 1.323, + "theoretical_loss": 3.617619329896815, + "tokens_seen": 1193869312 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003223671013039117, + "loss": 1.2951, + "theoretical_loss": 3.6176011896698013, + "tokens_seen": 1193934848 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032235707121364095, + "loss": 1.3281, + "theoretical_loss": 3.617583050717278, + "tokens_seen": 1194000384 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003223470411233701, + "loss": 1.1874, + "theoretical_loss": 3.617564913039087, + "tokens_seen": 1194065920 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003223370110330993, + "loss": 1.3461, + "theoretical_loss": 3.617546776635067, + "tokens_seen": 1194131456 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003223269809428285, + "loss": 1.2389, + "theoretical_loss": 3.61752864150506, + "tokens_seen": 1194196992 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032231695085255767, + "loss": 1.2953, + "theoretical_loss": 3.617510507648906, + "tokens_seen": 1194262528 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032230692076228685, + "loss": 1.3104, + "theoretical_loss": 3.617492375066446, + "tokens_seen": 1194328064 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003222968906720161, + "loss": 1.3746, + "theoretical_loss": 3.6174742437575196, + "tokens_seen": 1194393600 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003222868605817452, + "loss": 1.3315, + "theoretical_loss": 3.6174561137219685, + "tokens_seen": 1194459136 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032227683049147445, + "loss": 1.2846, + "theoretical_loss": 3.6174379849596336, + "tokens_seen": 1194524672 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003222668004012036, + "loss": 1.238, + "theoretical_loss": 3.617419857470355, + "tokens_seen": 1194590208 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003222567703109328, + "loss": 1.2874, + "theoretical_loss": 3.6174017312539735, + "tokens_seen": 1194655744 + }, + { + "epoch": 0.36, + "learning_rate": 0.000322246740220662, + "loss": 1.3842, + "theoretical_loss": 3.61738360631033, + "tokens_seen": 1194721280 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032223671013039117, + "loss": 1.3643, + "theoretical_loss": 3.617365482639266, + "tokens_seen": 1194786816 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032222668004012035, + "loss": 1.3336, + "theoretical_loss": 3.617347360240621, + "tokens_seen": 1194852352 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032221664994984953, + "loss": 1.2991, + "theoretical_loss": 3.6173292391142375, + "tokens_seen": 1194917888 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003222066198595787, + "loss": 1.3214, + "theoretical_loss": 3.6173111192599547, + "tokens_seen": 1194983424 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032219658976930795, + "loss": 1.3959, + "theoretical_loss": 3.6172930006776154, + "tokens_seen": 1195048960 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003221865596790371, + "loss": 1.3108, + "theoretical_loss": 3.6172748833670587, + "tokens_seen": 1195114496 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003221765295887663, + "loss": 1.2344, + "theoretical_loss": 3.6172567673281266, + "tokens_seen": 1195180032 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003221664994984955, + "loss": 1.3543, + "theoretical_loss": 3.61723865256066, + "tokens_seen": 1195245568 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003221564694082247, + "loss": 1.3655, + "theoretical_loss": 3.6172205390645003, + "tokens_seen": 1195311104 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032214643931795386, + "loss": 1.4682, + "theoretical_loss": 3.617202426839488, + "tokens_seen": 1195376640 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032213640922768304, + "loss": 1.3426, + "theoretical_loss": 3.6171843158854644, + "tokens_seen": 1195442176 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003221263791374122, + "loss": 1.2612, + "theoretical_loss": 3.6171662062022705, + "tokens_seen": 1195507712 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032211634904714145, + "loss": 1.3815, + "theoretical_loss": 3.617148097789748, + "tokens_seen": 1195573248 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003221063189568706, + "loss": 1.2884, + "theoretical_loss": 3.6171299906477374, + "tokens_seen": 1195638784 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003220962888665998, + "loss": 1.3494, + "theoretical_loss": 3.617111884776081, + "tokens_seen": 1195704320 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032208625877632894, + "loss": 1.412, + "theoretical_loss": 3.6170937801746184, + "tokens_seen": 1195769856 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003220762286860582, + "loss": 1.3883, + "theoretical_loss": 3.617075676843192, + "tokens_seen": 1195835392 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003220661985957874, + "loss": 1.3121, + "theoretical_loss": 3.6170575747816436, + "tokens_seen": 1195900928 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032205616850551654, + "loss": 1.2774, + "theoretical_loss": 3.6170394739898133, + "tokens_seen": 1195966464 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 400710, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3565858602523804, + "objective/train/theoretical_loss": 3.617021374467543, + "objective/train/tokens_used": 1196032000, + "theoretical_loss": 3.617021374467543, + "tokens_seen": 1196032000 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003220461384152458, + "loss": 1.3661, + "theoretical_loss": 3.617021374467543, + "tokens_seen": 1196032000 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003220361083249749, + "loss": 1.303, + "theoretical_loss": 3.617003276214674, + "tokens_seen": 1196097536 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032202607823470414, + "loss": 1.2492, + "theoretical_loss": 3.6169851792310483, + "tokens_seen": 1196163072 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003220160481444333, + "loss": 1.3343, + "theoretical_loss": 3.616967083516507, + "tokens_seen": 1196228608 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003220060180541625, + "loss": 1.3188, + "theoretical_loss": 3.616948989070891, + "tokens_seen": 1196294144 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003219959879638917, + "loss": 1.252, + "theoretical_loss": 3.6169308958940425, + "tokens_seen": 1196359680 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003219859578736209, + "loss": 1.2468, + "theoretical_loss": 3.616912803985803, + "tokens_seen": 1196425216 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032197592778335004, + "loss": 1.2707, + "theoretical_loss": 3.6168947133460136, + "tokens_seen": 1196490752 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003219658976930793, + "loss": 1.307, + "theoretical_loss": 3.6168766239745165, + "tokens_seen": 1196556288 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003219558676028084, + "loss": 1.2444, + "theoretical_loss": 3.616858535871153, + "tokens_seen": 1196621824 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032194583751253764, + "loss": 1.3376, + "theoretical_loss": 3.616840449035765, + "tokens_seen": 1196687360 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003219358074222668, + "loss": 1.3795, + "theoretical_loss": 3.6168223634681933, + "tokens_seen": 1196752896 + }, + { + "epoch": 0.36, + "learning_rate": 0.000321925777331996, + "loss": 1.3106, + "theoretical_loss": 3.6168042791682806, + "tokens_seen": 1196818432 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003219157472417252, + "loss": 1.2786, + "theoretical_loss": 3.6167861961358687, + "tokens_seen": 1196883968 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032190571715145437, + "loss": 1.3461, + "theoretical_loss": 3.6167681143707986, + "tokens_seen": 1196949504 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032189568706118355, + "loss": 1.3395, + "theoretical_loss": 3.616750033872912, + "tokens_seen": 1197015040 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003218856569709128, + "loss": 1.3929, + "theoretical_loss": 3.616731954642052, + "tokens_seen": 1197080576 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003218756268806419, + "loss": 1.3105, + "theoretical_loss": 3.6167138766780598, + "tokens_seen": 1197146112 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032186559679037115, + "loss": 1.3792, + "theoretical_loss": 3.6166957999807767, + "tokens_seen": 1197211648 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003218555667001003, + "loss": 1.2329, + "theoretical_loss": 3.616677724550045, + "tokens_seen": 1197277184 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003218455366098295, + "loss": 1.4247, + "theoretical_loss": 3.616659650385707, + "tokens_seen": 1197342720 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003218355065195587, + "loss": 1.3246, + "theoretical_loss": 3.616641577487604, + "tokens_seen": 1197408256 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032182547642928787, + "loss": 1.2794, + "theoretical_loss": 3.6166235058555785, + "tokens_seen": 1197473792 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032181544633901705, + "loss": 1.3267, + "theoretical_loss": 3.6166054354894728, + "tokens_seen": 1197539328 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003218054162487463, + "loss": 1.2733, + "theoretical_loss": 3.616587366389128, + "tokens_seen": 1197604864 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003217953861584754, + "loss": 1.2543, + "theoretical_loss": 3.6165692985543867, + "tokens_seen": 1197670400 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032178535606820465, + "loss": 1.2355, + "theoretical_loss": 3.616551231985091, + "tokens_seen": 1197735936 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003217753259779338, + "loss": 1.3556, + "theoretical_loss": 3.616533166681083, + "tokens_seen": 1197801472 + }, + { + "epoch": 0.36, + "learning_rate": 0.000321765295887663, + "loss": 1.279, + "theoretical_loss": 3.6165151026422055, + "tokens_seen": 1197867008 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003217552657973922, + "loss": 1.3856, + "theoretical_loss": 3.6164970398682996, + "tokens_seen": 1197932544 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032174523570712137, + "loss": 1.2179, + "theoretical_loss": 3.616478978359208, + "tokens_seen": 1197998080 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032173520561685055, + "loss": 1.3087, + "theoretical_loss": 3.6164609181147735, + "tokens_seen": 1198063616 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032172517552657974, + "loss": 1.2787, + "theoretical_loss": 3.616442859134837, + "tokens_seen": 1198129152 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003217151454363089, + "loss": 1.3957, + "theoretical_loss": 3.6164248014192424, + "tokens_seen": 1198194688 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032170511534603815, + "loss": 1.3034, + "theoretical_loss": 3.616406744967831, + "tokens_seen": 1198260224 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003216950852557673, + "loss": 1.2867, + "theoretical_loss": 3.616388689780446, + "tokens_seen": 1198325760 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003216850551654965, + "loss": 1.3402, + "theoretical_loss": 3.6163706358569288, + "tokens_seen": 1198391296 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003216750250752257, + "loss": 1.299, + "theoretical_loss": 3.6163525831971217, + "tokens_seen": 1198456832 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003216649949849549, + "loss": 1.3048, + "theoretical_loss": 3.6163345318008684, + "tokens_seen": 1198522368 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032165496489468406, + "loss": 1.3281, + "theoretical_loss": 3.6163164816680107, + "tokens_seen": 1198587904 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032164493480441324, + "loss": 1.3214, + "theoretical_loss": 3.616298432798391, + "tokens_seen": 1198653440 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003216349047141424, + "loss": 1.352, + "theoretical_loss": 3.6162803851918515, + "tokens_seen": 1198718976 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032162487462387165, + "loss": 1.3459, + "theoretical_loss": 3.6162623388482356, + "tokens_seen": 1198784512 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003216148445336008, + "loss": 1.4273, + "theoretical_loss": 3.6162442937673855, + "tokens_seen": 1198850048 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032160481444333, + "loss": 1.29, + "theoretical_loss": 3.6162262499491438, + "tokens_seen": 1198915584 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032159478435305914, + "loss": 1.3597, + "theoretical_loss": 3.616208207393353, + "tokens_seen": 1198981120 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003215847542627884, + "loss": 1.3248, + "theoretical_loss": 3.616190166099856, + "tokens_seen": 1199046656 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032157472417251756, + "loss": 1.3203, + "theoretical_loss": 3.616172126068496, + "tokens_seen": 1199112192 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032156469408224674, + "loss": 1.31, + "theoretical_loss": 3.6161540872991145, + "tokens_seen": 1199177728 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003215546639919759, + "loss": 1.2697, + "theoretical_loss": 3.616136049791555, + "tokens_seen": 1199243264 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 401604, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.463104248046875, + "objective/train/theoretical_loss": 3.61611801354566, + "objective/train/tokens_used": 1199308800, + "theoretical_loss": 3.61611801354566, + "tokens_seen": 1199308800 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003215446339017051, + "loss": 1.3601, + "theoretical_loss": 3.61611801354566, + "tokens_seen": 1199308800 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003215346038114343, + "loss": 1.3698, + "theoretical_loss": 3.6160999785612726, + "tokens_seen": 1199374336 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003215245737211635, + "loss": 1.2833, + "theoretical_loss": 3.6160819448382355, + "tokens_seen": 1199439872 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032151454363089265, + "loss": 1.384, + "theoretical_loss": 3.616063912376392, + "tokens_seen": 1199505408 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003215045135406219, + "loss": 1.3138, + "theoretical_loss": 3.6160458811755842, + "tokens_seen": 1199570944 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032149448345035106, + "loss": 1.3805, + "theoretical_loss": 3.6160278512356556, + "tokens_seen": 1199636480 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032148445336008024, + "loss": 1.3247, + "theoretical_loss": 3.6160098225564488, + "tokens_seen": 1199702016 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003214744232698094, + "loss": 1.3332, + "theoretical_loss": 3.6159917951378073, + "tokens_seen": 1199767552 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003214643931795386, + "loss": 1.2755, + "theoretical_loss": 3.6159737689795737, + "tokens_seen": 1199833088 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003214543630892678, + "loss": 1.3225, + "theoretical_loss": 3.6159557440815915, + "tokens_seen": 1199898624 + }, + { + "epoch": 0.36, + "learning_rate": 0.000321444332998997, + "loss": 1.3075, + "theoretical_loss": 3.6159377204437027, + "tokens_seen": 1199964160 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032143430290872615, + "loss": 1.2406, + "theoretical_loss": 3.6159196980657518, + "tokens_seen": 1200029696 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003214242728184554, + "loss": 1.218, + "theoretical_loss": 3.6159016769475807, + "tokens_seen": 1200095232 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003214142427281845, + "loss": 1.3719, + "theoretical_loss": 3.615883657089033, + "tokens_seen": 1200160768 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032140421263791375, + "loss": 1.3018, + "theoretical_loss": 3.615865638489953, + "tokens_seen": 1200226304 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032139418254764293, + "loss": 1.3398, + "theoretical_loss": 3.6158476211501815, + "tokens_seen": 1200291840 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003213841524573721, + "loss": 1.3464, + "theoretical_loss": 3.615829605069564, + "tokens_seen": 1200357376 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003213741223671013, + "loss": 1.2914, + "theoretical_loss": 3.6158115902479424, + "tokens_seen": 1200422912 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003213640922768305, + "loss": 1.3085, + "theoretical_loss": 3.6157935766851605, + "tokens_seen": 1200488448 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032135406218655965, + "loss": 1.2761, + "theoretical_loss": 3.615775564381062, + "tokens_seen": 1200553984 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003213440320962889, + "loss": 1.2913, + "theoretical_loss": 3.6157575533354893, + "tokens_seen": 1200619520 + }, + { + "epoch": 0.36, + "learning_rate": 0.000321334002006018, + "loss": 1.3504, + "theoretical_loss": 3.6157395435482864, + "tokens_seen": 1200685056 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032132397191574725, + "loss": 1.289, + "theoretical_loss": 3.615721535019297, + "tokens_seen": 1200750592 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003213139418254765, + "loss": 1.3159, + "theoretical_loss": 3.6157035277483636, + "tokens_seen": 1200816128 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003213039117352056, + "loss": 1.2697, + "theoretical_loss": 3.6156855217353305, + "tokens_seen": 1200881664 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032129388164493485, + "loss": 1.3953, + "theoretical_loss": 3.615667516980041, + "tokens_seen": 1200947200 + }, + { + "epoch": 0.36, + "learning_rate": 0.000321283851554664, + "loss": 1.287, + "theoretical_loss": 3.6156495134823383, + "tokens_seen": 1201012736 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003212738214643932, + "loss": 1.3252, + "theoretical_loss": 3.615631511242066, + "tokens_seen": 1201078272 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003212637913741224, + "loss": 1.2609, + "theoretical_loss": 3.6156135102590685, + "tokens_seen": 1201143808 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003212537612838516, + "loss": 1.3226, + "theoretical_loss": 3.615595510533188, + "tokens_seen": 1201209344 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032124373119358075, + "loss": 1.3896, + "theoretical_loss": 3.6155775120642692, + "tokens_seen": 1201274880 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032123370110330994, + "loss": 1.1977, + "theoretical_loss": 3.6155595148521553, + "tokens_seen": 1201340416 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003212236710130391, + "loss": 1.3508, + "theoretical_loss": 3.6155415188966904, + "tokens_seen": 1201405952 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032121364092276835, + "loss": 1.3298, + "theoretical_loss": 3.6155235241977177, + "tokens_seen": 1201471488 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003212036108324975, + "loss": 1.2381, + "theoretical_loss": 3.615505530755081, + "tokens_seen": 1201537024 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003211935807422267, + "loss": 1.2908, + "theoretical_loss": 3.6154875385686243, + "tokens_seen": 1201602560 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003211835506519559, + "loss": 1.3341, + "theoretical_loss": 3.6154695476381913, + "tokens_seen": 1201668096 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003211735205616851, + "loss": 1.3579, + "theoretical_loss": 3.6154515579636257, + "tokens_seen": 1201733632 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032116349047141426, + "loss": 1.3276, + "theoretical_loss": 3.6154335695447717, + "tokens_seen": 1201799168 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032115346038114344, + "loss": 1.3865, + "theoretical_loss": 3.615415582381473, + "tokens_seen": 1201864704 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003211434302908726, + "loss": 1.4072, + "theoretical_loss": 3.6153975964735734, + "tokens_seen": 1201930240 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032113340020060185, + "loss": 1.338, + "theoretical_loss": 3.615379611820917, + "tokens_seen": 1201995776 + }, + { + "epoch": 0.36, + "learning_rate": 0.000321123370110331, + "loss": 1.2609, + "theoretical_loss": 3.6153616284233476, + "tokens_seen": 1202061312 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003211133400200602, + "loss": 1.2731, + "theoretical_loss": 3.615343646280709, + "tokens_seen": 1202126848 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032110330992978934, + "loss": 1.1921, + "theoretical_loss": 3.615325665392846, + "tokens_seen": 1202192384 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003210932798395186, + "loss": 1.2099, + "theoretical_loss": 3.6153076857596016, + "tokens_seen": 1202257920 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032108324974924776, + "loss": 1.2026, + "theoretical_loss": 3.6152897073808212, + "tokens_seen": 1202323456 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032107321965897694, + "loss": 1.3701, + "theoretical_loss": 3.6152717302563477, + "tokens_seen": 1202388992 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003210631895687061, + "loss": 1.3877, + "theoretical_loss": 3.6152537543860257, + "tokens_seen": 1202454528 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003210531594784353, + "loss": 1.3174, + "theoretical_loss": 3.615235779769699, + "tokens_seen": 1202520064 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 402711, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.201128363609314, + "objective/train/theoretical_loss": 3.6152178064072125, + "objective/train/tokens_used": 1202585600, + "theoretical_loss": 3.6152178064072125, + "tokens_seen": 1202585600 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003210431293881645, + "loss": 1.2844, + "theoretical_loss": 3.6152178064072125, + "tokens_seen": 1202585600 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003210330992978937, + "loss": 1.2612, + "theoretical_loss": 3.6151998342984095, + "tokens_seen": 1202651136 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032102306920762285, + "loss": 1.338, + "theoretical_loss": 3.6151818634431354, + "tokens_seen": 1202716672 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003210130391173521, + "loss": 1.2928, + "theoretical_loss": 3.615163893841233, + "tokens_seen": 1202782208 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032100300902708126, + "loss": 1.3732, + "theoretical_loss": 3.6151459254925484, + "tokens_seen": 1202847744 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032099297893681044, + "loss": 1.3349, + "theoretical_loss": 3.6151279583969247, + "tokens_seen": 1202913280 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003209829488465396, + "loss": 1.4122, + "theoretical_loss": 3.6151099925542063, + "tokens_seen": 1202978816 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003209729187562688, + "loss": 1.3063, + "theoretical_loss": 3.6150920279642373, + "tokens_seen": 1203044352 + }, + { + "epoch": 0.36, + "learning_rate": 0.000320962888665998, + "loss": 1.2673, + "theoretical_loss": 3.615074064626863, + "tokens_seen": 1203109888 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003209528585757272, + "loss": 1.3226, + "theoretical_loss": 3.615056102541928, + "tokens_seen": 1203175424 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032094282848545635, + "loss": 1.2936, + "theoretical_loss": 3.6150381417092756, + "tokens_seen": 1203240960 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003209327983951856, + "loss": 1.385, + "theoretical_loss": 3.6150201821287506, + "tokens_seen": 1203306496 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003209227683049147, + "loss": 1.2925, + "theoretical_loss": 3.6150022238001984, + "tokens_seen": 1203372032 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032091273821464395, + "loss": 1.2834, + "theoretical_loss": 3.6149842667234626, + "tokens_seen": 1203437568 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032090270812437313, + "loss": 1.278, + "theoretical_loss": 3.6149663108983887, + "tokens_seen": 1203503104 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003208926780341023, + "loss": 1.3906, + "theoretical_loss": 3.6149483563248204, + "tokens_seen": 1203568640 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003208826479438315, + "loss": 1.1062, + "theoretical_loss": 3.614930403002602, + "tokens_seen": 1203634176 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003208726178535607, + "loss": 1.2517, + "theoretical_loss": 3.61491245093158, + "tokens_seen": 1203699712 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032086258776328985, + "loss": 1.3909, + "theoretical_loss": 3.614894500111597, + "tokens_seen": 1203765248 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003208525576730191, + "loss": 1.4387, + "theoretical_loss": 3.6148765505424993, + "tokens_seen": 1203830784 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003208425275827482, + "loss": 1.3151, + "theoretical_loss": 3.614858602224131, + "tokens_seen": 1203896320 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032083249749247745, + "loss": 1.3177, + "theoretical_loss": 3.6148406551563363, + "tokens_seen": 1203961856 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032082246740220663, + "loss": 1.2407, + "theoretical_loss": 3.614822709338961, + "tokens_seen": 1204027392 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003208124373119358, + "loss": 1.2714, + "theoretical_loss": 3.6148047647718493, + "tokens_seen": 1204092928 + }, + { + "epoch": 0.36, + "learning_rate": 0.000320802407221665, + "loss": 1.2997, + "theoretical_loss": 3.614786821454846, + "tokens_seen": 1204158464 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003207923771313942, + "loss": 1.3163, + "theoretical_loss": 3.6147688793877966, + "tokens_seen": 1204224000 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032078234704112336, + "loss": 1.3293, + "theoretical_loss": 3.614750938570545, + "tokens_seen": 1204289536 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003207723169508526, + "loss": 1.2965, + "theoretical_loss": 3.6147329990029373, + "tokens_seen": 1204355072 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003207622868605817, + "loss": 1.3081, + "theoretical_loss": 3.6147150606848184, + "tokens_seen": 1204420608 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032075225677031095, + "loss": 1.2371, + "theoretical_loss": 3.6146971236160317, + "tokens_seen": 1204486144 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003207422266800401, + "loss": 1.2683, + "theoretical_loss": 3.6146791877964244, + "tokens_seen": 1204551680 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003207321965897693, + "loss": 1.4351, + "theoretical_loss": 3.6146612532258398, + "tokens_seen": 1204617216 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003207221664994985, + "loss": 1.3436, + "theoretical_loss": 3.6146433199041237, + "tokens_seen": 1204682752 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003207121364092277, + "loss": 1.3374, + "theoretical_loss": 3.614625387831121, + "tokens_seen": 1204748288 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032070210631895686, + "loss": 1.2629, + "theoretical_loss": 3.6146074570066773, + "tokens_seen": 1204813824 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003206920762286861, + "loss": 1.3003, + "theoretical_loss": 3.6145895274306374, + "tokens_seen": 1204879360 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003206820461384152, + "loss": 1.3271, + "theoretical_loss": 3.6145715991028466, + "tokens_seen": 1204944896 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032067201604814446, + "loss": 1.4361, + "theoretical_loss": 3.61455367202315, + "tokens_seen": 1205010432 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003206619859578736, + "loss": 1.3394, + "theoretical_loss": 3.614535746191393, + "tokens_seen": 1205075968 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003206519558676028, + "loss": 1.2699, + "theoretical_loss": 3.6145178216074205, + "tokens_seen": 1205141504 + }, + { + "epoch": 0.37, + "learning_rate": 0.000320641925777332, + "loss": 1.2693, + "theoretical_loss": 3.6144998982710783, + "tokens_seen": 1205207040 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003206318956870612, + "loss": 1.4078, + "theoretical_loss": 3.614481976182211, + "tokens_seen": 1205272576 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032062186559679036, + "loss": 1.3199, + "theoretical_loss": 3.6144640553406653, + "tokens_seen": 1205338112 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032061183550651954, + "loss": 1.2748, + "theoretical_loss": 3.614446135746285, + "tokens_seen": 1205403648 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003206018054162487, + "loss": 1.2501, + "theoretical_loss": 3.6144282173989164, + "tokens_seen": 1205469184 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032059177532597796, + "loss": 1.2658, + "theoretical_loss": 3.6144103002984047, + "tokens_seen": 1205534720 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003205817452357071, + "loss": 1.3662, + "theoretical_loss": 3.6143923844445958, + "tokens_seen": 1205600256 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003205717151454363, + "loss": 1.3873, + "theoretical_loss": 3.614374469837334, + "tokens_seen": 1205665792 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003205616850551655, + "loss": 1.3509, + "theoretical_loss": 3.6143565564764666, + "tokens_seen": 1205731328 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003205516549648947, + "loss": 1.3196, + "theoretical_loss": 3.6143386443618373, + "tokens_seen": 1205796864 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 403576, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3396620750427246, + "objective/train/theoretical_loss": 3.614320733493293, + "objective/train/tokens_used": 1205862400, + "theoretical_loss": 3.614320733493293, + "tokens_seen": 1205862400 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003205416248746239, + "loss": 1.3458, + "theoretical_loss": 3.614320733493293, + "tokens_seen": 1205862400 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032053159478435305, + "loss": 1.2352, + "theoretical_loss": 3.6143028238706787, + "tokens_seen": 1205927936 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003205215646940823, + "loss": 1.3392, + "theoretical_loss": 3.61428491549384, + "tokens_seen": 1205993472 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032051153460381146, + "loss": 1.2977, + "theoretical_loss": 3.6142670083626225, + "tokens_seen": 1206059008 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032050150451354064, + "loss": 1.3488, + "theoretical_loss": 3.6142491024768724, + "tokens_seen": 1206124544 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003204914744232698, + "loss": 1.2993, + "theoretical_loss": 3.6142311978364354, + "tokens_seen": 1206190080 + }, + { + "epoch": 0.37, + "learning_rate": 0.000320481444332999, + "loss": 1.309, + "theoretical_loss": 3.6142132944411562, + "tokens_seen": 1206255616 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003204714142427282, + "loss": 1.3038, + "theoretical_loss": 3.614195392290882, + "tokens_seen": 1206321152 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003204613841524574, + "loss": 1.3593, + "theoretical_loss": 3.6141774913854574, + "tokens_seen": 1206386688 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032045135406218655, + "loss": 1.3468, + "theoretical_loss": 3.6141595917247296, + "tokens_seen": 1206452224 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003204413239719158, + "loss": 1.2975, + "theoretical_loss": 3.6141416933085426, + "tokens_seen": 1206517760 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003204312938816449, + "loss": 1.3513, + "theoretical_loss": 3.6141237961367434, + "tokens_seen": 1206583296 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032042126379137415, + "loss": 1.3489, + "theoretical_loss": 3.6141059002091778, + "tokens_seen": 1206648832 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032041123370110333, + "loss": 1.3413, + "theoretical_loss": 3.614088005525692, + "tokens_seen": 1206714368 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003204012036108325, + "loss": 1.2275, + "theoretical_loss": 3.6140701120861314, + "tokens_seen": 1206779904 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003203911735205617, + "loss": 1.2962, + "theoretical_loss": 3.6140522198903424, + "tokens_seen": 1206845440 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003203811434302909, + "loss": 1.2959, + "theoretical_loss": 3.6140343289381702, + "tokens_seen": 1206910976 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032037111334002005, + "loss": 1.232, + "theoretical_loss": 3.614016439229462, + "tokens_seen": 1206976512 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003203610832497493, + "loss": 1.35, + "theoretical_loss": 3.6139985507640633, + "tokens_seen": 1207042048 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003203510531594784, + "loss": 1.4531, + "theoretical_loss": 3.61398066354182, + "tokens_seen": 1207107584 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032034102306920765, + "loss": 1.3098, + "theoretical_loss": 3.6139627775625787, + "tokens_seen": 1207173120 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032033099297893683, + "loss": 1.2063, + "theoretical_loss": 3.6139448928261846, + "tokens_seen": 1207238656 + }, + { + "epoch": 0.37, + "learning_rate": 0.000320320962888666, + "loss": 1.4027, + "theoretical_loss": 3.6139270093324853, + "tokens_seen": 1207304192 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003203109327983952, + "loss": 1.2405, + "theoretical_loss": 3.6139091270813264, + "tokens_seen": 1207369728 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003203009027081244, + "loss": 1.3027, + "theoretical_loss": 3.613891246072553, + "tokens_seen": 1207435264 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032029087261785356, + "loss": 1.259, + "theoretical_loss": 3.613873366306013, + "tokens_seen": 1207500800 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003202808425275828, + "loss": 1.2783, + "theoretical_loss": 3.613855487781552, + "tokens_seen": 1207566336 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003202708124373119, + "loss": 1.1572, + "theoretical_loss": 3.6138376104990164, + "tokens_seen": 1207631872 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032026078234704115, + "loss": 1.3414, + "theoretical_loss": 3.6138197344582528, + "tokens_seen": 1207697408 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003202507522567703, + "loss": 1.3496, + "theoretical_loss": 3.613801859659107, + "tokens_seen": 1207762944 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003202407221664995, + "loss": 1.3075, + "theoretical_loss": 3.613783986101425, + "tokens_seen": 1207828480 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003202306920762287, + "loss": 1.248, + "theoretical_loss": 3.6137661137850543, + "tokens_seen": 1207894016 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003202206619859579, + "loss": 1.2779, + "theoretical_loss": 3.6137482427098417, + "tokens_seen": 1207959552 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032021063189568706, + "loss": 1.3414, + "theoretical_loss": 3.613730372875632, + "tokens_seen": 1208025088 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003202006018054163, + "loss": 1.3578, + "theoretical_loss": 3.613712504282273, + "tokens_seen": 1208090624 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003201905717151454, + "loss": 1.3721, + "theoretical_loss": 3.6136946369296106, + "tokens_seen": 1208156160 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032018054162487466, + "loss": 1.3732, + "theoretical_loss": 3.6136767708174915, + "tokens_seen": 1208221696 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003201705115346038, + "loss": 1.359, + "theoretical_loss": 3.613658905945763, + "tokens_seen": 1208287232 + }, + { + "epoch": 0.37, + "learning_rate": 0.000320160481444333, + "loss": 1.2533, + "theoretical_loss": 3.61364104231427, + "tokens_seen": 1208352768 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003201504513540622, + "loss": 1.2419, + "theoretical_loss": 3.613623179922861, + "tokens_seen": 1208418304 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003201404212637914, + "loss": 1.3935, + "theoretical_loss": 3.613605318771382, + "tokens_seen": 1208483840 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032013039117352056, + "loss": 1.2957, + "theoretical_loss": 3.613587458859679, + "tokens_seen": 1208549376 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032012036108324974, + "loss": 1.4235, + "theoretical_loss": 3.6135696001876, + "tokens_seen": 1208614912 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003201103309929789, + "loss": 1.3543, + "theoretical_loss": 3.613551742754991, + "tokens_seen": 1208680448 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032010030090270816, + "loss": 1.2411, + "theoretical_loss": 3.6135338865616986, + "tokens_seen": 1208745984 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003200902708124373, + "loss": 1.2525, + "theoretical_loss": 3.6135160316075696, + "tokens_seen": 1208811520 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003200802407221665, + "loss": 1.2336, + "theoretical_loss": 3.6134981778924518, + "tokens_seen": 1208877056 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032007021063189565, + "loss": 1.3306, + "theoretical_loss": 3.613480325416191, + "tokens_seen": 1208942592 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003200601805416249, + "loss": 1.2373, + "theoretical_loss": 3.613462474178634, + "tokens_seen": 1209008128 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032005015045135407, + "loss": 1.3421, + "theoretical_loss": 3.6134446241796283, + "tokens_seen": 1209073664 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 404776, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2108663320541382, + "objective/train/theoretical_loss": 3.613426775419021, + "objective/train/tokens_used": 1209139200, + "theoretical_loss": 3.613426775419021, + "tokens_seen": 1209139200 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032004012036108325, + "loss": 1.2379, + "theoretical_loss": 3.613426775419021, + "tokens_seen": 1209139200 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032003009027081243, + "loss": 1.2083, + "theoretical_loss": 3.6134089278966584, + "tokens_seen": 1209204736 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032002006018054166, + "loss": 1.3611, + "theoretical_loss": 3.613391081612388, + "tokens_seen": 1209270272 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003200100300902708, + "loss": 1.2302, + "theoretical_loss": 3.6133732365660567, + "tokens_seen": 1209335808 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032, + "loss": 1.3164, + "theoretical_loss": 3.6133553927575113, + "tokens_seen": 1209401344 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031998996990972915, + "loss": 1.4426, + "theoretical_loss": 3.6133375501865994, + "tokens_seen": 1209466880 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003199799398194584, + "loss": 1.4241, + "theoretical_loss": 3.613319708853168, + "tokens_seen": 1209532416 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031996990972918757, + "loss": 1.2573, + "theoretical_loss": 3.6133018687570635, + "tokens_seen": 1209597952 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031995987963891675, + "loss": 1.2178, + "theoretical_loss": 3.613284029898134, + "tokens_seen": 1209663488 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031994984954864593, + "loss": 1.4281, + "theoretical_loss": 3.613266192276226, + "tokens_seen": 1209729024 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003199398194583751, + "loss": 1.4058, + "theoretical_loss": 3.613248355891187, + "tokens_seen": 1209794560 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003199297893681043, + "loss": 1.3259, + "theoretical_loss": 3.613230520742864, + "tokens_seen": 1209860096 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031991975927783353, + "loss": 1.3015, + "theoretical_loss": 3.6132126868311047, + "tokens_seen": 1209925632 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031990972918756266, + "loss": 1.3781, + "theoretical_loss": 3.6131948541557564, + "tokens_seen": 1209991168 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003198996990972919, + "loss": 1.366, + "theoretical_loss": 3.613177022716666, + "tokens_seen": 1210056704 + }, + { + "epoch": 0.37, + "learning_rate": 0.000319889669007021, + "loss": 1.2877, + "theoretical_loss": 3.613159192513681, + "tokens_seen": 1210122240 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031987963891675025, + "loss": 1.3757, + "theoretical_loss": 3.6131413635466485, + "tokens_seen": 1210187776 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031986960882647943, + "loss": 1.2177, + "theoretical_loss": 3.613123535815417, + "tokens_seen": 1210253312 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003198595787362086, + "loss": 1.2762, + "theoretical_loss": 3.6131057093198327, + "tokens_seen": 1210318848 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003198495486459378, + "loss": 1.4509, + "theoretical_loss": 3.613087884059744, + "tokens_seen": 1210384384 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031983951855566703, + "loss": 1.3059, + "theoretical_loss": 3.613070060034997, + "tokens_seen": 1210449920 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031982948846539616, + "loss": 1.3371, + "theoretical_loss": 3.613052237245441, + "tokens_seen": 1210515456 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003198194583751254, + "loss": 1.3335, + "theoretical_loss": 3.6130344156909215, + "tokens_seen": 1210580992 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003198094282848546, + "loss": 1.2508, + "theoretical_loss": 3.6130165953712883, + "tokens_seen": 1210646528 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031979939819458376, + "loss": 1.2402, + "theoretical_loss": 3.6129987762863873, + "tokens_seen": 1210712064 + }, + { + "epoch": 0.37, + "learning_rate": 0.000319789368104313, + "loss": 1.2665, + "theoretical_loss": 3.612980958436067, + "tokens_seen": 1210777600 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003197793380140421, + "loss": 1.3616, + "theoretical_loss": 3.6129631418201744, + "tokens_seen": 1210843136 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031976930792377135, + "loss": 1.3117, + "theoretical_loss": 3.612945326438558, + "tokens_seen": 1210908672 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003197592778335005, + "loss": 1.2469, + "theoretical_loss": 3.6129275122910647, + "tokens_seen": 1210974208 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003197492477432297, + "loss": 1.3074, + "theoretical_loss": 3.6129096993775427, + "tokens_seen": 1211039744 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003197392176529589, + "loss": 1.3169, + "theoretical_loss": 3.6128918876978395, + "tokens_seen": 1211105280 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003197291875626881, + "loss": 1.3489, + "theoretical_loss": 3.612874077251803, + "tokens_seen": 1211170816 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031971915747241726, + "loss": 1.3009, + "theoretical_loss": 3.6128562680392813, + "tokens_seen": 1211236352 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003197091273821465, + "loss": 1.2922, + "theoretical_loss": 3.6128384600601215, + "tokens_seen": 1211301888 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003196990972918756, + "loss": 1.3311, + "theoretical_loss": 3.6128206533141722, + "tokens_seen": 1211367424 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031968906720160486, + "loss": 1.286, + "theoretical_loss": 3.612802847801281, + "tokens_seen": 1211432960 + }, + { + "epoch": 0.37, + "learning_rate": 0.000319679037111334, + "loss": 1.2407, + "theoretical_loss": 3.6127850435212956, + "tokens_seen": 1211498496 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003196690070210632, + "loss": 1.3326, + "theoretical_loss": 3.612767240474064, + "tokens_seen": 1211564032 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003196589769307924, + "loss": 1.2851, + "theoretical_loss": 3.612749438659435, + "tokens_seen": 1211629568 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003196489468405216, + "loss": 1.369, + "theoretical_loss": 3.612731638077255, + "tokens_seen": 1211695104 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031963891675025076, + "loss": 1.2453, + "theoretical_loss": 3.612713838727373, + "tokens_seen": 1211760640 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031962888665997994, + "loss": 1.348, + "theoretical_loss": 3.612696040609638, + "tokens_seen": 1211826176 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003196188565697091, + "loss": 1.2795, + "theoretical_loss": 3.612678243723896, + "tokens_seen": 1211891712 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031960882647943836, + "loss": 1.2379, + "theoretical_loss": 3.612660448069997, + "tokens_seen": 1211957248 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003195987963891675, + "loss": 1.398, + "theoretical_loss": 3.6126426536477876, + "tokens_seen": 1212022784 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003195887662988967, + "loss": 1.2895, + "theoretical_loss": 3.6126248604571174, + "tokens_seen": 1212088320 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031957873620862585, + "loss": 1.3096, + "theoretical_loss": 3.6126070684978333, + "tokens_seen": 1212153856 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003195687061183551, + "loss": 1.2988, + "theoretical_loss": 3.612589277769784, + "tokens_seen": 1212219392 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031955867602808427, + "loss": 1.3395, + "theoretical_loss": 3.612571488272818, + "tokens_seen": 1212284928 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031954864593781345, + "loss": 1.3866, + "theoretical_loss": 3.6125537000067833, + "tokens_seen": 1212350464 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 405723, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3674745559692383, + "objective/train/theoretical_loss": 3.6125359129715284, + "objective/train/tokens_used": 1212416000, + "theoretical_loss": 3.6125359129715284, + "tokens_seen": 1212416000 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031953861584754263, + "loss": 1.3763, + "theoretical_loss": 3.6125359129715284, + "tokens_seen": 1212416000 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031952858575727186, + "loss": 1.2852, + "theoretical_loss": 3.6125181271669016, + "tokens_seen": 1212481536 + }, + { + "epoch": 0.37, + "learning_rate": 0.000319518555667001, + "loss": 1.3417, + "theoretical_loss": 3.6125003425927504, + "tokens_seen": 1212547072 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003195085255767302, + "loss": 1.261, + "theoretical_loss": 3.6124825592489245, + "tokens_seen": 1212612608 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031949849548645935, + "loss": 1.3133, + "theoretical_loss": 3.612464777135272, + "tokens_seen": 1212678144 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003194884653961886, + "loss": 1.2526, + "theoretical_loss": 3.6124469962516406, + "tokens_seen": 1212743680 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031947843530591777, + "loss": 1.3398, + "theoretical_loss": 3.612429216597879, + "tokens_seen": 1212809216 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031946840521564695, + "loss": 1.4569, + "theoretical_loss": 3.6124114381738357, + "tokens_seen": 1212874752 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031945837512537613, + "loss": 1.276, + "theoretical_loss": 3.61239366097936, + "tokens_seen": 1212940288 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003194483450351053, + "loss": 1.3909, + "theoretical_loss": 3.6123758850142993, + "tokens_seen": 1213005824 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003194383149448345, + "loss": 1.2352, + "theoretical_loss": 3.612358110278503, + "tokens_seen": 1213071360 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031942828485456373, + "loss": 1.2941, + "theoretical_loss": 3.612340336771819, + "tokens_seen": 1213136896 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031941825476429286, + "loss": 1.3743, + "theoretical_loss": 3.6123225644940966, + "tokens_seen": 1213202432 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003194082246740221, + "loss": 1.3876, + "theoretical_loss": 3.612304793445184, + "tokens_seen": 1213267968 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003193981945837512, + "loss": 1.3651, + "theoretical_loss": 3.61228702362493, + "tokens_seen": 1213333504 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031938816449348045, + "loss": 1.428, + "theoretical_loss": 3.6122692550331834, + "tokens_seen": 1213399040 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031937813440320964, + "loss": 1.3066, + "theoretical_loss": 3.612251487669792, + "tokens_seen": 1213464576 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003193681043129388, + "loss": 1.2523, + "theoretical_loss": 3.6122337215346065, + "tokens_seen": 1213530112 + }, + { + "epoch": 0.37, + "learning_rate": 0.000319358074222668, + "loss": 1.3384, + "theoretical_loss": 3.612215956627474, + "tokens_seen": 1213595648 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031934804413239723, + "loss": 1.3284, + "theoretical_loss": 3.612198192948244, + "tokens_seen": 1213661184 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031933801404212636, + "loss": 1.3091, + "theoretical_loss": 3.612180430496765, + "tokens_seen": 1213726720 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003193279839518556, + "loss": 1.3422, + "theoretical_loss": 3.6121626692728857, + "tokens_seen": 1213792256 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003193179538615847, + "loss": 1.3159, + "theoretical_loss": 3.6121449092764557, + "tokens_seen": 1213857792 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031930792377131396, + "loss": 1.418, + "theoretical_loss": 3.6121271505073236, + "tokens_seen": 1213923328 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031929789368104314, + "loss": 1.3582, + "theoretical_loss": 3.612109392965338, + "tokens_seen": 1213988864 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003192878635907723, + "loss": 1.3944, + "theoretical_loss": 3.612091636650348, + "tokens_seen": 1214054400 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003192778335005015, + "loss": 1.2859, + "theoretical_loss": 3.6120738815622033, + "tokens_seen": 1214119936 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003192678034102307, + "loss": 1.3679, + "theoretical_loss": 3.6120561277007517, + "tokens_seen": 1214185472 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031925777331995986, + "loss": 1.2954, + "theoretical_loss": 3.612038375065843, + "tokens_seen": 1214251008 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003192477432296891, + "loss": 1.2976, + "theoretical_loss": 3.612020623657326, + "tokens_seen": 1214316544 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003192377131394182, + "loss": 1.4027, + "theoretical_loss": 3.61200287347505, + "tokens_seen": 1214382080 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031922768304914746, + "loss": 1.3414, + "theoretical_loss": 3.6119851245188643, + "tokens_seen": 1214447616 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003192176529588766, + "loss": 1.2361, + "theoretical_loss": 3.6119673767886176, + "tokens_seen": 1214513152 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003192076228686058, + "loss": 1.428, + "theoretical_loss": 3.611949630284159, + "tokens_seen": 1214578688 + }, + { + "epoch": 0.37, + "learning_rate": 0.000319197592778335, + "loss": 1.2877, + "theoretical_loss": 3.6119318850053386, + "tokens_seen": 1214644224 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003191875626880642, + "loss": 1.373, + "theoretical_loss": 3.611914140952005, + "tokens_seen": 1214709760 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031917753259779337, + "loss": 1.3152, + "theoretical_loss": 3.6118963981240073, + "tokens_seen": 1214775296 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003191675025075226, + "loss": 1.3647, + "theoretical_loss": 3.6118786565211947, + "tokens_seen": 1214840832 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031915747241725173, + "loss": 1.3269, + "theoretical_loss": 3.6118609161434168, + "tokens_seen": 1214906368 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031914744232698096, + "loss": 1.2429, + "theoretical_loss": 3.611843176990523, + "tokens_seen": 1214971904 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003191374122367101, + "loss": 1.3419, + "theoretical_loss": 3.6118254390623625, + "tokens_seen": 1215037440 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003191273821464393, + "loss": 1.362, + "theoretical_loss": 3.611807702358785, + "tokens_seen": 1215102976 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003191173520561685, + "loss": 1.3035, + "theoretical_loss": 3.6117899668796394, + "tokens_seen": 1215168512 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003191073219658977, + "loss": 1.4062, + "theoretical_loss": 3.6117722326247756, + "tokens_seen": 1215234048 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031909729187562687, + "loss": 1.3417, + "theoretical_loss": 3.611754499594043, + "tokens_seen": 1215299584 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031908726178535605, + "loss": 1.3695, + "theoretical_loss": 3.6117367677872907, + "tokens_seen": 1215365120 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031907723169508523, + "loss": 1.4011, + "theoretical_loss": 3.6117190372043684, + "tokens_seen": 1215430656 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031906720160481447, + "loss": 1.2438, + "theoretical_loss": 3.611701307845126, + "tokens_seen": 1215496192 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031905717151454365, + "loss": 1.3065, + "theoretical_loss": 3.611683579709413, + "tokens_seen": 1215561728 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031904714142427283, + "loss": 1.3819, + "theoretical_loss": 3.6116658527970786, + "tokens_seen": 1215627264 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 406930, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3191826343536377, + "objective/train/theoretical_loss": 3.611648127107973, + "objective/train/tokens_used": 1215692800, + "theoretical_loss": 3.611648127107973, + "tokens_seen": 1215692800 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031903711133400206, + "loss": 1.3246, + "theoretical_loss": 3.611648127107973, + "tokens_seen": 1215692800 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003190270812437312, + "loss": 1.3571, + "theoretical_loss": 3.6116304026419446, + "tokens_seen": 1215758336 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003190170511534604, + "loss": 1.3445, + "theoretical_loss": 3.611612679398845, + "tokens_seen": 1215823872 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031900702106318955, + "loss": 1.4183, + "theoretical_loss": 3.6115949573785224, + "tokens_seen": 1215889408 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003189969909729188, + "loss": 1.3383, + "theoretical_loss": 3.6115772365808274, + "tokens_seen": 1215954944 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031898696088264797, + "loss": 1.2618, + "theoretical_loss": 3.611559517005609, + "tokens_seen": 1216020480 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031897693079237715, + "loss": 1.2867, + "theoretical_loss": 3.611541798652718, + "tokens_seen": 1216086016 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031896690070210633, + "loss": 1.3089, + "theoretical_loss": 3.6115240815220035, + "tokens_seen": 1216151552 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003189568706118355, + "loss": 1.3895, + "theoretical_loss": 3.611506365613315, + "tokens_seen": 1216217088 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003189468405215647, + "loss": 1.3405, + "theoretical_loss": 3.6114886509265034, + "tokens_seen": 1216282624 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031893681043129393, + "loss": 1.4029, + "theoretical_loss": 3.6114709374614176, + "tokens_seen": 1216348160 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031892678034102306, + "loss": 1.3439, + "theoretical_loss": 3.611453225217908, + "tokens_seen": 1216413696 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003189167502507523, + "loss": 1.3766, + "theoretical_loss": 3.6114355141958248, + "tokens_seen": 1216479232 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003189067201604814, + "loss": 1.3609, + "theoretical_loss": 3.6114178043950176, + "tokens_seen": 1216544768 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031889669007021065, + "loss": 1.3308, + "theoretical_loss": 3.6114000958153363, + "tokens_seen": 1216610304 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031888665997993984, + "loss": 1.3527, + "theoretical_loss": 3.611382388456631, + "tokens_seen": 1216675840 + }, + { + "epoch": 0.37, + "learning_rate": 0.000318876629889669, + "loss": 1.3166, + "theoretical_loss": 3.611364682318752, + "tokens_seen": 1216741376 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003188665997993982, + "loss": 1.2619, + "theoretical_loss": 3.611346977401549, + "tokens_seen": 1216806912 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031885656970912743, + "loss": 1.2686, + "theoretical_loss": 3.6113292737048734, + "tokens_seen": 1216872448 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031884653961885656, + "loss": 1.3853, + "theoretical_loss": 3.6113115712285735, + "tokens_seen": 1216937984 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003188365095285858, + "loss": 1.3108, + "theoretical_loss": 3.6112938699725, + "tokens_seen": 1217003520 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003188264794383149, + "loss": 1.3519, + "theoretical_loss": 3.611276169936504, + "tokens_seen": 1217069056 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031881644934804416, + "loss": 1.39, + "theoretical_loss": 3.6112584711204345, + "tokens_seen": 1217134592 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031880641925777334, + "loss": 1.3804, + "theoretical_loss": 3.611240773524143, + "tokens_seen": 1217200128 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003187963891675025, + "loss": 1.3157, + "theoretical_loss": 3.6112230771474785, + "tokens_seen": 1217265664 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003187863590772317, + "loss": 1.3596, + "theoretical_loss": 3.611205381990292, + "tokens_seen": 1217331200 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003187763289869609, + "loss": 1.234, + "theoretical_loss": 3.611187688052433, + "tokens_seen": 1217396736 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031876629889669006, + "loss": 1.2715, + "theoretical_loss": 3.6111699953337535, + "tokens_seen": 1217462272 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003187562688064193, + "loss": 1.3775, + "theoretical_loss": 3.6111523038341025, + "tokens_seen": 1217527808 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003187462387161484, + "loss": 1.3865, + "theoretical_loss": 3.6111346135533307, + "tokens_seen": 1217593344 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031873620862587766, + "loss": 1.3166, + "theoretical_loss": 3.611116924491289, + "tokens_seen": 1217658880 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003187261785356068, + "loss": 1.4168, + "theoretical_loss": 3.611099236647827, + "tokens_seen": 1217724416 + }, + { + "epoch": 0.37, + "learning_rate": 0.000318716148445336, + "loss": 1.3265, + "theoretical_loss": 3.6110815500227957, + "tokens_seen": 1217789952 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003187061183550652, + "loss": 1.2689, + "theoretical_loss": 3.611063864616046, + "tokens_seen": 1217855488 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003186960882647944, + "loss": 1.311, + "theoretical_loss": 3.6110461804274276, + "tokens_seen": 1217921024 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031868605817452357, + "loss": 1.2365, + "theoretical_loss": 3.6110284974567914, + "tokens_seen": 1217986560 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003186760280842528, + "loss": 1.2694, + "theoretical_loss": 3.611010815703988, + "tokens_seen": 1218052096 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031866599799398193, + "loss": 1.3975, + "theoretical_loss": 3.610993135168868, + "tokens_seen": 1218117632 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031865596790371116, + "loss": 1.257, + "theoretical_loss": 3.6109754558512828, + "tokens_seen": 1218183168 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003186459378134403, + "loss": 1.3786, + "theoretical_loss": 3.6109577777510813, + "tokens_seen": 1218248704 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003186359077231695, + "loss": 1.3431, + "theoretical_loss": 3.610940100868116, + "tokens_seen": 1218314240 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003186258776328987, + "loss": 1.3187, + "theoretical_loss": 3.6109224252022365, + "tokens_seen": 1218379776 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003186158475426279, + "loss": 1.3498, + "theoretical_loss": 3.610904750753294, + "tokens_seen": 1218445312 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031860581745235707, + "loss": 1.3074, + "theoretical_loss": 3.610887077521139, + "tokens_seen": 1218510848 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031859578736208625, + "loss": 1.2947, + "theoretical_loss": 3.610869405505622, + "tokens_seen": 1218576384 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031858575727181543, + "loss": 1.3149, + "theoretical_loss": 3.6108517347065954, + "tokens_seen": 1218641920 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031857572718154467, + "loss": 1.2998, + "theoretical_loss": 3.610834065123908, + "tokens_seen": 1218707456 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003185656970912738, + "loss": 1.4106, + "theoretical_loss": 3.610816396757412, + "tokens_seen": 1218772992 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031855566700100303, + "loss": 1.3722, + "theoretical_loss": 3.610798729606958, + "tokens_seen": 1218838528 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003185456369107322, + "loss": 1.1896, + "theoretical_loss": 3.6107810636723965, + "tokens_seen": 1218904064 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 407741, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3172881603240967, + "objective/train/theoretical_loss": 3.6107633989535786, + "objective/train/tokens_used": 1218969600, + "theoretical_loss": 3.6107633989535786, + "tokens_seen": 1218969600 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003185356068204614, + "loss": 1.3701, + "theoretical_loss": 3.6107633989535786, + "tokens_seen": 1218969600 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031852557673019057, + "loss": 1.3051, + "theoretical_loss": 3.6107457354503554, + "tokens_seen": 1219035136 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031851554663991975, + "loss": 1.3733, + "theoretical_loss": 3.610728073162578, + "tokens_seen": 1219100672 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031850551654964893, + "loss": 1.2468, + "theoretical_loss": 3.610710412090098, + "tokens_seen": 1219166208 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031849548645937817, + "loss": 1.2404, + "theoretical_loss": 3.610692752232765, + "tokens_seen": 1219231744 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003184854563691073, + "loss": 1.3222, + "theoretical_loss": 3.6106750935904315, + "tokens_seen": 1219297280 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031847542627883653, + "loss": 1.3178, + "theoretical_loss": 3.610657436162948, + "tokens_seen": 1219362816 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031846539618856566, + "loss": 1.3013, + "theoretical_loss": 3.6106397799501653, + "tokens_seen": 1219428352 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003184553660982949, + "loss": 1.3878, + "theoretical_loss": 3.6106221249519352, + "tokens_seen": 1219493888 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003184453360080241, + "loss": 1.3021, + "theoretical_loss": 3.6106044711681085, + "tokens_seen": 1219559424 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031843530591775326, + "loss": 1.4465, + "theoretical_loss": 3.6105868185985366, + "tokens_seen": 1219624960 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031842527582748244, + "loss": 1.333, + "theoretical_loss": 3.6105691672430704, + "tokens_seen": 1219690496 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003184152457372116, + "loss": 1.4041, + "theoretical_loss": 3.6105515171015616, + "tokens_seen": 1219756032 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003184052156469408, + "loss": 1.3253, + "theoretical_loss": 3.6105338681738615, + "tokens_seen": 1219821568 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031839518555667004, + "loss": 1.3, + "theoretical_loss": 3.610516220459821, + "tokens_seen": 1219887104 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031838515546639916, + "loss": 1.2462, + "theoretical_loss": 3.610498573959292, + "tokens_seen": 1219952640 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003183751253761284, + "loss": 1.3306, + "theoretical_loss": 3.610480928672125, + "tokens_seen": 1220018176 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003183650952858576, + "loss": 1.2269, + "theoretical_loss": 3.610463284598173, + "tokens_seen": 1220083712 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031835506519558676, + "loss": 1.3507, + "theoretical_loss": 3.610445641737285, + "tokens_seen": 1220149248 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031834503510531594, + "loss": 1.395, + "theoretical_loss": 3.610428000089315, + "tokens_seen": 1220214784 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003183350050150451, + "loss": 1.2572, + "theoretical_loss": 3.6104103596541126, + "tokens_seen": 1220280320 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003183249749247743, + "loss": 1.4176, + "theoretical_loss": 3.6103927204315305, + "tokens_seen": 1220345856 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031831494483450354, + "loss": 1.2886, + "theoretical_loss": 3.6103750824214194, + "tokens_seen": 1220411392 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003183049147442327, + "loss": 1.3321, + "theoretical_loss": 3.6103574456236314, + "tokens_seen": 1220476928 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003182948846539619, + "loss": 1.3341, + "theoretical_loss": 3.6103398100380177, + "tokens_seen": 1220542464 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003182848545636911, + "loss": 1.2229, + "theoretical_loss": 3.61032217566443, + "tokens_seen": 1220608000 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031827482447342026, + "loss": 1.3015, + "theoretical_loss": 3.6103045425027203, + "tokens_seen": 1220673536 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003182647943831495, + "loss": 1.3246, + "theoretical_loss": 3.61028691055274, + "tokens_seen": 1220739072 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003182547642928786, + "loss": 1.2866, + "theoretical_loss": 3.6102692798143408, + "tokens_seen": 1220804608 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031824473420260786, + "loss": 1.3859, + "theoretical_loss": 3.610251650287374, + "tokens_seen": 1220870144 + }, + { + "epoch": 0.37, + "learning_rate": 0.000318234704112337, + "loss": 1.3386, + "theoretical_loss": 3.610234021971692, + "tokens_seen": 1220935680 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003182246740220662, + "loss": 1.3173, + "theoretical_loss": 3.6102163948671464, + "tokens_seen": 1221001216 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003182146439317954, + "loss": 1.3663, + "theoretical_loss": 3.6101987689735884, + "tokens_seen": 1221066752 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003182046138415246, + "loss": 1.335, + "theoretical_loss": 3.6101811442908707, + "tokens_seen": 1221132288 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031819458375125377, + "loss": 1.3796, + "theoretical_loss": 3.610163520818845, + "tokens_seen": 1221197824 + }, + { + "epoch": 0.37, + "learning_rate": 0.000318184553660983, + "loss": 1.3421, + "theoretical_loss": 3.6101458985573625, + "tokens_seen": 1221263360 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031817452357071213, + "loss": 1.3074, + "theoretical_loss": 3.6101282775062753, + "tokens_seen": 1221328896 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031816449348044136, + "loss": 1.2952, + "theoretical_loss": 3.610110657665436, + "tokens_seen": 1221394432 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003181544633901705, + "loss": 1.3334, + "theoretical_loss": 3.6100930390346955, + "tokens_seen": 1221459968 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003181444332998997, + "loss": 1.2615, + "theoretical_loss": 3.610075421613907, + "tokens_seen": 1221525504 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003181344032096289, + "loss": 1.3018, + "theoretical_loss": 3.6100578054029215, + "tokens_seen": 1221591040 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003181243731193581, + "loss": 1.3056, + "theoretical_loss": 3.610040190401591, + "tokens_seen": 1221656576 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031811434302908727, + "loss": 1.2469, + "theoretical_loss": 3.6100225766097687, + "tokens_seen": 1221722112 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031810431293881645, + "loss": 1.3986, + "theoretical_loss": 3.610004964027305, + "tokens_seen": 1221787648 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031809428284854563, + "loss": 1.3469, + "theoretical_loss": 3.609987352654054, + "tokens_seen": 1221853184 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031808425275827487, + "loss": 1.3297, + "theoretical_loss": 3.6099697424898656, + "tokens_seen": 1221918720 + }, + { + "epoch": 0.37, + "learning_rate": 0.000318074222668004, + "loss": 1.3384, + "theoretical_loss": 3.609952133534594, + "tokens_seen": 1221984256 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031806419257773323, + "loss": 1.3902, + "theoretical_loss": 3.6099345257880895, + "tokens_seen": 1222049792 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003180541624874624, + "loss": 1.3194, + "theoretical_loss": 3.609916919250206, + "tokens_seen": 1222115328 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003180441323971916, + "loss": 1.3024, + "theoretical_loss": 3.6098993139207947, + "tokens_seen": 1222180864 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 409065, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2790570259094238, + "objective/train/theoretical_loss": 3.6098817097997085, + "objective/train/tokens_used": 1222246400, + "theoretical_loss": 3.6098817097997085, + "tokens_seen": 1222246400 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031803410230692077, + "loss": 1.2589, + "theoretical_loss": 3.6098817097997085, + "tokens_seen": 1222246400 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031802407221664995, + "loss": 1.3617, + "theoretical_loss": 3.609864106886799, + "tokens_seen": 1222311936 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031801404212637913, + "loss": 1.3771, + "theoretical_loss": 3.6098465051819186, + "tokens_seen": 1222377472 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031800401203610837, + "loss": 1.3451, + "theoretical_loss": 3.60982890468492, + "tokens_seen": 1222443008 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003179939819458375, + "loss": 1.2011, + "theoretical_loss": 3.609811305395656, + "tokens_seen": 1222508544 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031798395185556673, + "loss": 1.2645, + "theoretical_loss": 3.609793707313978, + "tokens_seen": 1222574080 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031797392176529586, + "loss": 1.2648, + "theoretical_loss": 3.609776110439739, + "tokens_seen": 1222639616 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003179638916750251, + "loss": 1.2794, + "theoretical_loss": 3.609758514772791, + "tokens_seen": 1222705152 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003179538615847543, + "loss": 1.2817, + "theoretical_loss": 3.609740920312987, + "tokens_seen": 1222770688 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031794383149448346, + "loss": 1.3388, + "theoretical_loss": 3.609723327060179, + "tokens_seen": 1222836224 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031793380140421264, + "loss": 1.4348, + "theoretical_loss": 3.60970573501422, + "tokens_seen": 1222901760 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003179237713139418, + "loss": 1.3446, + "theoretical_loss": 3.6096881441749624, + "tokens_seen": 1222967296 + }, + { + "epoch": 0.37, + "learning_rate": 0.000317913741223671, + "loss": 1.3425, + "theoretical_loss": 3.6096705545422587, + "tokens_seen": 1223032832 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031790371113340024, + "loss": 1.414, + "theoretical_loss": 3.609652966115961, + "tokens_seen": 1223098368 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031789368104312936, + "loss": 1.2835, + "theoretical_loss": 3.609635378895923, + "tokens_seen": 1223163904 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003178836509528586, + "loss": 1.3538, + "theoretical_loss": 3.6096177928819966, + "tokens_seen": 1223229440 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003178736208625878, + "loss": 1.4114, + "theoretical_loss": 3.609600208074035, + "tokens_seen": 1223294976 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031786359077231696, + "loss": 1.3392, + "theoretical_loss": 3.60958262447189, + "tokens_seen": 1223360512 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031785356068204614, + "loss": 1.3889, + "theoretical_loss": 3.609565042075414, + "tokens_seen": 1223426048 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003178435305917753, + "loss": 1.3702, + "theoretical_loss": 3.609547460884462, + "tokens_seen": 1223491584 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003178335005015045, + "loss": 1.2876, + "theoretical_loss": 3.6095298808988847, + "tokens_seen": 1223557120 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031782347041123374, + "loss": 1.3449, + "theoretical_loss": 3.609512302118536, + "tokens_seen": 1223622656 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031781344032096287, + "loss": 1.2531, + "theoretical_loss": 3.6094947245432683, + "tokens_seen": 1223688192 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003178034102306921, + "loss": 1.3223, + "theoretical_loss": 3.6094771481729344, + "tokens_seen": 1223753728 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031779338014042123, + "loss": 1.36, + "theoretical_loss": 3.609459573007387, + "tokens_seen": 1223819264 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031778335005015046, + "loss": 1.41, + "theoretical_loss": 3.609441999046479, + "tokens_seen": 1223884800 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031777331995987964, + "loss": 1.3237, + "theoretical_loss": 3.6094244262900643, + "tokens_seen": 1223950336 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003177632898696088, + "loss": 1.229, + "theoretical_loss": 3.609406854737995, + "tokens_seen": 1224015872 + }, + { + "epoch": 0.37, + "learning_rate": 0.000317753259779338, + "loss": 1.4632, + "theoretical_loss": 3.609389284390124, + "tokens_seen": 1224081408 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003177432296890672, + "loss": 1.1325, + "theoretical_loss": 3.6093717152463043, + "tokens_seen": 1224146944 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031773319959879637, + "loss": 1.3561, + "theoretical_loss": 3.60935414730639, + "tokens_seen": 1224212480 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003177231695085256, + "loss": 1.2696, + "theoretical_loss": 3.6093365805702327, + "tokens_seen": 1224278016 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031771313941825473, + "loss": 1.3854, + "theoretical_loss": 3.609319015037686, + "tokens_seen": 1224343552 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031770310932798397, + "loss": 1.3314, + "theoretical_loss": 3.609301450708603, + "tokens_seen": 1224409088 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031769307923771315, + "loss": 1.3708, + "theoretical_loss": 3.609283887582838, + "tokens_seen": 1224474624 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031768304914744233, + "loss": 1.3229, + "theoretical_loss": 3.6092663256602426, + "tokens_seen": 1224540160 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003176730190571715, + "loss": 1.2649, + "theoretical_loss": 3.60924876494067, + "tokens_seen": 1224605696 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003176629889669007, + "loss": 1.3804, + "theoretical_loss": 3.6092312054239746, + "tokens_seen": 1224671232 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031765295887662987, + "loss": 1.3265, + "theoretical_loss": 3.6092136471100087, + "tokens_seen": 1224736768 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003176429287863591, + "loss": 1.3604, + "theoretical_loss": 3.609196089998626, + "tokens_seen": 1224802304 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031763289869608823, + "loss": 1.453, + "theoretical_loss": 3.6091785340896796, + "tokens_seen": 1224867840 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031762286860581747, + "loss": 1.3361, + "theoretical_loss": 3.609160979383023, + "tokens_seen": 1224933376 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003176128385155466, + "loss": 1.3488, + "theoretical_loss": 3.609143425878509, + "tokens_seen": 1224998912 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031760280842527583, + "loss": 1.3802, + "theoretical_loss": 3.6091258735759917, + "tokens_seen": 1225064448 + }, + { + "epoch": 0.37, + "learning_rate": 0.000317592778335005, + "loss": 1.3449, + "theoretical_loss": 3.609108322475324, + "tokens_seen": 1225129984 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003175827482447342, + "loss": 1.3544, + "theoretical_loss": 3.60909077257636, + "tokens_seen": 1225195520 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003175727181544634, + "loss": 1.407, + "theoretical_loss": 3.6090732238789522, + "tokens_seen": 1225261056 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003175626880641926, + "loss": 1.3277, + "theoretical_loss": 3.6090556763829547, + "tokens_seen": 1225326592 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003175526579739218, + "loss": 1.2546, + "theoretical_loss": 3.609038130088221, + "tokens_seen": 1225392128 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031754262788365097, + "loss": 1.3014, + "theoretical_loss": 3.609020584994604, + "tokens_seen": 1225457664 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 410324, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4842886924743652, + "objective/train/theoretical_loss": 3.609003041101958, + "objective/train/tokens_used": 1225523200, + "theoretical_loss": 3.609003041101958, + "tokens_seen": 1225523200 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031753259779338015, + "loss": 1.3831, + "theoretical_loss": 3.609003041101958, + "tokens_seen": 1225523200 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031752256770310933, + "loss": 1.2758, + "theoretical_loss": 3.608985498410136, + "tokens_seen": 1225588736 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031751253761283857, + "loss": 1.3263, + "theoretical_loss": 3.6089679569189923, + "tokens_seen": 1225654272 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003175025075225677, + "loss": 1.3422, + "theoretical_loss": 3.60895041662838, + "tokens_seen": 1225719808 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031749247743229693, + "loss": 1.3058, + "theoretical_loss": 3.6089328775381526, + "tokens_seen": 1225785344 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031748244734202606, + "loss": 1.3081, + "theoretical_loss": 3.6089153396481644, + "tokens_seen": 1225850880 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003174724172517553, + "loss": 1.3717, + "theoretical_loss": 3.6088978029582686, + "tokens_seen": 1225916416 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003174623871614845, + "loss": 1.3756, + "theoretical_loss": 3.608880267468319, + "tokens_seen": 1225981952 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031745235707121366, + "loss": 1.3527, + "theoretical_loss": 3.6088627331781695, + "tokens_seen": 1226047488 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031744232698094284, + "loss": 1.3556, + "theoretical_loss": 3.608845200087674, + "tokens_seen": 1226113024 + }, + { + "epoch": 0.37, + "learning_rate": 0.000317432296890672, + "loss": 1.3056, + "theoretical_loss": 3.6088276681966858, + "tokens_seen": 1226178560 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003174222668004012, + "loss": 1.3255, + "theoretical_loss": 3.608810137505059, + "tokens_seen": 1226244096 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031741223671013044, + "loss": 1.3165, + "theoretical_loss": 3.6087926080126476, + "tokens_seen": 1226309632 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031740220661985956, + "loss": 1.3785, + "theoretical_loss": 3.6087750797193054, + "tokens_seen": 1226375168 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003173921765295888, + "loss": 1.4173, + "theoretical_loss": 3.608757552624887, + "tokens_seen": 1226440704 + }, + { + "epoch": 0.37, + "learning_rate": 0.000317382146439318, + "loss": 1.4556, + "theoretical_loss": 3.6087400267292447, + "tokens_seen": 1226506240 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031737211634904716, + "loss": 1.3894, + "theoretical_loss": 3.6087225020322338, + "tokens_seen": 1226571776 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031736208625877634, + "loss": 1.3452, + "theoretical_loss": 3.6087049785337078, + "tokens_seen": 1226637312 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003173520561685055, + "loss": 1.3792, + "theoretical_loss": 3.608687456233521, + "tokens_seen": 1226702848 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003173420260782347, + "loss": 1.267, + "theoretical_loss": 3.6086699351315272, + "tokens_seen": 1226768384 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031733199598796394, + "loss": 1.2386, + "theoretical_loss": 3.6086524152275805, + "tokens_seen": 1226833920 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031732196589769307, + "loss": 1.3271, + "theoretical_loss": 3.6086348965215347, + "tokens_seen": 1226899456 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003173119358074223, + "loss": 1.2306, + "theoretical_loss": 3.6086173790132445, + "tokens_seen": 1226964992 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031730190571715143, + "loss": 1.3219, + "theoretical_loss": 3.608599862702564, + "tokens_seen": 1227030528 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031729187562688066, + "loss": 1.3255, + "theoretical_loss": 3.608582347589347, + "tokens_seen": 1227096064 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031728184553660984, + "loss": 1.3985, + "theoretical_loss": 3.608564833673447, + "tokens_seen": 1227161600 + }, + { + "epoch": 0.37, + "learning_rate": 0.000317271815446339, + "loss": 1.322, + "theoretical_loss": 3.60854732095472, + "tokens_seen": 1227227136 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003172617853560682, + "loss": 1.3427, + "theoretical_loss": 3.608529809433019, + "tokens_seen": 1227292672 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003172517552657974, + "loss": 1.4062, + "theoretical_loss": 3.6085122991081984, + "tokens_seen": 1227358208 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031724172517552657, + "loss": 1.2566, + "theoretical_loss": 3.6084947899801127, + "tokens_seen": 1227423744 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003172316950852558, + "loss": 1.2854, + "theoretical_loss": 3.608477282048616, + "tokens_seen": 1227489280 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031722166499498493, + "loss": 1.3979, + "theoretical_loss": 3.6084597753135625, + "tokens_seen": 1227554816 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031721163490471417, + "loss": 1.3606, + "theoretical_loss": 3.6084422697748075, + "tokens_seen": 1227620352 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031720160481444335, + "loss": 1.3136, + "theoretical_loss": 3.608424765432205, + "tokens_seen": 1227685888 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031719157472417253, + "loss": 1.4297, + "theoretical_loss": 3.608407262285608, + "tokens_seen": 1227751424 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003171815446339017, + "loss": 1.404, + "theoretical_loss": 3.608389760334873, + "tokens_seen": 1227816960 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003171715145436309, + "loss": 1.3375, + "theoretical_loss": 3.6083722595798537, + "tokens_seen": 1227882496 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031716148445336007, + "loss": 1.4213, + "theoretical_loss": 3.6083547600204042, + "tokens_seen": 1227948032 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003171514543630893, + "loss": 1.3461, + "theoretical_loss": 3.6083372616563794, + "tokens_seen": 1228013568 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031714142427281843, + "loss": 1.2803, + "theoretical_loss": 3.6083197644876335, + "tokens_seen": 1228079104 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031713139418254767, + "loss": 1.306, + "theoretical_loss": 3.608302268514022, + "tokens_seen": 1228144640 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003171213640922768, + "loss": 1.2637, + "theoretical_loss": 3.608284773735398, + "tokens_seen": 1228210176 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031711133400200603, + "loss": 1.3821, + "theoretical_loss": 3.6082672801516176, + "tokens_seen": 1228275712 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003171013039117352, + "loss": 1.4441, + "theoretical_loss": 3.608249787762534, + "tokens_seen": 1228341248 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003170912738214644, + "loss": 1.282, + "theoretical_loss": 3.6082322965680036, + "tokens_seen": 1228406784 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003170812437311936, + "loss": 1.2078, + "theoretical_loss": 3.6082148065678794, + "tokens_seen": 1228472320 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003170712136409228, + "loss": 1.2495, + "theoretical_loss": 3.608197317762018, + "tokens_seen": 1228537856 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031706118355065194, + "loss": 1.296, + "theoretical_loss": 3.6081798301502723, + "tokens_seen": 1228603392 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003170511534603812, + "loss": 1.3921, + "theoretical_loss": 3.6081623437324977, + "tokens_seen": 1228668928 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003170411233701103, + "loss": 1.4276, + "theoretical_loss": 3.60814485850855, + "tokens_seen": 1228734464 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 411219, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2712643146514893, + "objective/train/theoretical_loss": 3.608127374478282, + "objective/train/tokens_used": 1228800000, + "theoretical_loss": 3.608127374478282, + "tokens_seen": 1228800000 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031703109327983954, + "loss": 1.2766, + "theoretical_loss": 3.608127374478282, + "tokens_seen": 1228800000 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003170210631895687, + "loss": 1.4157, + "theoretical_loss": 3.6081098916415506, + "tokens_seen": 1228865536 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003170110330992979, + "loss": 1.5354, + "theoretical_loss": 3.6080924099982092, + "tokens_seen": 1228931072 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003170010030090271, + "loss": 1.3722, + "theoretical_loss": 3.608074929548114, + "tokens_seen": 1228996608 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031699097291875626, + "loss": 1.2741, + "theoretical_loss": 3.6080574502911187, + "tokens_seen": 1229062144 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031698094282848544, + "loss": 1.3294, + "theoretical_loss": 3.608039972227079, + "tokens_seen": 1229127680 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003169709127382147, + "loss": 1.3234, + "theoretical_loss": 3.6080224953558497, + "tokens_seen": 1229193216 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003169608826479438, + "loss": 1.3487, + "theoretical_loss": 3.6080050196772855, + "tokens_seen": 1229258752 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031695085255767304, + "loss": 1.3902, + "theoretical_loss": 3.607987545191242, + "tokens_seen": 1229324288 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031694082246740217, + "loss": 1.3379, + "theoretical_loss": 3.607970071897574, + "tokens_seen": 1229389824 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003169307923771314, + "loss": 1.4386, + "theoretical_loss": 3.6079525997961364, + "tokens_seen": 1229455360 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003169207622868606, + "loss": 1.2751, + "theoretical_loss": 3.6079351288867842, + "tokens_seen": 1229520896 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031691073219658976, + "loss": 1.3294, + "theoretical_loss": 3.607917659169373, + "tokens_seen": 1229586432 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031690070210631894, + "loss": 1.2941, + "theoretical_loss": 3.607900190643758, + "tokens_seen": 1229651968 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003168906720160482, + "loss": 1.2711, + "theoretical_loss": 3.607882723309794, + "tokens_seen": 1229717504 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003168806419257773, + "loss": 1.3933, + "theoretical_loss": 3.607865257167336, + "tokens_seen": 1229783040 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031687061183550654, + "loss": 1.2198, + "theoretical_loss": 3.60784779221624, + "tokens_seen": 1229848576 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031686058174523567, + "loss": 1.3526, + "theoretical_loss": 3.6078303284563606, + "tokens_seen": 1229914112 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003168505516549649, + "loss": 1.3014, + "theoretical_loss": 3.6078128658875537, + "tokens_seen": 1229979648 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003168405215646941, + "loss": 1.3101, + "theoretical_loss": 3.607795404509674, + "tokens_seen": 1230045184 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031683049147442327, + "loss": 1.3824, + "theoretical_loss": 3.607777944322577, + "tokens_seen": 1230110720 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031682046138415245, + "loss": 1.2877, + "theoretical_loss": 3.607760485326118, + "tokens_seen": 1230176256 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031681043129388163, + "loss": 1.353, + "theoretical_loss": 3.6077430275201525, + "tokens_seen": 1230241792 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031680040120361086, + "loss": 1.3128, + "theoretical_loss": 3.607725570904536, + "tokens_seen": 1230307328 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031679037111334004, + "loss": 1.2873, + "theoretical_loss": 3.6077081154791237, + "tokens_seen": 1230372864 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003167803410230692, + "loss": 1.3186, + "theoretical_loss": 3.6076906612437716, + "tokens_seen": 1230438400 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003167703109327984, + "loss": 1.3336, + "theoretical_loss": 3.6076732081983347, + "tokens_seen": 1230503936 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003167602808425276, + "loss": 1.2936, + "theoretical_loss": 3.6076557563426688, + "tokens_seen": 1230569472 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031675025075225677, + "loss": 1.2815, + "theoretical_loss": 3.6076383056766286, + "tokens_seen": 1230635008 + }, + { + "epoch": 0.37, + "learning_rate": 0.000316740220661986, + "loss": 1.4003, + "theoretical_loss": 3.6076208562000707, + "tokens_seen": 1230700544 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031673019057171513, + "loss": 1.3528, + "theoretical_loss": 3.6076034079128503, + "tokens_seen": 1230766080 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031672016048144437, + "loss": 1.253, + "theoretical_loss": 3.6075859608148226, + "tokens_seen": 1230831616 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031671013039117355, + "loss": 1.2929, + "theoretical_loss": 3.6075685149058443, + "tokens_seen": 1230897152 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031670010030090273, + "loss": 1.2981, + "theoretical_loss": 3.60755107018577, + "tokens_seen": 1230962688 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003166900702106319, + "loss": 1.2374, + "theoretical_loss": 3.607533626654456, + "tokens_seen": 1231028224 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003166800401203611, + "loss": 1.3541, + "theoretical_loss": 3.607516184311758, + "tokens_seen": 1231093760 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031667001003009027, + "loss": 1.3613, + "theoretical_loss": 3.6074987431575307, + "tokens_seen": 1231159296 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003166599799398195, + "loss": 1.3124, + "theoretical_loss": 3.6074813031916317, + "tokens_seen": 1231224832 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031664994984954863, + "loss": 1.2399, + "theoretical_loss": 3.607463864413915, + "tokens_seen": 1231290368 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031663991975927787, + "loss": 1.3136, + "theoretical_loss": 3.6074464268242377, + "tokens_seen": 1231355904 + }, + { + "epoch": 0.37, + "learning_rate": 0.000316629889669007, + "loss": 1.3106, + "theoretical_loss": 3.607428990422455, + "tokens_seen": 1231421440 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031661985957873623, + "loss": 1.2697, + "theoretical_loss": 3.607411555208423, + "tokens_seen": 1231486976 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003166098294884654, + "loss": 1.3512, + "theoretical_loss": 3.6073941211819975, + "tokens_seen": 1231552512 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003165997993981946, + "loss": 1.2825, + "theoretical_loss": 3.6073766883430345, + "tokens_seen": 1231618048 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003165897693079238, + "loss": 1.2932, + "theoretical_loss": 3.6073592566913897, + "tokens_seen": 1231683584 + }, + { + "epoch": 0.37, + "learning_rate": 0.000316579739217653, + "loss": 1.3681, + "theoretical_loss": 3.6073418262269197, + "tokens_seen": 1231749120 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031656970912738214, + "loss": 1.2482, + "theoretical_loss": 3.6073243969494793, + "tokens_seen": 1231814656 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003165596790371114, + "loss": 1.3035, + "theoretical_loss": 3.6073069688589254, + "tokens_seen": 1231880192 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003165496489468405, + "loss": 1.2984, + "theoretical_loss": 3.6072895419551143, + "tokens_seen": 1231945728 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031653961885656974, + "loss": 1.3642, + "theoretical_loss": 3.607272116237901, + "tokens_seen": 1232011264 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 412487, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2386159896850586, + "objective/train/theoretical_loss": 3.6072546917071424, + "objective/train/tokens_used": 1232076800, + "theoretical_loss": 3.6072546917071424, + "tokens_seen": 1232076800 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003165295887662989, + "loss": 1.3252, + "theoretical_loss": 3.6072546917071424, + "tokens_seen": 1232076800 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003165195586760281, + "loss": 1.2568, + "theoretical_loss": 3.607237268362695, + "tokens_seen": 1232142336 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003165095285857573, + "loss": 1.2641, + "theoretical_loss": 3.607219846204414, + "tokens_seen": 1232207872 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031649949849548646, + "loss": 1.3407, + "theoretical_loss": 3.607202425232156, + "tokens_seen": 1232273408 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031648946840521564, + "loss": 1.2701, + "theoretical_loss": 3.607185005445777, + "tokens_seen": 1232338944 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003164794383149449, + "loss": 1.3383, + "theoretical_loss": 3.607167586845134, + "tokens_seen": 1232404480 + }, + { + "epoch": 0.37, + "learning_rate": 0.000316469408224674, + "loss": 1.296, + "theoretical_loss": 3.6071501694300814, + "tokens_seen": 1232470016 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031645937813440324, + "loss": 1.2399, + "theoretical_loss": 3.6071327532004775, + "tokens_seen": 1232535552 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031644934804413237, + "loss": 1.2963, + "theoretical_loss": 3.607115338156178, + "tokens_seen": 1232601088 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003164393179538616, + "loss": 1.2968, + "theoretical_loss": 3.6070979242970385, + "tokens_seen": 1232666624 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003164292878635908, + "loss": 1.3734, + "theoretical_loss": 3.6070805116229154, + "tokens_seen": 1232732160 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031641925777331996, + "loss": 1.3368, + "theoretical_loss": 3.607063100133666, + "tokens_seen": 1232797696 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031640922768304914, + "loss": 1.2157, + "theoretical_loss": 3.607045689829146, + "tokens_seen": 1232863232 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003163991975927784, + "loss": 1.3089, + "theoretical_loss": 3.607028280709213, + "tokens_seen": 1232928768 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003163891675025075, + "loss": 1.3128, + "theoretical_loss": 3.6070108727737207, + "tokens_seen": 1232994304 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031637913741223674, + "loss": 1.2085, + "theoretical_loss": 3.6069934660225287, + "tokens_seen": 1233059840 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031636910732196587, + "loss": 1.4498, + "theoretical_loss": 3.6069760604554912, + "tokens_seen": 1233125376 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003163590772316951, + "loss": 1.3598, + "theoretical_loss": 3.6069586560724662, + "tokens_seen": 1233190912 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003163490471414243, + "loss": 1.4059, + "theoretical_loss": 3.6069412528733094, + "tokens_seen": 1233256448 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031633901705115347, + "loss": 1.3166, + "theoretical_loss": 3.6069238508578776, + "tokens_seen": 1233321984 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031632898696088265, + "loss": 1.4161, + "theoretical_loss": 3.6069064500260275, + "tokens_seen": 1233387520 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031631895687061183, + "loss": 1.3195, + "theoretical_loss": 3.6068890503776156, + "tokens_seen": 1233453056 + }, + { + "epoch": 0.37, + "learning_rate": 0.000316308926780341, + "loss": 1.3902, + "theoretical_loss": 3.606871651912498, + "tokens_seen": 1233518592 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031629889669007024, + "loss": 1.3114, + "theoretical_loss": 3.606854254630533, + "tokens_seen": 1233584128 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031628886659979937, + "loss": 1.2467, + "theoretical_loss": 3.606836858531576, + "tokens_seen": 1233649664 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003162788365095286, + "loss": 1.3506, + "theoretical_loss": 3.606819463615483, + "tokens_seen": 1233715200 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031626880641925773, + "loss": 1.285, + "theoretical_loss": 3.6068020698821126, + "tokens_seen": 1233780736 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031625877632898697, + "loss": 1.3133, + "theoretical_loss": 3.60678467733132, + "tokens_seen": 1233846272 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031624874623871615, + "loss": 1.241, + "theoretical_loss": 3.606767285962963, + "tokens_seen": 1233911808 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031623871614844533, + "loss": 1.3046, + "theoretical_loss": 3.6067498957768986, + "tokens_seen": 1233977344 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003162286860581745, + "loss": 1.4569, + "theoretical_loss": 3.606732506772982, + "tokens_seen": 1234042880 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031621865596790375, + "loss": 1.2981, + "theoretical_loss": 3.606715118951072, + "tokens_seen": 1234108416 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003162086258776329, + "loss": 1.294, + "theoretical_loss": 3.6066977323110243, + "tokens_seen": 1234173952 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003161985957873621, + "loss": 1.3083, + "theoretical_loss": 3.6066803468526967, + "tokens_seen": 1234239488 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031618856569709124, + "loss": 1.2948, + "theoretical_loss": 3.6066629625759448, + "tokens_seen": 1234305024 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031617853560682047, + "loss": 1.3442, + "theoretical_loss": 3.6066455794806265, + "tokens_seen": 1234370560 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031616850551654965, + "loss": 1.3391, + "theoretical_loss": 3.6066281975665992, + "tokens_seen": 1234436096 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031615847542627883, + "loss": 1.3363, + "theoretical_loss": 3.6066108168337188, + "tokens_seen": 1234501632 + }, + { + "epoch": 0.37, + "learning_rate": 0.000316148445336008, + "loss": 1.4091, + "theoretical_loss": 3.6065934372818433, + "tokens_seen": 1234567168 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003161384152457372, + "loss": 1.325, + "theoretical_loss": 3.6065760589108296, + "tokens_seen": 1234632704 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003161283851554664, + "loss": 1.2902, + "theoretical_loss": 3.606558681720534, + "tokens_seen": 1234698240 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003161183550651956, + "loss": 1.3679, + "theoretical_loss": 3.6065413057108144, + "tokens_seen": 1234763776 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031610832497492474, + "loss": 1.3251, + "theoretical_loss": 3.606523930881528, + "tokens_seen": 1234829312 + }, + { + "epoch": 0.37, + "learning_rate": 0.000316098294884654, + "loss": 1.3499, + "theoretical_loss": 3.6065065572325317, + "tokens_seen": 1234894848 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003160882647943831, + "loss": 1.2812, + "theoretical_loss": 3.6064891847636824, + "tokens_seen": 1234960384 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031607823470411234, + "loss": 1.4181, + "theoretical_loss": 3.6064718134748377, + "tokens_seen": 1235025920 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003160682046138415, + "loss": 1.3343, + "theoretical_loss": 3.6064544433658545, + "tokens_seen": 1235091456 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003160581745235707, + "loss": 1.2929, + "theoretical_loss": 3.606437074436591, + "tokens_seen": 1235156992 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031604814443329994, + "loss": 1.2985, + "theoretical_loss": 3.6064197066869035, + "tokens_seen": 1235222528 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003160381143430291, + "loss": 1.3609, + "theoretical_loss": 3.606402340116649, + "tokens_seen": 1235288064 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 413449, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.310478925704956, + "objective/train/theoretical_loss": 3.606384974725686, + "objective/train/tokens_used": 1235353600, + "theoretical_loss": 3.606384974725686, + "tokens_seen": 1235353600 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003160280842527583, + "loss": 1.2343, + "theoretical_loss": 3.606384974725686, + "tokens_seen": 1235353600 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003160180541624875, + "loss": 1.3205, + "theoretical_loss": 3.606367610513871, + "tokens_seen": 1235419136 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031600802407221666, + "loss": 1.3455, + "theoretical_loss": 3.6063502474810623, + "tokens_seen": 1235484672 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031599799398194584, + "loss": 1.2334, + "theoretical_loss": 3.6063328856271166, + "tokens_seen": 1235550208 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003159879638916751, + "loss": 1.4276, + "theoretical_loss": 3.6063155249518912, + "tokens_seen": 1235615744 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003159779338014042, + "loss": 1.2899, + "theoretical_loss": 3.606298165455244, + "tokens_seen": 1235681280 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031596790371113344, + "loss": 1.3294, + "theoretical_loss": 3.606280807137032, + "tokens_seen": 1235746816 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031595787362086257, + "loss": 1.4495, + "theoretical_loss": 3.606263449997113, + "tokens_seen": 1235812352 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003159478435305918, + "loss": 1.3289, + "theoretical_loss": 3.606246094035345, + "tokens_seen": 1235877888 + }, + { + "epoch": 0.37, + "learning_rate": 0.000315937813440321, + "loss": 1.4101, + "theoretical_loss": 3.606228739251585, + "tokens_seen": 1235943424 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031592778335005016, + "loss": 1.3289, + "theoretical_loss": 3.6062113856456905, + "tokens_seen": 1236008960 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031591775325977934, + "loss": 1.3703, + "theoretical_loss": 3.6061940332175193, + "tokens_seen": 1236074496 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003159077231695086, + "loss": 1.2968, + "theoretical_loss": 3.606176681966929, + "tokens_seen": 1236140032 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003158976930792377, + "loss": 1.3521, + "theoretical_loss": 3.6061593318937777, + "tokens_seen": 1236205568 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031588766298896694, + "loss": 1.345, + "theoretical_loss": 3.606141982997922, + "tokens_seen": 1236271104 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031587763289869607, + "loss": 1.3973, + "theoretical_loss": 3.6061246352792216, + "tokens_seen": 1236336640 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003158676028084253, + "loss": 1.2957, + "theoretical_loss": 3.606107288737532, + "tokens_seen": 1236402176 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003158575727181545, + "loss": 1.2592, + "theoretical_loss": 3.606089943372712, + "tokens_seen": 1236467712 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031584754262788367, + "loss": 1.3559, + "theoretical_loss": 3.6060725991846194, + "tokens_seen": 1236533248 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031583751253761285, + "loss": 1.2751, + "theoretical_loss": 3.6060552561731116, + "tokens_seen": 1236598784 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031582748244734203, + "loss": 1.3429, + "theoretical_loss": 3.6060379143380468, + "tokens_seen": 1236664320 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003158174523570712, + "loss": 1.351, + "theoretical_loss": 3.6060205736792836, + "tokens_seen": 1236729856 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031580742226680044, + "loss": 1.3985, + "theoretical_loss": 3.6060032341966783, + "tokens_seen": 1236795392 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031579739217652957, + "loss": 1.3492, + "theoretical_loss": 3.6059858958900897, + "tokens_seen": 1236860928 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003157873620862588, + "loss": 1.3591, + "theoretical_loss": 3.6059685587593755, + "tokens_seen": 1236926464 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031577733199598793, + "loss": 1.3693, + "theoretical_loss": 3.605951222804394, + "tokens_seen": 1236992000 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031576730190571717, + "loss": 1.2758, + "theoretical_loss": 3.605933888025003, + "tokens_seen": 1237057536 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031575727181544635, + "loss": 1.3738, + "theoretical_loss": 3.6059165544210607, + "tokens_seen": 1237123072 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031574724172517553, + "loss": 1.42, + "theoretical_loss": 3.6058992219924244, + "tokens_seen": 1237188608 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003157372116349047, + "loss": 1.3654, + "theoretical_loss": 3.605881890738953, + "tokens_seen": 1237254144 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031572718154463395, + "loss": 1.3514, + "theoretical_loss": 3.605864560660504, + "tokens_seen": 1237319680 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003157171514543631, + "loss": 1.3661, + "theoretical_loss": 3.605847231756936, + "tokens_seen": 1237385216 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003157071213640923, + "loss": 1.352, + "theoretical_loss": 3.6058299040281065, + "tokens_seen": 1237450752 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031569709127382144, + "loss": 1.3431, + "theoretical_loss": 3.6058125774738743, + "tokens_seen": 1237516288 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031568706118355067, + "loss": 1.3176, + "theoretical_loss": 3.6057952520940972, + "tokens_seen": 1237581824 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031567703109327985, + "loss": 1.337, + "theoretical_loss": 3.6057779278886337, + "tokens_seen": 1237647360 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031566700100300903, + "loss": 1.2292, + "theoretical_loss": 3.6057606048573416, + "tokens_seen": 1237712896 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003156569709127382, + "loss": 1.2948, + "theoretical_loss": 3.6057432830000797, + "tokens_seen": 1237778432 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003156469408224674, + "loss": 1.3077, + "theoretical_loss": 3.605725962316706, + "tokens_seen": 1237843968 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003156369107321966, + "loss": 1.2844, + "theoretical_loss": 3.6057086428070786, + "tokens_seen": 1237909504 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003156268806419258, + "loss": 1.3061, + "theoretical_loss": 3.6056913244710556, + "tokens_seen": 1237975040 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031561685055165494, + "loss": 1.276, + "theoretical_loss": 3.6056740073084965, + "tokens_seen": 1238040576 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003156068204613842, + "loss": 1.2964, + "theoretical_loss": 3.6056566913192585, + "tokens_seen": 1238106112 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003155967903711133, + "loss": 1.3476, + "theoretical_loss": 3.6056393765032007, + "tokens_seen": 1238171648 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031558676028084254, + "loss": 1.3887, + "theoretical_loss": 3.605622062860181, + "tokens_seen": 1238237184 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003155767301905717, + "loss": 1.3451, + "theoretical_loss": 3.605604750390058, + "tokens_seen": 1238302720 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003155667001003009, + "loss": 1.3839, + "theoretical_loss": 3.6055874390926905, + "tokens_seen": 1238368256 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003155566700100301, + "loss": 1.269, + "theoretical_loss": 3.605570128967937, + "tokens_seen": 1238433792 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003155466399197593, + "loss": 1.2743, + "theoretical_loss": 3.6055528200156557, + "tokens_seen": 1238499328 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031553660982948844, + "loss": 1.323, + "theoretical_loss": 3.605535512235705, + "tokens_seen": 1238564864 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 414658, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3231377601623535, + "objective/train/theoretical_loss": 3.605518205627944, + "objective/train/tokens_used": 1238630400, + "theoretical_loss": 3.605518205627944, + "tokens_seen": 1238630400 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003155265797392177, + "loss": 1.2737, + "theoretical_loss": 3.605518205627944, + "tokens_seen": 1238630400 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003155165496489468, + "loss": 1.2825, + "theoretical_loss": 3.605500900192231, + "tokens_seen": 1238695936 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031550651955867604, + "loss": 1.3376, + "theoretical_loss": 3.6054835959284244, + "tokens_seen": 1238761472 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003154964894684052, + "loss": 1.3921, + "theoretical_loss": 3.6054662928363834, + "tokens_seen": 1238827008 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003154864593781344, + "loss": 1.2984, + "theoretical_loss": 3.605448990915966, + "tokens_seen": 1238892544 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003154764292878636, + "loss": 1.2444, + "theoretical_loss": 3.605431690167032, + "tokens_seen": 1238958080 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031546639919759277, + "loss": 1.3729, + "theoretical_loss": 3.6054143905894387, + "tokens_seen": 1239023616 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031545636910732195, + "loss": 1.3365, + "theoretical_loss": 3.605397092183046, + "tokens_seen": 1239089152 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003154463390170512, + "loss": 1.225, + "theoretical_loss": 3.6053797949477118, + "tokens_seen": 1239154688 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003154363089267803, + "loss": 1.3886, + "theoretical_loss": 3.605362498883296, + "tokens_seen": 1239220224 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031542627883650954, + "loss": 1.3218, + "theoretical_loss": 3.605345203989656, + "tokens_seen": 1239285760 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003154162487462387, + "loss": 1.3511, + "theoretical_loss": 3.6053279102666513, + "tokens_seen": 1239351296 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003154062186559679, + "loss": 1.2908, + "theoretical_loss": 3.6053106177141414, + "tokens_seen": 1239416832 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003153961885656971, + "loss": 1.3166, + "theoretical_loss": 3.605293326331984, + "tokens_seen": 1239482368 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031538615847542627, + "loss": 1.3655, + "theoretical_loss": 3.6052760361200393, + "tokens_seen": 1239547904 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031537612838515545, + "loss": 1.2652, + "theoretical_loss": 3.6052587470781647, + "tokens_seen": 1239613440 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003153660982948847, + "loss": 1.3873, + "theoretical_loss": 3.605241459206221, + "tokens_seen": 1239678976 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003153560682046138, + "loss": 1.3879, + "theoretical_loss": 3.605224172504066, + "tokens_seen": 1239744512 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031534603811434305, + "loss": 1.2623, + "theoretical_loss": 3.6052068869715583, + "tokens_seen": 1239810048 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003153360080240722, + "loss": 1.3676, + "theoretical_loss": 3.605189602608558, + "tokens_seen": 1239875584 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003153259779338014, + "loss": 1.29, + "theoretical_loss": 3.605172319414924, + "tokens_seen": 1239941120 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003153159478435306, + "loss": 1.4263, + "theoretical_loss": 3.6051550373905146, + "tokens_seen": 1240006656 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031530591775325977, + "loss": 1.303, + "theoretical_loss": 3.60513775653519, + "tokens_seen": 1240072192 + }, + { + "epoch": 0.38, + "learning_rate": 0.000315295887662989, + "loss": 1.3997, + "theoretical_loss": 3.6051204768488088, + "tokens_seen": 1240137728 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031528585757271813, + "loss": 1.3904, + "theoretical_loss": 3.6051031983312294, + "tokens_seen": 1240203264 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031527582748244737, + "loss": 1.3272, + "theoretical_loss": 3.6050859209823125, + "tokens_seen": 1240268800 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031526579739217655, + "loss": 1.2534, + "theoretical_loss": 3.605068644801916, + "tokens_seen": 1240334336 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031525576730190573, + "loss": 1.3619, + "theoretical_loss": 3.6050513697899, + "tokens_seen": 1240399872 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003152457372116349, + "loss": 1.3505, + "theoretical_loss": 3.6050340959461233, + "tokens_seen": 1240465408 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031523570712136415, + "loss": 1.2669, + "theoretical_loss": 3.6050168232704456, + "tokens_seen": 1240530944 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003152256770310933, + "loss": 1.339, + "theoretical_loss": 3.6049995517627256, + "tokens_seen": 1240596480 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003152156469408225, + "loss": 1.235, + "theoretical_loss": 3.604982281422823, + "tokens_seen": 1240662016 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031520561685055164, + "loss": 1.2704, + "theoretical_loss": 3.604965012250597, + "tokens_seen": 1240727552 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031519558676028087, + "loss": 1.3779, + "theoretical_loss": 3.6049477442459077, + "tokens_seen": 1240793088 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031518555667001005, + "loss": 1.2375, + "theoretical_loss": 3.604930477408613, + "tokens_seen": 1240858624 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031517552657973924, + "loss": 1.2614, + "theoretical_loss": 3.6049132117385736, + "tokens_seen": 1240924160 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003151654964894684, + "loss": 1.3092, + "theoretical_loss": 3.604895947235649, + "tokens_seen": 1240989696 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003151554663991976, + "loss": 1.3147, + "theoretical_loss": 3.604878683899698, + "tokens_seen": 1241055232 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003151454363089268, + "loss": 1.3303, + "theoretical_loss": 3.60486142173058, + "tokens_seen": 1241120768 + }, + { + "epoch": 0.38, + "learning_rate": 0.000315135406218656, + "loss": 1.3282, + "theoretical_loss": 3.604844160728155, + "tokens_seen": 1241186304 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031512537612838514, + "loss": 1.3534, + "theoretical_loss": 3.6048269008922826, + "tokens_seen": 1241251840 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003151153460381144, + "loss": 1.3539, + "theoretical_loss": 3.604809642222822, + "tokens_seen": 1241317376 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003151053159478435, + "loss": 1.3049, + "theoretical_loss": 3.6047923847196337, + "tokens_seen": 1241382912 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031509528585757274, + "loss": 1.3207, + "theoretical_loss": 3.6047751283825757, + "tokens_seen": 1241448448 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003150852557673019, + "loss": 1.3205, + "theoretical_loss": 3.6047578732115086, + "tokens_seen": 1241513984 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003150752256770311, + "loss": 1.3092, + "theoretical_loss": 3.604740619206293, + "tokens_seen": 1241579520 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003150651955867603, + "loss": 1.4084, + "theoretical_loss": 3.6047233663667866, + "tokens_seen": 1241645056 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003150551654964895, + "loss": 1.3159, + "theoretical_loss": 3.604706114692851, + "tokens_seen": 1241710592 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031504513540621864, + "loss": 1.2806, + "theoretical_loss": 3.604688864184345, + "tokens_seen": 1241776128 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003150351053159479, + "loss": 1.2784, + "theoretical_loss": 3.604671614841128, + "tokens_seen": 1241841664 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 415473, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2679164409637451, + "objective/train/theoretical_loss": 3.6046543666630604, + "objective/train/tokens_used": 1241907200, + "theoretical_loss": 3.6046543666630604, + "tokens_seen": 1241907200 + }, + { + "epoch": 0.38, + "learning_rate": 0.000315025075225677, + "loss": 1.2621, + "theoretical_loss": 3.6046543666630604, + "tokens_seen": 1241907200 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031501504513540624, + "loss": 1.2963, + "theoretical_loss": 3.604637119650002, + "tokens_seen": 1241972736 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003150050150451354, + "loss": 1.3125, + "theoretical_loss": 3.6046198738018127, + "tokens_seen": 1242038272 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003149949849548646, + "loss": 1.352, + "theoretical_loss": 3.604602629118352, + "tokens_seen": 1242103808 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003149849548645938, + "loss": 1.2914, + "theoretical_loss": 3.6045853855994796, + "tokens_seen": 1242169344 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031497492477432297, + "loss": 1.3738, + "theoretical_loss": 3.6045681432450567, + "tokens_seen": 1242234880 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031496489468405215, + "loss": 1.3721, + "theoretical_loss": 3.6045509020549416, + "tokens_seen": 1242300416 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003149548645937814, + "loss": 1.3151, + "theoretical_loss": 3.6045336620289956, + "tokens_seen": 1242365952 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003149448345035105, + "loss": 1.3143, + "theoretical_loss": 3.6045164231670777, + "tokens_seen": 1242431488 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031493480441323974, + "loss": 1.4184, + "theoretical_loss": 3.6044991854690487, + "tokens_seen": 1242497024 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003149247743229689, + "loss": 1.4661, + "theoretical_loss": 3.604481948934768, + "tokens_seen": 1242562560 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003149147442326981, + "loss": 1.2817, + "theoretical_loss": 3.6044647135640955, + "tokens_seen": 1242628096 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003149047141424273, + "loss": 1.2588, + "theoretical_loss": 3.6044474793568924, + "tokens_seen": 1242693632 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031489468405215647, + "loss": 1.329, + "theoretical_loss": 3.6044302463130173, + "tokens_seen": 1242759168 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031488465396188565, + "loss": 1.2743, + "theoretical_loss": 3.604413014432332, + "tokens_seen": 1242824704 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003148746238716149, + "loss": 1.3881, + "theoretical_loss": 3.604395783714695, + "tokens_seen": 1242890240 + }, + { + "epoch": 0.38, + "learning_rate": 0.000314864593781344, + "loss": 1.2578, + "theoretical_loss": 3.6043785541599678, + "tokens_seen": 1242955776 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031485456369107325, + "loss": 1.3554, + "theoretical_loss": 3.60436132576801, + "tokens_seen": 1243021312 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003148445336008024, + "loss": 1.3174, + "theoretical_loss": 3.6043440985386814, + "tokens_seen": 1243086848 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003148345035105316, + "loss": 1.2864, + "theoretical_loss": 3.6043268724718436, + "tokens_seen": 1243152384 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003148244734202608, + "loss": 1.2968, + "theoretical_loss": 3.604309647567356, + "tokens_seen": 1243217920 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031481444332998997, + "loss": 1.2355, + "theoretical_loss": 3.604292423825078, + "tokens_seen": 1243283456 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031480441323971915, + "loss": 1.3534, + "theoretical_loss": 3.6042752012448718, + "tokens_seen": 1243348992 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031479438314944833, + "loss": 1.3973, + "theoretical_loss": 3.6042579798265963, + "tokens_seen": 1243414528 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003147843530591775, + "loss": 1.2958, + "theoretical_loss": 3.6042407595701125, + "tokens_seen": 1243480064 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031477432296890675, + "loss": 1.3432, + "theoretical_loss": 3.6042235404752807, + "tokens_seen": 1243545600 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003147642928786359, + "loss": 1.3936, + "theoretical_loss": 3.6042063225419616, + "tokens_seen": 1243611136 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003147542627883651, + "loss": 1.4622, + "theoretical_loss": 3.604189105770015, + "tokens_seen": 1243676672 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003147442326980943, + "loss": 1.3395, + "theoretical_loss": 3.604171890159302, + "tokens_seen": 1243742208 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003147342026078235, + "loss": 1.3045, + "theoretical_loss": 3.6041546757096823, + "tokens_seen": 1243807744 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031472417251755266, + "loss": 1.4004, + "theoretical_loss": 3.6041374624210176, + "tokens_seen": 1243873280 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031471414242728184, + "loss": 1.3634, + "theoretical_loss": 3.6041202502931675, + "tokens_seen": 1243938816 + }, + { + "epoch": 0.38, + "learning_rate": 0.000314704112337011, + "loss": 1.3687, + "theoretical_loss": 3.6041030393259925, + "tokens_seen": 1244004352 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031469408224674025, + "loss": 1.4341, + "theoretical_loss": 3.604085829519354, + "tokens_seen": 1244069888 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003146840521564694, + "loss": 1.3941, + "theoretical_loss": 3.604068620873112, + "tokens_seen": 1244135424 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003146740220661986, + "loss": 1.438, + "theoretical_loss": 3.6040514133871273, + "tokens_seen": 1244200960 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031466399197592774, + "loss": 1.2542, + "theoretical_loss": 3.604034207061261, + "tokens_seen": 1244266496 + }, + { + "epoch": 0.38, + "learning_rate": 0.000314653961885657, + "loss": 1.3049, + "theoretical_loss": 3.6040170018953726, + "tokens_seen": 1244332032 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031464393179538616, + "loss": 1.3863, + "theoretical_loss": 3.603999797889324, + "tokens_seen": 1244397568 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031463390170511534, + "loss": 1.2641, + "theoretical_loss": 3.6039825950429756, + "tokens_seen": 1244463104 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003146238716148445, + "loss": 1.2461, + "theoretical_loss": 3.6039653933561877, + "tokens_seen": 1244528640 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003146138415245737, + "loss": 1.2577, + "theoretical_loss": 3.6039481928288217, + "tokens_seen": 1244594176 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003146038114343029, + "loss": 1.3594, + "theoretical_loss": 3.603930993460738, + "tokens_seen": 1244659712 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003145937813440321, + "loss": 1.4736, + "theoretical_loss": 3.603913795251798, + "tokens_seen": 1244725248 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031458375125376125, + "loss": 1.2375, + "theoretical_loss": 3.603896598201862, + "tokens_seen": 1244790784 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003145737211634905, + "loss": 1.3599, + "theoretical_loss": 3.6038794023107905, + "tokens_seen": 1244856320 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003145636910732197, + "loss": 1.4156, + "theoretical_loss": 3.6038622075784454, + "tokens_seen": 1244921856 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031455366098294884, + "loss": 1.3837, + "theoretical_loss": 3.603845014004687, + "tokens_seen": 1244987392 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003145436308926781, + "loss": 1.2496, + "theoretical_loss": 3.6038278215893764, + "tokens_seen": 1245052928 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003145336008024072, + "loss": 1.2962, + "theoretical_loss": 3.603810630332375, + "tokens_seen": 1245118464 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 416692, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3903861045837402, + "objective/train/theoretical_loss": 3.6037934402335425, + "objective/train/tokens_used": 1245184000, + "theoretical_loss": 3.6037934402335425, + "tokens_seen": 1245184000 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031452357071213644, + "loss": 1.4091, + "theoretical_loss": 3.6037934402335425, + "tokens_seen": 1245184000 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003145135406218656, + "loss": 1.3205, + "theoretical_loss": 3.6037762512927416, + "tokens_seen": 1245249536 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003145035105315948, + "loss": 1.4192, + "theoretical_loss": 3.603759063509832, + "tokens_seen": 1245315072 + }, + { + "epoch": 0.38, + "learning_rate": 0.000314493480441324, + "loss": 1.2574, + "theoretical_loss": 3.603741876884676, + "tokens_seen": 1245380608 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031448345035105317, + "loss": 1.2748, + "theoretical_loss": 3.6037246914171335, + "tokens_seen": 1245446144 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031447342026078235, + "loss": 1.2896, + "theoretical_loss": 3.6037075071070666, + "tokens_seen": 1245511680 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003144633901705116, + "loss": 1.2544, + "theoretical_loss": 3.6036903239543356, + "tokens_seen": 1245577216 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003144533600802407, + "loss": 1.4147, + "theoretical_loss": 3.6036731419588017, + "tokens_seen": 1245642752 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031444332998996994, + "loss": 1.3973, + "theoretical_loss": 3.603655961120327, + "tokens_seen": 1245708288 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003144332998996991, + "loss": 1.3594, + "theoretical_loss": 3.6036387814387725, + "tokens_seen": 1245773824 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003144232698094283, + "loss": 1.3411, + "theoretical_loss": 3.6036216029139982, + "tokens_seen": 1245839360 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003144132397191575, + "loss": 1.2917, + "theoretical_loss": 3.603604425545867, + "tokens_seen": 1245904896 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031440320962888667, + "loss": 1.3312, + "theoretical_loss": 3.603587249334239, + "tokens_seen": 1245970432 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031439317953861585, + "loss": 1.284, + "theoretical_loss": 3.6035700742789762, + "tokens_seen": 1246035968 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003143831494483451, + "loss": 1.3882, + "theoretical_loss": 3.60355290037994, + "tokens_seen": 1246101504 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003143731193580742, + "loss": 1.3085, + "theoretical_loss": 3.6035357276369906, + "tokens_seen": 1246167040 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031436308926780345, + "loss": 1.4101, + "theoretical_loss": 3.603518556049991, + "tokens_seen": 1246232576 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003143530591775326, + "loss": 1.3285, + "theoretical_loss": 3.603501385618802, + "tokens_seen": 1246298112 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003143430290872618, + "loss": 1.3773, + "theoretical_loss": 3.6034842163432845, + "tokens_seen": 1246363648 + }, + { + "epoch": 0.38, + "learning_rate": 0.000314332998996991, + "loss": 1.3103, + "theoretical_loss": 3.6034670482233, + "tokens_seen": 1246429184 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031432296890672017, + "loss": 1.1701, + "theoretical_loss": 3.603449881258711, + "tokens_seen": 1246494720 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031431293881644935, + "loss": 1.4823, + "theoretical_loss": 3.6034327154493777, + "tokens_seen": 1246560256 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031430290872617853, + "loss": 1.38, + "theoretical_loss": 3.603415550795163, + "tokens_seen": 1246625792 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003142928786359077, + "loss": 1.2706, + "theoretical_loss": 3.603398387295927, + "tokens_seen": 1246691328 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031428284854563695, + "loss": 1.2889, + "theoretical_loss": 3.6033812249515327, + "tokens_seen": 1246756864 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003142728184553661, + "loss": 1.3503, + "theoretical_loss": 3.6033640637618403, + "tokens_seen": 1246822400 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003142627883650953, + "loss": 1.3045, + "theoretical_loss": 3.6033469037267123, + "tokens_seen": 1246887936 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003142527582748245, + "loss": 1.3637, + "theoretical_loss": 3.60332974484601, + "tokens_seen": 1246953472 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003142427281845537, + "loss": 1.4352, + "theoretical_loss": 3.6033125871195955, + "tokens_seen": 1247019008 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031423269809428286, + "loss": 1.3706, + "theoretical_loss": 3.6032954305473304, + "tokens_seen": 1247084544 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031422266800401204, + "loss": 1.3561, + "theoretical_loss": 3.6032782751290755, + "tokens_seen": 1247150080 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003142126379137412, + "loss": 1.2361, + "theoretical_loss": 3.603261120864694, + "tokens_seen": 1247215616 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031420260782347045, + "loss": 1.3317, + "theoretical_loss": 3.6032439677540467, + "tokens_seen": 1247281152 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003141925777331996, + "loss": 1.3703, + "theoretical_loss": 3.6032268157969956, + "tokens_seen": 1247346688 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003141825476429288, + "loss": 1.2103, + "theoretical_loss": 3.6032096649934022, + "tokens_seen": 1247412224 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031417251755265794, + "loss": 1.3225, + "theoretical_loss": 3.603192515343129, + "tokens_seen": 1247477760 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003141624874623872, + "loss": 1.3483, + "theoretical_loss": 3.6031753668460373, + "tokens_seen": 1247543296 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031415245737211636, + "loss": 1.4436, + "theoretical_loss": 3.6031582195019896, + "tokens_seen": 1247608832 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031414242728184554, + "loss": 1.4887, + "theoretical_loss": 3.6031410733108475, + "tokens_seen": 1247674368 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003141323971915747, + "loss": 1.2546, + "theoretical_loss": 3.6031239282724723, + "tokens_seen": 1247739904 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003141223671013039, + "loss": 1.2818, + "theoretical_loss": 3.603106784386727, + "tokens_seen": 1247805440 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003141123370110331, + "loss": 1.2582, + "theoretical_loss": 3.6030896416534732, + "tokens_seen": 1247870976 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003141023069207623, + "loss": 1.4099, + "theoretical_loss": 3.6030725000725723, + "tokens_seen": 1247936512 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031409227683049145, + "loss": 1.3178, + "theoretical_loss": 3.6030553596438875, + "tokens_seen": 1248002048 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003140822467402207, + "loss": 1.3136, + "theoretical_loss": 3.6030382203672793, + "tokens_seen": 1248067584 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031407221664994986, + "loss": 1.2777, + "theoretical_loss": 3.603021082242611, + "tokens_seen": 1248133120 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031406218655967904, + "loss": 1.3387, + "theoretical_loss": 3.6030039452697444, + "tokens_seen": 1248198656 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003140521564694082, + "loss": 1.3091, + "theoretical_loss": 3.6029868094485415, + "tokens_seen": 1248264192 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003140421263791374, + "loss": 1.3645, + "theoretical_loss": 3.6029696747788647, + "tokens_seen": 1248329728 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003140320962888666, + "loss": 1.364, + "theoretical_loss": 3.602952541260576, + "tokens_seen": 1248395264 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 417786, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2820783853530884, + "objective/train/theoretical_loss": 3.6029354088935373, + "objective/train/tokens_used": 1248460800, + "theoretical_loss": 3.6029354088935373, + "tokens_seen": 1248460800 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003140220661985958, + "loss": 1.3271, + "theoretical_loss": 3.6029354088935373, + "tokens_seen": 1248460800 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031401203610832495, + "loss": 1.2778, + "theoretical_loss": 3.6029182776776114, + "tokens_seen": 1248526336 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003140020060180542, + "loss": 1.3422, + "theoretical_loss": 3.60290114761266, + "tokens_seen": 1248591872 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003139919759277833, + "loss": 1.3557, + "theoretical_loss": 3.6028840186985462, + "tokens_seen": 1248657408 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031398194583751255, + "loss": 1.3352, + "theoretical_loss": 3.6028668909351307, + "tokens_seen": 1248722944 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031397191574724173, + "loss": 1.431, + "theoretical_loss": 3.6028497643222774, + "tokens_seen": 1248788480 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003139618856569709, + "loss": 1.2607, + "theoretical_loss": 3.602832638859848, + "tokens_seen": 1248854016 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003139518555667001, + "loss": 1.2555, + "theoretical_loss": 3.6028155145477054, + "tokens_seen": 1248919552 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003139418254764293, + "loss": 1.4015, + "theoretical_loss": 3.602798391385711, + "tokens_seen": 1248985088 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031393179538615845, + "loss": 1.2862, + "theoretical_loss": 3.602781269373727, + "tokens_seen": 1249050624 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003139217652958877, + "loss": 1.3565, + "theoretical_loss": 3.6027641485116177, + "tokens_seen": 1249116160 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003139117352056168, + "loss": 1.3721, + "theoretical_loss": 3.602747028799244, + "tokens_seen": 1249181696 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031390170511534605, + "loss": 1.3716, + "theoretical_loss": 3.6027299102364685, + "tokens_seen": 1249247232 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031389167502507523, + "loss": 1.3266, + "theoretical_loss": 3.6027127928231537, + "tokens_seen": 1249312768 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003138816449348044, + "loss": 1.2679, + "theoretical_loss": 3.602695676559163, + "tokens_seen": 1249378304 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003138716148445336, + "loss": 1.3853, + "theoretical_loss": 3.6026785614443577, + "tokens_seen": 1249443840 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003138615847542628, + "loss": 1.3728, + "theoretical_loss": 3.6026614474786016, + "tokens_seen": 1249509376 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031385155466399196, + "loss": 1.3786, + "theoretical_loss": 3.6026443346617567, + "tokens_seen": 1249574912 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003138415245737212, + "loss": 1.3229, + "theoretical_loss": 3.602627222993685, + "tokens_seen": 1249640448 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003138314944834503, + "loss": 1.3896, + "theoretical_loss": 3.6026101124742502, + "tokens_seen": 1249705984 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031382146439317955, + "loss": 1.4682, + "theoretical_loss": 3.6025930031033147, + "tokens_seen": 1249771520 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031381143430290873, + "loss": 1.3116, + "theoretical_loss": 3.602575894880741, + "tokens_seen": 1249837056 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003138014042126379, + "loss": 1.3512, + "theoretical_loss": 3.602558787806391, + "tokens_seen": 1249902592 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031379137412236715, + "loss": 1.3656, + "theoretical_loss": 3.602541681880129, + "tokens_seen": 1249968128 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003137813440320963, + "loss": 1.4348, + "theoretical_loss": 3.602524577101817, + "tokens_seen": 1250033664 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003137713139418255, + "loss": 1.3745, + "theoretical_loss": 3.602507473471318, + "tokens_seen": 1250099200 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003137612838515547, + "loss": 1.3725, + "theoretical_loss": 3.602490370988494, + "tokens_seen": 1250164736 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003137512537612839, + "loss": 1.4435, + "theoretical_loss": 3.602473269653209, + "tokens_seen": 1250230272 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031374122367101306, + "loss": 1.3839, + "theoretical_loss": 3.602456169465325, + "tokens_seen": 1250295808 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031373119358074224, + "loss": 1.3362, + "theoretical_loss": 3.6024390704247056, + "tokens_seen": 1250361344 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003137211634904714, + "loss": 1.3669, + "theoretical_loss": 3.6024219725312125, + "tokens_seen": 1250426880 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031371113340020065, + "loss": 1.3614, + "theoretical_loss": 3.6024048757847105, + "tokens_seen": 1250492416 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003137011033099298, + "loss": 1.3553, + "theoretical_loss": 3.6023877801850603, + "tokens_seen": 1250557952 + }, + { + "epoch": 0.38, + "learning_rate": 0.000313691073219659, + "loss": 1.3499, + "theoretical_loss": 3.6023706857321267, + "tokens_seen": 1250623488 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031368104312938814, + "loss": 1.3022, + "theoretical_loss": 3.6023535924257724, + "tokens_seen": 1250689024 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003136710130391174, + "loss": 1.3295, + "theoretical_loss": 3.602336500265859, + "tokens_seen": 1250754560 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031366098294884656, + "loss": 1.3375, + "theoretical_loss": 3.6023194092522517, + "tokens_seen": 1250820096 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031365095285857574, + "loss": 1.2939, + "theoretical_loss": 3.602302319384812, + "tokens_seen": 1250885632 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003136409227683049, + "loss": 1.3136, + "theoretical_loss": 3.6022852306634032, + "tokens_seen": 1250951168 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003136308926780341, + "loss": 1.3807, + "theoretical_loss": 3.602268143087889, + "tokens_seen": 1251016704 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003136208625877633, + "loss": 1.3678, + "theoretical_loss": 3.6022510566581323, + "tokens_seen": 1251082240 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003136108324974925, + "loss": 1.356, + "theoretical_loss": 3.6022339713739964, + "tokens_seen": 1251147776 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031360080240722165, + "loss": 1.2626, + "theoretical_loss": 3.6022168872353437, + "tokens_seen": 1251213312 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003135907723169509, + "loss": 1.3138, + "theoretical_loss": 3.602199804242038, + "tokens_seen": 1251278848 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031358074222668006, + "loss": 1.4071, + "theoretical_loss": 3.602182722393943, + "tokens_seen": 1251344384 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031357071213640924, + "loss": 1.3703, + "theoretical_loss": 3.602165641690921, + "tokens_seen": 1251409920 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003135606820461384, + "loss": 1.3094, + "theoretical_loss": 3.602148562132836, + "tokens_seen": 1251475456 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003135506519558676, + "loss": 1.3365, + "theoretical_loss": 3.6021314837195515, + "tokens_seen": 1251540992 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003135406218655968, + "loss": 1.3278, + "theoretical_loss": 3.6021144064509296, + "tokens_seen": 1251606528 + }, + { + "epoch": 0.38, + "learning_rate": 0.000313530591775326, + "loss": 1.3152, + "theoretical_loss": 3.6020973303268344, + "tokens_seen": 1251672064 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 418680, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3702607154846191, + "objective/train/theoretical_loss": 3.6020802553471296, + "objective/train/tokens_used": 1251737600, + "theoretical_loss": 3.6020802553471296, + "tokens_seen": 1251737600 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031352056168505515, + "loss": 1.3154, + "theoretical_loss": 3.6020802553471296, + "tokens_seen": 1251737600 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003135105315947844, + "loss": 1.2486, + "theoretical_loss": 3.6020631815116784, + "tokens_seen": 1251803136 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003135005015045135, + "loss": 1.2935, + "theoretical_loss": 3.602046108820344, + "tokens_seen": 1251868672 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031349047141424275, + "loss": 1.3696, + "theoretical_loss": 3.60202903727299, + "tokens_seen": 1251934208 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031348044132397193, + "loss": 1.4716, + "theoretical_loss": 3.6020119668694797, + "tokens_seen": 1251999744 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003134704112337011, + "loss": 1.3968, + "theoretical_loss": 3.6019948976096767, + "tokens_seen": 1252065280 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003134603811434303, + "loss": 1.2763, + "theoretical_loss": 3.6019778294934444, + "tokens_seen": 1252130816 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003134503510531595, + "loss": 1.2805, + "theoretical_loss": 3.6019607625206467, + "tokens_seen": 1252196352 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031344032096288865, + "loss": 1.3533, + "theoretical_loss": 3.601943696691147, + "tokens_seen": 1252261888 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003134302908726179, + "loss": 1.2729, + "theoretical_loss": 3.601926632004809, + "tokens_seen": 1252327424 + }, + { + "epoch": 0.38, + "learning_rate": 0.000313420260782347, + "loss": 1.2818, + "theoretical_loss": 3.6019095684614957, + "tokens_seen": 1252392960 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031341023069207625, + "loss": 1.2851, + "theoretical_loss": 3.601892506061071, + "tokens_seen": 1252458496 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031340020060180543, + "loss": 1.3477, + "theoretical_loss": 3.601875444803399, + "tokens_seen": 1252524032 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003133901705115346, + "loss": 1.2904, + "theoretical_loss": 3.6018583846883434, + "tokens_seen": 1252589568 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003133801404212638, + "loss": 1.3914, + "theoretical_loss": 3.601841325715767, + "tokens_seen": 1252655104 + }, + { + "epoch": 0.38, + "learning_rate": 0.000313370110330993, + "loss": 1.3433, + "theoretical_loss": 3.601824267885535, + "tokens_seen": 1252720640 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031336008024072216, + "loss": 1.1838, + "theoretical_loss": 3.6018072111975092, + "tokens_seen": 1252786176 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003133500501504514, + "loss": 1.3491, + "theoretical_loss": 3.6017901556515546, + "tokens_seen": 1252851712 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003133400200601805, + "loss": 1.3458, + "theoretical_loss": 3.6017731012475354, + "tokens_seen": 1252917248 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031332998996990975, + "loss": 1.376, + "theoretical_loss": 3.6017560479853143, + "tokens_seen": 1252982784 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003133199598796389, + "loss": 1.444, + "theoretical_loss": 3.601738995864756, + "tokens_seen": 1253048320 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003133099297893681, + "loss": 1.1803, + "theoretical_loss": 3.601721944885724, + "tokens_seen": 1253113856 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003132998996990973, + "loss": 1.2753, + "theoretical_loss": 3.6017048950480817, + "tokens_seen": 1253179392 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003132898696088265, + "loss": 1.1907, + "theoretical_loss": 3.601687846351694, + "tokens_seen": 1253244928 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031327983951855566, + "loss": 1.3168, + "theoretical_loss": 3.6016707987964245, + "tokens_seen": 1253310464 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003132698094282849, + "loss": 1.3167, + "theoretical_loss": 3.601653752382137, + "tokens_seen": 1253376000 + }, + { + "epoch": 0.38, + "learning_rate": 0.000313259779338014, + "loss": 1.2494, + "theoretical_loss": 3.601636707108695, + "tokens_seen": 1253441536 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031324974924774326, + "loss": 1.2813, + "theoretical_loss": 3.601619662975964, + "tokens_seen": 1253507072 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003132397191574724, + "loss": 1.2853, + "theoretical_loss": 3.6016026199838063, + "tokens_seen": 1253572608 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003132296890672016, + "loss": 1.3292, + "theoretical_loss": 3.601585578132087, + "tokens_seen": 1253638144 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003132196589769308, + "loss": 1.2435, + "theoretical_loss": 3.6015685374206696, + "tokens_seen": 1253703680 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031320962888666, + "loss": 1.3308, + "theoretical_loss": 3.601551497849419, + "tokens_seen": 1253769216 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031319959879638916, + "loss": 1.399, + "theoretical_loss": 3.6015344594181986, + "tokens_seen": 1253834752 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031318956870611834, + "loss": 1.3205, + "theoretical_loss": 3.6015174221268724, + "tokens_seen": 1253900288 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003131795386158475, + "loss": 1.3139, + "theoretical_loss": 3.601500385975305, + "tokens_seen": 1253965824 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031316950852557676, + "loss": 1.3588, + "theoretical_loss": 3.6014833509633606, + "tokens_seen": 1254031360 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003131594784353059, + "loss": 1.271, + "theoretical_loss": 3.6014663170909036, + "tokens_seen": 1254096896 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003131494483450351, + "loss": 1.2214, + "theoretical_loss": 3.6014492843577974, + "tokens_seen": 1254162432 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031313941825476425, + "loss": 1.2529, + "theoretical_loss": 3.6014322527639075, + "tokens_seen": 1254227968 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003131293881644935, + "loss": 1.2406, + "theoretical_loss": 3.6014152223090967, + "tokens_seen": 1254293504 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031311935807422267, + "loss": 1.2564, + "theoretical_loss": 3.601398192993231, + "tokens_seen": 1254359040 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031310932798395185, + "loss": 1.3771, + "theoretical_loss": 3.601381164816173, + "tokens_seen": 1254424576 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031309929789368103, + "loss": 1.3677, + "theoretical_loss": 3.6013641377777885, + "tokens_seen": 1254490112 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031308926780341026, + "loss": 1.3046, + "theoretical_loss": 3.601347111877941, + "tokens_seen": 1254555648 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003130792377131394, + "loss": 1.2676, + "theoretical_loss": 3.6013300871164953, + "tokens_seen": 1254621184 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003130692076228686, + "loss": 1.3878, + "theoretical_loss": 3.6013130634933157, + "tokens_seen": 1254686720 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003130591775325978, + "loss": 1.281, + "theoretical_loss": 3.6012960410082666, + "tokens_seen": 1254752256 + }, + { + "epoch": 0.38, + "learning_rate": 0.000313049147442327, + "loss": 1.2311, + "theoretical_loss": 3.601279019661213, + "tokens_seen": 1254817792 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003130391173520562, + "loss": 1.2504, + "theoretical_loss": 3.601261999452018, + "tokens_seen": 1254883328 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031302908726178535, + "loss": 1.4271, + "theoretical_loss": 3.6012449803805477, + "tokens_seen": 1254948864 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 419762, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4049756526947021, + "objective/train/theoretical_loss": 3.601227962446665, + "objective/train/tokens_used": 1255014400, + "theoretical_loss": 3.601227962446665, + "tokens_seen": 1255014400 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003130190571715146, + "loss": 1.3445, + "theoretical_loss": 3.601227962446665, + "tokens_seen": 1255014400 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003130090270812437, + "loss": 1.3645, + "theoretical_loss": 3.6012109456502364, + "tokens_seen": 1255079936 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031299899699097295, + "loss": 1.327, + "theoretical_loss": 3.6011939299911253, + "tokens_seen": 1255145472 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031298896690070213, + "loss": 1.2692, + "theoretical_loss": 3.601176915469196, + "tokens_seen": 1255211008 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003129789368104313, + "loss": 1.3475, + "theoretical_loss": 3.601159902084315, + "tokens_seen": 1255276544 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003129689067201605, + "loss": 1.3705, + "theoretical_loss": 3.6011428898363445, + "tokens_seen": 1255342080 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003129588766298897, + "loss": 1.3871, + "theoretical_loss": 3.60112587872515, + "tokens_seen": 1255407616 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031294884653961885, + "loss": 1.3302, + "theoretical_loss": 3.6011088687505977, + "tokens_seen": 1255473152 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003129388164493481, + "loss": 1.2565, + "theoretical_loss": 3.6010918599125503, + "tokens_seen": 1255538688 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003129287863590772, + "loss": 1.4258, + "theoretical_loss": 3.601074852210873, + "tokens_seen": 1255604224 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031291875626880645, + "loss": 1.3413, + "theoretical_loss": 3.601057845645432, + "tokens_seen": 1255669760 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031290872617853563, + "loss": 1.2562, + "theoretical_loss": 3.6010408402160907, + "tokens_seen": 1255735296 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003128986960882648, + "loss": 1.2326, + "theoretical_loss": 3.601023835922714, + "tokens_seen": 1255800832 + }, + { + "epoch": 0.38, + "learning_rate": 0.000312888665997994, + "loss": 1.3921, + "theoretical_loss": 3.601006832765167, + "tokens_seen": 1255866368 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003128786359077232, + "loss": 1.2589, + "theoretical_loss": 3.6009898307433144, + "tokens_seen": 1255931904 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031286860581745236, + "loss": 1.33, + "theoretical_loss": 3.6009728298570214, + "tokens_seen": 1255997440 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003128585757271816, + "loss": 1.2659, + "theoretical_loss": 3.6009558301061526, + "tokens_seen": 1256062976 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003128485456369107, + "loss": 1.3016, + "theoretical_loss": 3.6009388314905735, + "tokens_seen": 1256128512 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031283851554663995, + "loss": 1.316, + "theoretical_loss": 3.6009218340101485, + "tokens_seen": 1256194048 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003128284854563691, + "loss": 1.2575, + "theoretical_loss": 3.600904837664743, + "tokens_seen": 1256259584 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003128184553660983, + "loss": 1.2589, + "theoretical_loss": 3.6008878424542208, + "tokens_seen": 1256325120 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003128084252758275, + "loss": 1.3324, + "theoretical_loss": 3.600870848378449, + "tokens_seen": 1256390656 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003127983951855567, + "loss": 1.2098, + "theoretical_loss": 3.6008538554372906, + "tokens_seen": 1256456192 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031278836509528586, + "loss": 1.3379, + "theoretical_loss": 3.6008368636306116, + "tokens_seen": 1256521728 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003127783350050151, + "loss": 1.3413, + "theoretical_loss": 3.6008198729582777, + "tokens_seen": 1256587264 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003127683049147442, + "loss": 1.3233, + "theoretical_loss": 3.600802883420153, + "tokens_seen": 1256652800 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031275827482447346, + "loss": 1.2905, + "theoretical_loss": 3.600785895016103, + "tokens_seen": 1256718336 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003127482447342026, + "loss": 1.3479, + "theoretical_loss": 3.600768907745993, + "tokens_seen": 1256783872 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003127382146439318, + "loss": 1.3996, + "theoretical_loss": 3.6007519216096884, + "tokens_seen": 1256849408 + }, + { + "epoch": 0.38, + "learning_rate": 0.000312728184553661, + "loss": 1.3536, + "theoretical_loss": 3.6007349366070533, + "tokens_seen": 1256914944 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003127181544633902, + "loss": 1.2741, + "theoretical_loss": 3.6007179527379547, + "tokens_seen": 1256980480 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031270812437311936, + "loss": 1.3483, + "theoretical_loss": 3.600700970002256, + "tokens_seen": 1257046016 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031269809428284854, + "loss": 1.2621, + "theoretical_loss": 3.6006839883998243, + "tokens_seen": 1257111552 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003126880641925777, + "loss": 1.4562, + "theoretical_loss": 3.6006670079305234, + "tokens_seen": 1257177088 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031267803410230696, + "loss": 1.2882, + "theoretical_loss": 3.6006500285942193, + "tokens_seen": 1257242624 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003126680040120361, + "loss": 1.3106, + "theoretical_loss": 3.600633050390777, + "tokens_seen": 1257308160 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003126579739217653, + "loss": 1.4163, + "theoretical_loss": 3.6006160733200625, + "tokens_seen": 1257373696 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031264794383149445, + "loss": 1.3137, + "theoretical_loss": 3.6005990973819406, + "tokens_seen": 1257439232 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003126379137412237, + "loss": 1.4323, + "theoretical_loss": 3.6005821225762764, + "tokens_seen": 1257504768 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031262788365095287, + "loss": 1.2216, + "theoretical_loss": 3.6005651489029367, + "tokens_seen": 1257570304 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031261785356068205, + "loss": 1.2663, + "theoretical_loss": 3.600548176361786, + "tokens_seen": 1257635840 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031260782347041123, + "loss": 1.3457, + "theoretical_loss": 3.6005312049526896, + "tokens_seen": 1257701376 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031259779338014046, + "loss": 1.406, + "theoretical_loss": 3.6005142346755132, + "tokens_seen": 1257766912 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003125877632898696, + "loss": 1.3202, + "theoretical_loss": 3.600497265530123, + "tokens_seen": 1257832448 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003125777331995988, + "loss": 1.2401, + "theoretical_loss": 3.600480297516384, + "tokens_seen": 1257897984 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031256770310932795, + "loss": 1.3295, + "theoretical_loss": 3.6004633306341614, + "tokens_seen": 1257963520 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003125576730190572, + "loss": 1.3571, + "theoretical_loss": 3.6004463648833216, + "tokens_seen": 1258029056 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031254764292878637, + "loss": 1.2532, + "theoretical_loss": 3.6004294002637294, + "tokens_seen": 1258094592 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031253761283851555, + "loss": 1.2539, + "theoretical_loss": 3.600412436775251, + "tokens_seen": 1258160128 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031252758274824473, + "loss": 1.2968, + "theoretical_loss": 3.600395474417752, + "tokens_seen": 1258225664 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 420788, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4176241159439087, + "objective/train/theoretical_loss": 3.600378513191098, + "objective/train/tokens_used": 1258291200, + "theoretical_loss": 3.600378513191098, + "tokens_seen": 1258291200 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003125175526579739, + "loss": 1.3408, + "theoretical_loss": 3.600378513191098, + "tokens_seen": 1258291200 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003125075225677031, + "loss": 1.2425, + "theoretical_loss": 3.600361553095155, + "tokens_seen": 1258356736 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031249749247743233, + "loss": 1.419, + "theoretical_loss": 3.6003445941297882, + "tokens_seen": 1258422272 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031248746238716146, + "loss": 1.3387, + "theoretical_loss": 3.600327636294864, + "tokens_seen": 1258487808 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003124774322968907, + "loss": 1.2525, + "theoretical_loss": 3.6003106795902475, + "tokens_seen": 1258553344 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003124674022066198, + "loss": 1.3133, + "theoretical_loss": 3.600293724015805, + "tokens_seen": 1258618880 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031245737211634905, + "loss": 1.3924, + "theoretical_loss": 3.600276769571402, + "tokens_seen": 1258684416 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031244734202607823, + "loss": 1.3606, + "theoretical_loss": 3.6002598162569046, + "tokens_seen": 1258749952 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003124373119358074, + "loss": 1.3205, + "theoretical_loss": 3.600242864072179, + "tokens_seen": 1258815488 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003124272818455366, + "loss": 1.2437, + "theoretical_loss": 3.6002259130170904, + "tokens_seen": 1258881024 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031241725175526583, + "loss": 1.3464, + "theoretical_loss": 3.6002089630915046, + "tokens_seen": 1258946560 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031240722166499496, + "loss": 1.2263, + "theoretical_loss": 3.6001920142952883, + "tokens_seen": 1259012096 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003123971915747242, + "loss": 1.3002, + "theoretical_loss": 3.6001750666283074, + "tokens_seen": 1259077632 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003123871614844533, + "loss": 1.367, + "theoretical_loss": 3.600158120090428, + "tokens_seen": 1259143168 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031237713139418256, + "loss": 1.3495, + "theoretical_loss": 3.600141174681515, + "tokens_seen": 1259208704 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031236710130391174, + "loss": 1.3262, + "theoretical_loss": 3.6001242304014354, + "tokens_seen": 1259274240 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003123570712136409, + "loss": 1.3029, + "theoretical_loss": 3.6001072872500552, + "tokens_seen": 1259339776 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003123470411233701, + "loss": 1.3452, + "theoretical_loss": 3.6000903452272404, + "tokens_seen": 1259405312 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003123370110330993, + "loss": 1.3441, + "theoretical_loss": 3.600073404332856, + "tokens_seen": 1259470848 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031232698094282846, + "loss": 1.3935, + "theoretical_loss": 3.6000564645667703, + "tokens_seen": 1259536384 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003123169508525577, + "loss": 1.2161, + "theoretical_loss": 3.6000395259288482, + "tokens_seen": 1259601920 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003123069207622869, + "loss": 1.2328, + "theoretical_loss": 3.6000225884189563, + "tokens_seen": 1259667456 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031229689067201606, + "loss": 1.2894, + "theoretical_loss": 3.60000565203696, + "tokens_seen": 1259732992 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003122868605817453, + "loss": 1.2071, + "theoretical_loss": 3.599988716782726, + "tokens_seen": 1259798528 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003122768304914744, + "loss": 1.3803, + "theoretical_loss": 3.5999717826561204, + "tokens_seen": 1259864064 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031226680040120366, + "loss": 1.2786, + "theoretical_loss": 3.59995484965701, + "tokens_seen": 1259929600 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003122567703109328, + "loss": 1.323, + "theoretical_loss": 3.5999379177852604, + "tokens_seen": 1259995136 + }, + { + "epoch": 0.38, + "learning_rate": 0.000312246740220662, + "loss": 1.2907, + "theoretical_loss": 3.5999209870407385, + "tokens_seen": 1260060672 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003122367101303912, + "loss": 1.2225, + "theoretical_loss": 3.59990405742331, + "tokens_seen": 1260126208 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003122266800401204, + "loss": 1.4117, + "theoretical_loss": 3.599887128932842, + "tokens_seen": 1260191744 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031221664994984956, + "loss": 1.2995, + "theoretical_loss": 3.5998702015692006, + "tokens_seen": 1260257280 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031220661985957874, + "loss": 1.3441, + "theoretical_loss": 3.5998532753322516, + "tokens_seen": 1260322816 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003121965897693079, + "loss": 1.349, + "theoretical_loss": 3.599836350221862, + "tokens_seen": 1260388352 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031218655967903716, + "loss": 1.4063, + "theoretical_loss": 3.5998194262378984, + "tokens_seen": 1260453888 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003121765295887663, + "loss": 1.3495, + "theoretical_loss": 3.599802503380227, + "tokens_seen": 1260519424 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003121664994984955, + "loss": 1.2905, + "theoretical_loss": 3.5997855816487143, + "tokens_seen": 1260584960 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031215646940822465, + "loss": 1.3355, + "theoretical_loss": 3.599768661043227, + "tokens_seen": 1260650496 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003121464393179539, + "loss": 1.2853, + "theoretical_loss": 3.5997517415636313, + "tokens_seen": 1260716032 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031213640922768307, + "loss": 1.3578, + "theoretical_loss": 3.599734823209794, + "tokens_seen": 1260781568 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031212637913741225, + "loss": 1.2003, + "theoretical_loss": 3.5997179059815814, + "tokens_seen": 1260847104 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031211634904714143, + "loss": 1.3072, + "theoretical_loss": 3.599700989878861, + "tokens_seen": 1260912640 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031210631895687066, + "loss": 1.3208, + "theoretical_loss": 3.599684074901498, + "tokens_seen": 1260978176 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003120962888665998, + "loss": 1.2388, + "theoretical_loss": 3.5996671610493602, + "tokens_seen": 1261043712 + }, + { + "epoch": 0.38, + "learning_rate": 0.000312086258776329, + "loss": 1.2759, + "theoretical_loss": 3.599650248322314, + "tokens_seen": 1261109248 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031207622868605815, + "loss": 1.2952, + "theoretical_loss": 3.599633336720226, + "tokens_seen": 1261174784 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003120661985957874, + "loss": 1.2488, + "theoretical_loss": 3.599616426242963, + "tokens_seen": 1261240320 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031205616850551657, + "loss": 1.3651, + "theoretical_loss": 3.5995995168903914, + "tokens_seen": 1261305856 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031204613841524575, + "loss": 1.2584, + "theoretical_loss": 3.599582608662378, + "tokens_seen": 1261371392 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031203610832497493, + "loss": 1.2517, + "theoretical_loss": 3.59956570155879, + "tokens_seen": 1261436928 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003120260782347041, + "loss": 1.3287, + "theoretical_loss": 3.5995487955794943, + "tokens_seen": 1261502464 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 421951, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2547379732131958, + "objective/train/theoretical_loss": 3.599531890724357, + "objective/train/tokens_used": 1261568000, + "theoretical_loss": 3.599531890724357, + "tokens_seen": 1261568000 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003120160481444333, + "loss": 1.2658, + "theoretical_loss": 3.599531890724357, + "tokens_seen": 1261568000 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031200601805416253, + "loss": 1.3366, + "theoretical_loss": 3.599514986993246, + "tokens_seen": 1261633536 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031199598796389166, + "loss": 1.2251, + "theoretical_loss": 3.5994980843860276, + "tokens_seen": 1261699072 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003119859578736209, + "loss": 1.2879, + "theoretical_loss": 3.5994811829025686, + "tokens_seen": 1261764608 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031197592778335, + "loss": 1.3885, + "theoretical_loss": 3.599464282542735, + "tokens_seen": 1261830144 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031196589769307925, + "loss": 1.3439, + "theoretical_loss": 3.599447383306396, + "tokens_seen": 1261895680 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031195586760280843, + "loss": 1.2157, + "theoretical_loss": 3.5994304851934174, + "tokens_seen": 1261961216 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003119458375125376, + "loss": 1.2684, + "theoretical_loss": 3.5994135882036655, + "tokens_seen": 1262026752 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003119358074222668, + "loss": 1.2771, + "theoretical_loss": 3.5993966923370087, + "tokens_seen": 1262092288 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031192577733199603, + "loss": 1.262, + "theoretical_loss": 3.599379797593312, + "tokens_seen": 1262157824 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031191574724172516, + "loss": 1.271, + "theoretical_loss": 3.599362903972445, + "tokens_seen": 1262223360 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003119057171514544, + "loss": 1.3104, + "theoretical_loss": 3.5993460114742732, + "tokens_seen": 1262288896 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003118956870611835, + "loss": 1.3093, + "theoretical_loss": 3.599329120098664, + "tokens_seen": 1262354432 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031188565697091276, + "loss": 1.2008, + "theoretical_loss": 3.5993122298454843, + "tokens_seen": 1262419968 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031187562688064194, + "loss": 1.3294, + "theoretical_loss": 3.599295340714602, + "tokens_seen": 1262485504 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003118655967903711, + "loss": 1.3462, + "theoretical_loss": 3.5992784527058834, + "tokens_seen": 1262551040 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003118555667001003, + "loss": 1.3395, + "theoretical_loss": 3.5992615658191967, + "tokens_seen": 1262616576 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003118455366098295, + "loss": 1.3089, + "theoretical_loss": 3.599244680054408, + "tokens_seen": 1262682112 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031183550651955866, + "loss": 1.281, + "theoretical_loss": 3.599227795411385, + "tokens_seen": 1262747648 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003118254764292879, + "loss": 1.293, + "theoretical_loss": 3.5992109118899958, + "tokens_seen": 1262813184 + }, + { + "epoch": 0.38, + "learning_rate": 0.000311815446339017, + "loss": 1.3374, + "theoretical_loss": 3.599194029490106, + "tokens_seen": 1262878720 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031180541624874626, + "loss": 1.206, + "theoretical_loss": 3.5991771482115844, + "tokens_seen": 1262944256 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003117953861584754, + "loss": 1.226, + "theoretical_loss": 3.5991602680542973, + "tokens_seen": 1263009792 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003117853560682046, + "loss": 1.3911, + "theoretical_loss": 3.5991433890181126, + "tokens_seen": 1263075328 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003117753259779338, + "loss": 1.2047, + "theoretical_loss": 3.599126511102898, + "tokens_seen": 1263140864 + }, + { + "epoch": 0.38, + "learning_rate": 0.000311765295887663, + "loss": 1.3451, + "theoretical_loss": 3.59910963430852, + "tokens_seen": 1263206400 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031175526579739217, + "loss": 1.3049, + "theoretical_loss": 3.599092758634847, + "tokens_seen": 1263271936 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003117452357071214, + "loss": 1.3032, + "theoretical_loss": 3.5990758840817456, + "tokens_seen": 1263337472 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031173520561685053, + "loss": 1.2498, + "theoretical_loss": 3.599059010649084, + "tokens_seen": 1263403008 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031172517552657976, + "loss": 1.3516, + "theoretical_loss": 3.5990421383367286, + "tokens_seen": 1263468544 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003117151454363089, + "loss": 1.1016, + "theoretical_loss": 3.5990252671445484, + "tokens_seen": 1263534080 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003117051153460381, + "loss": 1.3401, + "theoretical_loss": 3.5990083970724096, + "tokens_seen": 1263599616 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003116950852557673, + "loss": 1.3615, + "theoretical_loss": 3.5989915281201807, + "tokens_seen": 1263665152 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003116850551654965, + "loss": 1.2732, + "theoretical_loss": 3.5989746602877286, + "tokens_seen": 1263730688 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031167502507522567, + "loss": 1.3046, + "theoretical_loss": 3.5989577935749217, + "tokens_seen": 1263796224 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031166499498495485, + "loss": 1.2286, + "theoretical_loss": 3.598940927981627, + "tokens_seen": 1263861760 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031165496489468403, + "loss": 1.1933, + "theoretical_loss": 3.598924063507712, + "tokens_seen": 1263927296 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031164493480441327, + "loss": 1.134, + "theoretical_loss": 3.598907200153045, + "tokens_seen": 1263992832 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003116349047141424, + "loss": 1.3188, + "theoretical_loss": 3.598890337917493, + "tokens_seen": 1264058368 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031162487462387163, + "loss": 1.2955, + "theoretical_loss": 3.5988734768009243, + "tokens_seen": 1264123904 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003116148445336008, + "loss": 1.3477, + "theoretical_loss": 3.5988566168032063, + "tokens_seen": 1264189440 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031160481444333, + "loss": 1.2395, + "theoretical_loss": 3.598839757924207, + "tokens_seen": 1264254976 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031159478435305917, + "loss": 1.2964, + "theoretical_loss": 3.5988229001637935, + "tokens_seen": 1264320512 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031158475426278835, + "loss": 1.2642, + "theoretical_loss": 3.5988060435218348, + "tokens_seen": 1264386048 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031157472417251753, + "loss": 1.306, + "theoretical_loss": 3.598789187998198, + "tokens_seen": 1264451584 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031156469408224677, + "loss": 1.2521, + "theoretical_loss": 3.5987723335927506, + "tokens_seen": 1264517120 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031155466399197595, + "loss": 1.2405, + "theoretical_loss": 3.5987554803053614, + "tokens_seen": 1264582656 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031154463390170513, + "loss": 1.3019, + "theoretical_loss": 3.5987386281358975, + "tokens_seen": 1264648192 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003115346038114343, + "loss": 1.2614, + "theoretical_loss": 3.5987217770842275, + "tokens_seen": 1264713728 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003115245737211635, + "loss": 1.2086, + "theoretical_loss": 3.5987049271502185, + "tokens_seen": 1264779264 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 422920, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3524900674819946, + "objective/train/theoretical_loss": 3.5986880783337387, + "objective/train/tokens_used": 1264844800, + "theoretical_loss": 3.5986880783337387, + "tokens_seen": 1264844800 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031151454363089273, + "loss": 1.4371, + "theoretical_loss": 3.5986880783337387, + "tokens_seen": 1264844800 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031150451354062186, + "loss": 1.1653, + "theoretical_loss": 3.598671230634657, + "tokens_seen": 1264910336 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003114944834503511, + "loss": 1.1969, + "theoretical_loss": 3.59865438405284, + "tokens_seen": 1264975872 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003114844533600802, + "loss": 1.3029, + "theoretical_loss": 3.5986375385881573, + "tokens_seen": 1265041408 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031147442326980945, + "loss": 1.2504, + "theoretical_loss": 3.5986206942404757, + "tokens_seen": 1265106944 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031146439317953863, + "loss": 1.2073, + "theoretical_loss": 3.5986038510096634, + "tokens_seen": 1265172480 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003114543630892678, + "loss": 1.2034, + "theoretical_loss": 3.5985870088955894, + "tokens_seen": 1265238016 + }, + { + "epoch": 0.38, + "learning_rate": 0.000311444332998997, + "loss": 1.41, + "theoretical_loss": 3.598570167898121, + "tokens_seen": 1265303552 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031143430290872623, + "loss": 1.3137, + "theoretical_loss": 3.5985533280171262, + "tokens_seen": 1265369088 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031142427281845536, + "loss": 1.2763, + "theoretical_loss": 3.5985364892524734, + "tokens_seen": 1265434624 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003114142427281846, + "loss": 1.2741, + "theoretical_loss": 3.598519651604031, + "tokens_seen": 1265500160 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003114042126379137, + "loss": 1.2562, + "theoretical_loss": 3.598502815071667, + "tokens_seen": 1265565696 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031139418254764296, + "loss": 1.2616, + "theoretical_loss": 3.5984859796552504, + "tokens_seen": 1265631232 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031138415245737214, + "loss": 1.1968, + "theoretical_loss": 3.598469145354648, + "tokens_seen": 1265696768 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003113741223671013, + "loss": 1.4402, + "theoretical_loss": 3.598452312169729, + "tokens_seen": 1265762304 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003113640922768305, + "loss": 1.2539, + "theoretical_loss": 3.598435480100362, + "tokens_seen": 1265827840 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003113540621865597, + "loss": 1.263, + "theoretical_loss": 3.5984186491464145, + "tokens_seen": 1265893376 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031134403209628886, + "loss": 1.4745, + "theoretical_loss": 3.598401819307756, + "tokens_seen": 1265958912 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003113340020060181, + "loss": 1.3108, + "theoretical_loss": 3.598384990584253, + "tokens_seen": 1266024448 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003113239719157472, + "loss": 1.2259, + "theoretical_loss": 3.5983681629757753, + "tokens_seen": 1266089984 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031131394182547646, + "loss": 1.3802, + "theoretical_loss": 3.598351336482191, + "tokens_seen": 1266155520 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003113039117352056, + "loss": 1.2723, + "theoretical_loss": 3.598334511103369, + "tokens_seen": 1266221056 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003112938816449348, + "loss": 1.2131, + "theoretical_loss": 3.5983176868391764, + "tokens_seen": 1266286592 + }, + { + "epoch": 0.38, + "learning_rate": 0.000311283851554664, + "loss": 1.1616, + "theoretical_loss": 3.5983008636894835, + "tokens_seen": 1266352128 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003112738214643932, + "loss": 1.3311, + "theoretical_loss": 3.598284041654157, + "tokens_seen": 1266417664 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031126379137412237, + "loss": 1.2472, + "theoretical_loss": 3.598267220733067, + "tokens_seen": 1266483200 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003112537612838516, + "loss": 1.2997, + "theoretical_loss": 3.598250400926081, + "tokens_seen": 1266548736 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031124373119358073, + "loss": 1.3077, + "theoretical_loss": 3.5982335822330676, + "tokens_seen": 1266614272 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031123370110330996, + "loss": 1.2346, + "theoretical_loss": 3.5982167646538956, + "tokens_seen": 1266679808 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003112236710130391, + "loss": 1.315, + "theoretical_loss": 3.5981999481884346, + "tokens_seen": 1266745344 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003112136409227683, + "loss": 1.306, + "theoretical_loss": 3.5981831328365512, + "tokens_seen": 1266810880 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003112036108324975, + "loss": 1.2541, + "theoretical_loss": 3.598166318598116, + "tokens_seen": 1266876416 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003111935807422267, + "loss": 1.2215, + "theoretical_loss": 3.5981495054729966, + "tokens_seen": 1266941952 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031118355065195587, + "loss": 1.2315, + "theoretical_loss": 3.5981326934610616, + "tokens_seen": 1267007488 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031117352056168505, + "loss": 1.2165, + "theoretical_loss": 3.5981158825621806, + "tokens_seen": 1267073024 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031116349047141423, + "loss": 1.2643, + "theoretical_loss": 3.598099072776222, + "tokens_seen": 1267138560 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031115346038114347, + "loss": 1.2515, + "theoretical_loss": 3.598082264103054, + "tokens_seen": 1267204096 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003111434302908726, + "loss": 1.2269, + "theoretical_loss": 3.5980654565425456, + "tokens_seen": 1267269632 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031113340020060183, + "loss": 1.3289, + "theoretical_loss": 3.598048650094566, + "tokens_seen": 1267335168 + }, + { + "epoch": 0.38, + "learning_rate": 0.000311123370110331, + "loss": 1.1826, + "theoretical_loss": 3.598031844758984, + "tokens_seen": 1267400704 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003111133400200602, + "loss": 1.3338, + "theoretical_loss": 3.598015040535668, + "tokens_seen": 1267466240 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031110330992978937, + "loss": 1.1696, + "theoretical_loss": 3.597998237424487, + "tokens_seen": 1267531776 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031109327983951855, + "loss": 1.3655, + "theoretical_loss": 3.597981435425311, + "tokens_seen": 1267597312 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031108324974924773, + "loss": 1.3027, + "theoretical_loss": 3.597964634538007, + "tokens_seen": 1267662848 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031107321965897697, + "loss": 1.3408, + "theoretical_loss": 3.5979478347624454, + "tokens_seen": 1267728384 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003110631895687061, + "loss": 1.2773, + "theoretical_loss": 3.5979310360984944, + "tokens_seen": 1267793920 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031105315947843533, + "loss": 1.2168, + "theoretical_loss": 3.597914238546023, + "tokens_seen": 1267859456 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031104312938816446, + "loss": 1.2619, + "theoretical_loss": 3.5978974421049017, + "tokens_seen": 1267924992 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003110330992978937, + "loss": 1.2551, + "theoretical_loss": 3.5978806467749975, + "tokens_seen": 1267990528 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003110230692076229, + "loss": 1.4105, + "theoretical_loss": 3.5978638525561806, + "tokens_seen": 1268056064 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 424145, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3245627880096436, + "objective/train/theoretical_loss": 3.5978470594483194, + "objective/train/tokens_used": 1268121600, + "theoretical_loss": 3.5978470594483194, + "tokens_seen": 1268121600 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031101303911735206, + "loss": 1.2696, + "theoretical_loss": 3.5978470594483194, + "tokens_seen": 1268121600 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031100300902708124, + "loss": 1.2455, + "theoretical_loss": 3.5978302674512834, + "tokens_seen": 1268187136 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003109929789368104, + "loss": 1.2983, + "theoretical_loss": 3.5978134765649417, + "tokens_seen": 1268252672 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003109829488465396, + "loss": 1.3695, + "theoretical_loss": 3.597796686789164, + "tokens_seen": 1268318208 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031097291875626883, + "loss": 1.3172, + "theoretical_loss": 3.5977798981238185, + "tokens_seen": 1268383744 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031096288866599796, + "loss": 1.2873, + "theoretical_loss": 3.597763110568775, + "tokens_seen": 1268449280 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003109528585757272, + "loss": 1.2688, + "theoretical_loss": 3.597746324123902, + "tokens_seen": 1268514816 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003109428284854564, + "loss": 1.4604, + "theoretical_loss": 3.5977295387890695, + "tokens_seen": 1268580352 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031093279839518556, + "loss": 1.2079, + "theoretical_loss": 3.5977127545641467, + "tokens_seen": 1268645888 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031092276830491474, + "loss": 1.2768, + "theoretical_loss": 3.597695971449003, + "tokens_seen": 1268711424 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003109127382146439, + "loss": 1.231, + "theoretical_loss": 3.5976791894435065, + "tokens_seen": 1268776960 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003109027081243731, + "loss": 1.3201, + "theoretical_loss": 3.597662408547528, + "tokens_seen": 1268842496 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031089267803410234, + "loss": 1.2195, + "theoretical_loss": 3.5976456287609357, + "tokens_seen": 1268908032 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031088264794383147, + "loss": 1.2021, + "theoretical_loss": 3.5976288500836002, + "tokens_seen": 1268973568 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003108726178535607, + "loss": 1.3079, + "theoretical_loss": 3.59761207251539, + "tokens_seen": 1269039104 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031086258776328983, + "loss": 1.3186, + "theoretical_loss": 3.5975952960561743, + "tokens_seen": 1269104640 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031085255767301906, + "loss": 1.3079, + "theoretical_loss": 3.5975785207058237, + "tokens_seen": 1269170176 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031084252758274824, + "loss": 1.2746, + "theoretical_loss": 3.5975617464642067, + "tokens_seen": 1269235712 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003108324974924774, + "loss": 1.191, + "theoretical_loss": 3.5975449733311926, + "tokens_seen": 1269301248 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003108224674022066, + "loss": 1.1984, + "theoretical_loss": 3.597528201306651, + "tokens_seen": 1269366784 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003108124373119358, + "loss": 1.2335, + "theoretical_loss": 3.5975114303904525, + "tokens_seen": 1269432320 + }, + { + "epoch": 0.38, + "learning_rate": 0.000310802407221665, + "loss": 1.2445, + "theoretical_loss": 3.5974946605824654, + "tokens_seen": 1269497856 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003107923771313942, + "loss": 1.302, + "theoretical_loss": 3.59747789188256, + "tokens_seen": 1269563392 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003107823470411234, + "loss": 1.2685, + "theoretical_loss": 3.5974611242906054, + "tokens_seen": 1269628928 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031077231695085257, + "loss": 1.3142, + "theoretical_loss": 3.597444357806471, + "tokens_seen": 1269694464 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003107622868605818, + "loss": 1.2569, + "theoretical_loss": 3.5974275924300274, + "tokens_seen": 1269760000 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031075225677031093, + "loss": 1.2492, + "theoretical_loss": 3.597410828161144, + "tokens_seen": 1269825536 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031074222668004016, + "loss": 1.2493, + "theoretical_loss": 3.5973940649996896, + "tokens_seen": 1269891072 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003107321965897693, + "loss": 1.3272, + "theoretical_loss": 3.5973773029455343, + "tokens_seen": 1269956608 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003107221664994985, + "loss": 1.345, + "theoretical_loss": 3.5973605419985484, + "tokens_seen": 1270022144 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003107121364092277, + "loss": 1.2561, + "theoretical_loss": 3.5973437821586014, + "tokens_seen": 1270087680 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003107021063189569, + "loss": 1.3121, + "theoretical_loss": 3.5973270234255628, + "tokens_seen": 1270153216 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031069207622868607, + "loss": 1.3791, + "theoretical_loss": 3.597310265799302, + "tokens_seen": 1270218752 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031068204613841525, + "loss": 1.3023, + "theoretical_loss": 3.5972935092796896, + "tokens_seen": 1270284288 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031067201604814443, + "loss": 1.4872, + "theoretical_loss": 3.5972767538665953, + "tokens_seen": 1270349824 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031066198595787367, + "loss": 1.3327, + "theoretical_loss": 3.5972599995598884, + "tokens_seen": 1270415360 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003106519558676028, + "loss": 1.2235, + "theoretical_loss": 3.5972432463594393, + "tokens_seen": 1270480896 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031064192577733203, + "loss": 1.3665, + "theoretical_loss": 3.5972264942651178, + "tokens_seen": 1270546432 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003106318956870612, + "loss": 1.2692, + "theoretical_loss": 3.597209743276794, + "tokens_seen": 1270611968 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003106218655967904, + "loss": 1.2707, + "theoretical_loss": 3.5971929933943376, + "tokens_seen": 1270677504 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031061183550651957, + "loss": 1.3235, + "theoretical_loss": 3.5971762446176183, + "tokens_seen": 1270743040 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031060180541624875, + "loss": 1.2683, + "theoretical_loss": 3.5971594969465066, + "tokens_seen": 1270808576 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031059177532597793, + "loss": 1.1827, + "theoretical_loss": 3.597142750380872, + "tokens_seen": 1270874112 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031058174523570717, + "loss": 1.2035, + "theoretical_loss": 3.597126004920585, + "tokens_seen": 1270939648 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003105717151454363, + "loss": 1.3075, + "theoretical_loss": 3.5971092605655155, + "tokens_seen": 1271005184 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031056168505516553, + "loss": 1.3341, + "theoretical_loss": 3.597092517315533, + "tokens_seen": 1271070720 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031055165496489466, + "loss": 1.2981, + "theoretical_loss": 3.5970757751705085, + "tokens_seen": 1271136256 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003105416248746239, + "loss": 1.2887, + "theoretical_loss": 3.5970590341303117, + "tokens_seen": 1271201792 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003105315947843531, + "loss": 1.1692, + "theoretical_loss": 3.5970422941948126, + "tokens_seen": 1271267328 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031052156469408226, + "loss": 1.3654, + "theoretical_loss": 3.5970255553638815, + "tokens_seen": 1271332864 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 425032, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3436036109924316, + "objective/train/theoretical_loss": 3.5970088176373887, + "objective/train/tokens_used": 1271398400, + "theoretical_loss": 3.5970088176373887, + "tokens_seen": 1271398400 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031051153460381144, + "loss": 1.4137, + "theoretical_loss": 3.5970088176373887, + "tokens_seen": 1271398400 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003105015045135406, + "loss": 1.2865, + "theoretical_loss": 3.5969920810152045, + "tokens_seen": 1271463936 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003104914744232698, + "loss": 1.2774, + "theoretical_loss": 3.596975345497199, + "tokens_seen": 1271529472 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031048144433299904, + "loss": 1.3115, + "theoretical_loss": 3.596958611083242, + "tokens_seen": 1271595008 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031047141424272816, + "loss": 1.3002, + "theoretical_loss": 3.5969418777732045, + "tokens_seen": 1271660544 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003104613841524574, + "loss": 1.4172, + "theoretical_loss": 3.5969251455669564, + "tokens_seen": 1271726080 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003104513540621866, + "loss": 1.2928, + "theoretical_loss": 3.5969084144643677, + "tokens_seen": 1271791616 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031044132397191576, + "loss": 1.1425, + "theoretical_loss": 3.5968916844653096, + "tokens_seen": 1271857152 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031043129388164494, + "loss": 1.3024, + "theoretical_loss": 3.5968749555696515, + "tokens_seen": 1271922688 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003104212637913741, + "loss": 1.2891, + "theoretical_loss": 3.5968582277772647, + "tokens_seen": 1271988224 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003104112337011033, + "loss": 1.1666, + "theoretical_loss": 3.5968415010880186, + "tokens_seen": 1272053760 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031040120361083254, + "loss": 1.4092, + "theoretical_loss": 3.5968247755017844, + "tokens_seen": 1272119296 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031039117352056167, + "loss": 1.2716, + "theoretical_loss": 3.5968080510184324, + "tokens_seen": 1272184832 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003103811434302909, + "loss": 1.2929, + "theoretical_loss": 3.596791327637833, + "tokens_seen": 1272250368 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031037111334002003, + "loss": 1.2692, + "theoretical_loss": 3.596774605359857, + "tokens_seen": 1272315904 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031036108324974926, + "loss": 1.2463, + "theoretical_loss": 3.596757884184374, + "tokens_seen": 1272381440 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031035105315947844, + "loss": 1.2559, + "theoretical_loss": 3.596741164111255, + "tokens_seen": 1272446976 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003103410230692076, + "loss": 1.1534, + "theoretical_loss": 3.596724445140371, + "tokens_seen": 1272512512 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003103309929789368, + "loss": 1.3335, + "theoretical_loss": 3.596707727271592, + "tokens_seen": 1272578048 + }, + { + "epoch": 0.39, + "learning_rate": 0.000310320962888666, + "loss": 1.3298, + "theoretical_loss": 3.596691010504789, + "tokens_seen": 1272643584 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031031093279839517, + "loss": 1.2235, + "theoretical_loss": 3.5966742948398327, + "tokens_seen": 1272709120 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003103009027081244, + "loss": 1.2237, + "theoretical_loss": 3.596657580276593, + "tokens_seen": 1272774656 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031029087261785353, + "loss": 1.2978, + "theoretical_loss": 3.5966408668149414, + "tokens_seen": 1272840192 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031028084252758277, + "loss": 1.2303, + "theoretical_loss": 3.5966241544547484, + "tokens_seen": 1272905728 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031027081243731195, + "loss": 1.269, + "theoretical_loss": 3.596607443195884, + "tokens_seen": 1272971264 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031026078234704113, + "loss": 1.2362, + "theoretical_loss": 3.59659073303822, + "tokens_seen": 1273036800 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003102507522567703, + "loss": 1.3749, + "theoretical_loss": 3.5965740239816264, + "tokens_seen": 1273102336 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003102407221664995, + "loss": 1.2059, + "theoretical_loss": 3.596557316025974, + "tokens_seen": 1273167872 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031023069207622867, + "loss": 1.3317, + "theoretical_loss": 3.596540609171134, + "tokens_seen": 1273233408 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003102206619859579, + "loss": 1.3277, + "theoretical_loss": 3.596523903416977, + "tokens_seen": 1273298944 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031021063189568703, + "loss": 1.2062, + "theoretical_loss": 3.5965071987633745, + "tokens_seen": 1273364480 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031020060180541627, + "loss": 1.3628, + "theoretical_loss": 3.5964904952101957, + "tokens_seen": 1273430016 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003101905717151454, + "loss": 1.3154, + "theoretical_loss": 3.5964737927573136, + "tokens_seen": 1273495552 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031018054162487463, + "loss": 1.3377, + "theoretical_loss": 3.596457091404597, + "tokens_seen": 1273561088 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003101705115346038, + "loss": 1.2015, + "theoretical_loss": 3.5964403911519183, + "tokens_seen": 1273626624 + }, + { + "epoch": 0.39, + "learning_rate": 0.000310160481444333, + "loss": 1.1942, + "theoretical_loss": 3.596423691999148, + "tokens_seen": 1273692160 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003101504513540622, + "loss": 1.3178, + "theoretical_loss": 3.5964069939461565, + "tokens_seen": 1273757696 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003101404212637914, + "loss": 1.2379, + "theoretical_loss": 3.596390296992816, + "tokens_seen": 1273823232 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031013039117352054, + "loss": 1.2513, + "theoretical_loss": 3.5963736011389966, + "tokens_seen": 1273888768 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031012036108324977, + "loss": 1.2645, + "theoretical_loss": 3.59635690638457, + "tokens_seen": 1273954304 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003101103309929789, + "loss": 1.2401, + "theoretical_loss": 3.596340212729406, + "tokens_seen": 1274019840 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031010030090270813, + "loss": 1.2037, + "theoretical_loss": 3.596323520173377, + "tokens_seen": 1274085376 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003100902708124373, + "loss": 1.3759, + "theoretical_loss": 3.5963068287163535, + "tokens_seen": 1274150912 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003100802407221665, + "loss": 1.2718, + "theoretical_loss": 3.5962901383582073, + "tokens_seen": 1274216448 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003100702106318957, + "loss": 1.3428, + "theoretical_loss": 3.596273449098809, + "tokens_seen": 1274281984 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031006018054162486, + "loss": 1.2856, + "theoretical_loss": 3.5962567609380285, + "tokens_seen": 1274347520 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003100501504513541, + "loss": 1.2988, + "theoretical_loss": 3.596240073875739, + "tokens_seen": 1274413056 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003100401203610833, + "loss": 1.2317, + "theoretical_loss": 3.5962233879118113, + "tokens_seen": 1274478592 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031003009027081246, + "loss": 1.2754, + "theoretical_loss": 3.596206703046116, + "tokens_seen": 1274544128 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031002006018054164, + "loss": 1.3104, + "theoretical_loss": 3.5961900192785246, + "tokens_seen": 1274609664 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 426234, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2053074836730957, + "objective/train/theoretical_loss": 3.5961733366089086, + "objective/train/tokens_used": 1274675200, + "theoretical_loss": 3.5961733366089086, + "tokens_seen": 1274675200 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003100100300902708, + "loss": 1.2556, + "theoretical_loss": 3.5961733366089086, + "tokens_seen": 1274675200 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031, + "loss": 1.1948, + "theoretical_loss": 3.596156655037139, + "tokens_seen": 1274740736 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030998996990972924, + "loss": 1.2444, + "theoretical_loss": 3.596139974563087, + "tokens_seen": 1274806272 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030997993981945836, + "loss": 1.3649, + "theoretical_loss": 3.5961232951866244, + "tokens_seen": 1274871808 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003099699097291876, + "loss": 1.3118, + "theoretical_loss": 3.5961066169076226, + "tokens_seen": 1274937344 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003099598796389168, + "loss": 1.2719, + "theoretical_loss": 3.5960899397259523, + "tokens_seen": 1275002880 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030994984954864596, + "loss": 1.3224, + "theoretical_loss": 3.596073263641485, + "tokens_seen": 1275068416 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030993981945837514, + "loss": 1.3939, + "theoretical_loss": 3.596056588654093, + "tokens_seen": 1275133952 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003099297893681043, + "loss": 1.1605, + "theoretical_loss": 3.596039914763647, + "tokens_seen": 1275199488 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003099197592778335, + "loss": 1.1844, + "theoretical_loss": 3.596023241970019, + "tokens_seen": 1275265024 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030990972918756274, + "loss": 1.3818, + "theoretical_loss": 3.5960065702730795, + "tokens_seen": 1275330560 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030989969909729187, + "loss": 1.2781, + "theoretical_loss": 3.5959898996727007, + "tokens_seen": 1275396096 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003098896690070211, + "loss": 1.2378, + "theoretical_loss": 3.595973230168754, + "tokens_seen": 1275461632 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030987963891675023, + "loss": 1.2682, + "theoretical_loss": 3.5959565617611116, + "tokens_seen": 1275527168 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030986960882647946, + "loss": 1.2515, + "theoretical_loss": 3.5959398944496446, + "tokens_seen": 1275592704 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030985957873620864, + "loss": 1.2905, + "theoretical_loss": 3.5959232282342235, + "tokens_seen": 1275658240 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003098495486459378, + "loss": 1.3368, + "theoretical_loss": 3.595906563114722, + "tokens_seen": 1275723776 + }, + { + "epoch": 0.39, + "learning_rate": 0.000309839518555667, + "loss": 1.319, + "theoretical_loss": 3.59588989909101, + "tokens_seen": 1275789312 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003098294884653962, + "loss": 1.1978, + "theoretical_loss": 3.5958732361629604, + "tokens_seen": 1275854848 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030981945837512537, + "loss": 1.2767, + "theoretical_loss": 3.595856574330444, + "tokens_seen": 1275920384 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003098094282848546, + "loss": 1.3243, + "theoretical_loss": 3.595839913593333, + "tokens_seen": 1275985920 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030979939819458373, + "loss": 1.2461, + "theoretical_loss": 3.595823253951499, + "tokens_seen": 1276051456 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030978936810431297, + "loss": 1.3011, + "theoretical_loss": 3.595806595404813, + "tokens_seen": 1276116992 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030977933801404215, + "loss": 1.2628, + "theoretical_loss": 3.5957899379531484, + "tokens_seen": 1276182528 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030976930792377133, + "loss": 1.2385, + "theoretical_loss": 3.595773281596376, + "tokens_seen": 1276248064 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003097592778335005, + "loss": 1.3421, + "theoretical_loss": 3.595756626334367, + "tokens_seen": 1276313600 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003097492477432297, + "loss": 1.2567, + "theoretical_loss": 3.595739972166995, + "tokens_seen": 1276379136 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030973921765295887, + "loss": 1.3713, + "theoretical_loss": 3.5957233190941302, + "tokens_seen": 1276444672 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003097291875626881, + "loss": 1.3781, + "theoretical_loss": 3.595706667115645, + "tokens_seen": 1276510208 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030971915747241723, + "loss": 1.3265, + "theoretical_loss": 3.595690016231411, + "tokens_seen": 1276575744 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030970912738214647, + "loss": 1.2684, + "theoretical_loss": 3.595673366441301, + "tokens_seen": 1276641280 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003096990972918756, + "loss": 1.3049, + "theoretical_loss": 3.595656717745187, + "tokens_seen": 1276706816 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030968906720160483, + "loss": 1.3799, + "theoretical_loss": 3.5956400701429394, + "tokens_seen": 1276772352 + }, + { + "epoch": 0.39, + "learning_rate": 0.000309679037111334, + "loss": 1.2819, + "theoretical_loss": 3.5956234236344313, + "tokens_seen": 1276837888 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003096690070210632, + "loss": 1.2932, + "theoretical_loss": 3.595606778219535, + "tokens_seen": 1276903424 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003096589769307924, + "loss": 1.2274, + "theoretical_loss": 3.595590133898122, + "tokens_seen": 1276968960 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003096489468405216, + "loss": 1.3389, + "theoretical_loss": 3.5955734906700645, + "tokens_seen": 1277034496 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030963891675025074, + "loss": 1.1499, + "theoretical_loss": 3.5955568485352343, + "tokens_seen": 1277100032 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030962888665997997, + "loss": 1.2983, + "theoretical_loss": 3.5955402074935043, + "tokens_seen": 1277165568 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003096188565697091, + "loss": 1.2896, + "theoretical_loss": 3.5955235675447454, + "tokens_seen": 1277231104 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030960882647943833, + "loss": 1.3096, + "theoretical_loss": 3.595506928688831, + "tokens_seen": 1277296640 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003095987963891675, + "loss": 1.2792, + "theoretical_loss": 3.5954902909256328, + "tokens_seen": 1277362176 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003095887662988967, + "loss": 1.3867, + "theoretical_loss": 3.5954736542550227, + "tokens_seen": 1277427712 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003095787362086259, + "loss": 1.3346, + "theoretical_loss": 3.5954570186768726, + "tokens_seen": 1277493248 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030956870611835506, + "loss": 1.3708, + "theoretical_loss": 3.5954403841910554, + "tokens_seen": 1277558784 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030955867602808424, + "loss": 1.2746, + "theoretical_loss": 3.595423750797443, + "tokens_seen": 1277624320 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003095486459378135, + "loss": 1.2545, + "theoretical_loss": 3.595407118495908, + "tokens_seen": 1277689856 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003095386158475426, + "loss": 1.2597, + "theoretical_loss": 3.5953904872863225, + "tokens_seen": 1277755392 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030952858575727184, + "loss": 1.3239, + "theoretical_loss": 3.5953738571685587, + "tokens_seen": 1277820928 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030951855566700096, + "loss": 1.2145, + "theoretical_loss": 3.5953572281424893, + "tokens_seen": 1277886464 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 427316, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3015271425247192, + "objective/train/theoretical_loss": 3.595340600207986, + "objective/train/tokens_used": 1277952000, + "theoretical_loss": 3.595340600207986, + "tokens_seen": 1277952000 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003095085255767302, + "loss": 1.2897, + "theoretical_loss": 3.595340600207986, + "tokens_seen": 1277952000 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003094984954864594, + "loss": 1.2525, + "theoretical_loss": 3.5953239733649216, + "tokens_seen": 1278017536 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030948846539618856, + "loss": 1.1707, + "theoretical_loss": 3.595307347613169, + "tokens_seen": 1278083072 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030947843530591774, + "loss": 1.3527, + "theoretical_loss": 3.5952907229525994, + "tokens_seen": 1278148608 + }, + { + "epoch": 0.39, + "learning_rate": 0.000309468405215647, + "loss": 1.3266, + "theoretical_loss": 3.5952740993830865, + "tokens_seen": 1278214144 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003094583751253761, + "loss": 1.3475, + "theoretical_loss": 3.595257476904502, + "tokens_seen": 1278279680 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030944834503510534, + "loss": 1.367, + "theoretical_loss": 3.595240855516718, + "tokens_seen": 1278345216 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030943831494483447, + "loss": 1.2807, + "theoretical_loss": 3.5952242352196078, + "tokens_seen": 1278410752 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003094282848545637, + "loss": 1.3136, + "theoretical_loss": 3.5952076160130435, + "tokens_seen": 1278476288 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003094182547642929, + "loss": 1.1163, + "theoretical_loss": 3.5951909978968986, + "tokens_seen": 1278541824 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030940822467402207, + "loss": 1.3097, + "theoretical_loss": 3.595174380871044, + "tokens_seen": 1278607360 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030939819458375125, + "loss": 1.2388, + "theoretical_loss": 3.5951577649353537, + "tokens_seen": 1278672896 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030938816449348043, + "loss": 1.3248, + "theoretical_loss": 3.5951411500896997, + "tokens_seen": 1278738432 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003093781344032096, + "loss": 1.1056, + "theoretical_loss": 3.595124536333955, + "tokens_seen": 1278803968 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030936810431293884, + "loss": 1.1888, + "theoretical_loss": 3.5951079236679915, + "tokens_seen": 1278869504 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030935807422266797, + "loss": 1.2349, + "theoretical_loss": 3.5950913120916823, + "tokens_seen": 1278935040 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003093480441323972, + "loss": 1.2727, + "theoretical_loss": 3.5950747016049007, + "tokens_seen": 1279000576 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030933801404212633, + "loss": 1.2277, + "theoretical_loss": 3.595058092207519, + "tokens_seen": 1279066112 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030932798395185557, + "loss": 1.2554, + "theoretical_loss": 3.5950414838994087, + "tokens_seen": 1279131648 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030931795386158475, + "loss": 1.3416, + "theoretical_loss": 3.5950248766804447, + "tokens_seen": 1279197184 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030930792377131393, + "loss": 1.2155, + "theoretical_loss": 3.5950082705504984, + "tokens_seen": 1279262720 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030929789368104317, + "loss": 1.2965, + "theoretical_loss": 3.594991665509443, + "tokens_seen": 1279328256 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030928786359077235, + "loss": 1.2892, + "theoretical_loss": 3.594975061557151, + "tokens_seen": 1279393792 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030927783350050153, + "loss": 1.2688, + "theoretical_loss": 3.5949584586934957, + "tokens_seen": 1279459328 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003092678034102307, + "loss": 1.2827, + "theoretical_loss": 3.59494185691835, + "tokens_seen": 1279524864 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003092577733199599, + "loss": 1.1808, + "theoretical_loss": 3.594925256231586, + "tokens_seen": 1279590400 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030924774322968907, + "loss": 1.3217, + "theoretical_loss": 3.594908656633078, + "tokens_seen": 1279655936 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003092377131394183, + "loss": 1.185, + "theoretical_loss": 3.5948920581226975, + "tokens_seen": 1279721472 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030922768304914743, + "loss": 1.3529, + "theoretical_loss": 3.594875460700319, + "tokens_seen": 1279787008 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030921765295887667, + "loss": 1.2857, + "theoretical_loss": 3.594858864365813, + "tokens_seen": 1279852544 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003092076228686058, + "loss": 1.2066, + "theoretical_loss": 3.594842269119055, + "tokens_seen": 1279918080 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030919759277833503, + "loss": 1.2964, + "theoretical_loss": 3.5948256749599174, + "tokens_seen": 1279983616 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003091875626880642, + "loss": 1.3279, + "theoretical_loss": 3.594809081888272, + "tokens_seen": 1280049152 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003091775325977934, + "loss": 1.2071, + "theoretical_loss": 3.5947924899039934, + "tokens_seen": 1280114688 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003091675025075226, + "loss": 1.3636, + "theoretical_loss": 3.5947758990069536, + "tokens_seen": 1280180224 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003091574724172518, + "loss": 1.291, + "theoretical_loss": 3.594759309197027, + "tokens_seen": 1280245760 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030914744232698094, + "loss": 1.3261, + "theoretical_loss": 3.594742720474085, + "tokens_seen": 1280311296 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030913741223671017, + "loss": 1.3477, + "theoretical_loss": 3.5947261328380016, + "tokens_seen": 1280376832 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003091273821464393, + "loss": 1.382, + "theoretical_loss": 3.5947095462886502, + "tokens_seen": 1280442368 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030911735205616853, + "loss": 1.3445, + "theoretical_loss": 3.594692960825904, + "tokens_seen": 1280507904 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003091073219658977, + "loss": 1.2704, + "theoretical_loss": 3.5946763764496357, + "tokens_seen": 1280573440 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003090972918756269, + "loss": 1.3916, + "theoretical_loss": 3.594659793159719, + "tokens_seen": 1280638976 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003090872617853561, + "loss": 1.2501, + "theoretical_loss": 3.5946432109560265, + "tokens_seen": 1280704512 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030907723169508526, + "loss": 1.3687, + "theoretical_loss": 3.594626629838433, + "tokens_seen": 1280770048 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030906720160481444, + "loss": 1.2011, + "theoretical_loss": 3.5946100498068096, + "tokens_seen": 1280835584 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003090571715145437, + "loss": 1.3336, + "theoretical_loss": 3.5945934708610308, + "tokens_seen": 1280901120 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003090471414242728, + "loss": 1.212, + "theoretical_loss": 3.59457689300097, + "tokens_seen": 1280966656 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030903711133400204, + "loss": 1.191, + "theoretical_loss": 3.5945603162265005, + "tokens_seen": 1281032192 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030902708124373116, + "loss": 1.2111, + "theoretical_loss": 3.594543740537496, + "tokens_seen": 1281097728 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003090170511534604, + "loss": 1.2155, + "theoretical_loss": 3.594527165933829, + "tokens_seen": 1281163264 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 428199, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2860890626907349, + "objective/train/theoretical_loss": 3.594510592415374, + "objective/train/tokens_used": 1281228800, + "theoretical_loss": 3.594510592415374, + "tokens_seen": 1281228800 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003090070210631896, + "loss": 1.2522, + "theoretical_loss": 3.594510592415374, + "tokens_seen": 1281228800 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030899699097291876, + "loss": 1.3275, + "theoretical_loss": 3.5944940199820037, + "tokens_seen": 1281294336 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030898696088264794, + "loss": 1.2629, + "theoretical_loss": 3.594477448633591, + "tokens_seen": 1281359872 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003089769307923772, + "loss": 1.2081, + "theoretical_loss": 3.594460878370011, + "tokens_seen": 1281425408 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003089669007021063, + "loss": 1.2896, + "theoretical_loss": 3.594444309191136, + "tokens_seen": 1281490944 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030895687061183554, + "loss": 1.3735, + "theoretical_loss": 3.5944277410968395, + "tokens_seen": 1281556480 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030894684052156467, + "loss": 1.3093, + "theoretical_loss": 3.594411174086996, + "tokens_seen": 1281622016 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003089368104312939, + "loss": 1.4171, + "theoretical_loss": 3.594394608161478, + "tokens_seen": 1281687552 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003089267803410231, + "loss": 1.2271, + "theoretical_loss": 3.5943780433201598, + "tokens_seen": 1281753088 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030891675025075227, + "loss": 1.3489, + "theoretical_loss": 3.594361479562915, + "tokens_seen": 1281818624 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030890672016048145, + "loss": 1.3006, + "theoretical_loss": 3.594344916889616, + "tokens_seen": 1281884160 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030889669007021063, + "loss": 1.2827, + "theoretical_loss": 3.5943283553001386, + "tokens_seen": 1281949696 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003088866599799398, + "loss": 1.3264, + "theoretical_loss": 3.594311794794355, + "tokens_seen": 1282015232 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030887662988966904, + "loss": 1.3363, + "theoretical_loss": 3.5942952353721394, + "tokens_seen": 1282080768 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030886659979939817, + "loss": 1.3672, + "theoretical_loss": 3.5942786770333646, + "tokens_seen": 1282146304 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003088565697091274, + "loss": 1.1075, + "theoretical_loss": 3.594262119777906, + "tokens_seen": 1282211840 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030884653961885653, + "loss": 1.3429, + "theoretical_loss": 3.594245563605636, + "tokens_seen": 1282277376 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030883650952858577, + "loss": 1.2338, + "theoretical_loss": 3.594229008516429, + "tokens_seen": 1282342912 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030882647943831495, + "loss": 1.3814, + "theoretical_loss": 3.594212454510158, + "tokens_seen": 1282408448 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030881644934804413, + "loss": 1.2439, + "theoretical_loss": 3.594195901586698, + "tokens_seen": 1282473984 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003088064192577733, + "loss": 1.2604, + "theoretical_loss": 3.5941793497459225, + "tokens_seen": 1282539520 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030879638916750255, + "loss": 1.4003, + "theoretical_loss": 3.5941627989877047, + "tokens_seen": 1282605056 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003087863590772317, + "loss": 1.3902, + "theoretical_loss": 3.594146249311919, + "tokens_seen": 1282670592 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003087763289869609, + "loss": 1.2555, + "theoretical_loss": 3.59412970071844, + "tokens_seen": 1282736128 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030876629889669004, + "loss": 1.3822, + "theoretical_loss": 3.59411315320714, + "tokens_seen": 1282801664 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030875626880641927, + "loss": 1.3528, + "theoretical_loss": 3.594096606777894, + "tokens_seen": 1282867200 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030874623871614845, + "loss": 1.4175, + "theoretical_loss": 3.594080061430576, + "tokens_seen": 1282932736 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030873620862587763, + "loss": 1.28, + "theoretical_loss": 3.5940635171650603, + "tokens_seen": 1282998272 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003087261785356068, + "loss": 1.23, + "theoretical_loss": 3.5940469739812198, + "tokens_seen": 1283063808 + }, + { + "epoch": 0.39, + "learning_rate": 0.000308716148445336, + "loss": 1.2252, + "theoretical_loss": 3.594030431878929, + "tokens_seen": 1283129344 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003087061183550652, + "loss": 1.2847, + "theoretical_loss": 3.5940138908580628, + "tokens_seen": 1283194880 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003086960882647944, + "loss": 1.3053, + "theoretical_loss": 3.593997350918494, + "tokens_seen": 1283260416 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030868605817452354, + "loss": 1.2818, + "theoretical_loss": 3.5939808120600976, + "tokens_seen": 1283325952 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003086760280842528, + "loss": 1.276, + "theoretical_loss": 3.5939642742827473, + "tokens_seen": 1283391488 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003086659979939819, + "loss": 1.2138, + "theoretical_loss": 3.5939477375863174, + "tokens_seen": 1283457024 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030865596790371114, + "loss": 1.302, + "theoretical_loss": 3.593931201970682, + "tokens_seen": 1283522560 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003086459378134403, + "loss": 1.3292, + "theoretical_loss": 3.5939146674357154, + "tokens_seen": 1283588096 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003086359077231695, + "loss": 1.3839, + "theoretical_loss": 3.593898133981291, + "tokens_seen": 1283653632 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003086258776328987, + "loss": 1.2773, + "theoretical_loss": 3.5938816016072845, + "tokens_seen": 1283719168 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003086158475426279, + "loss": 1.2578, + "theoretical_loss": 3.5938650703135693, + "tokens_seen": 1283784704 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030860581745235704, + "loss": 1.3718, + "theoretical_loss": 3.5938485401000193, + "tokens_seen": 1283850240 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003085957873620863, + "loss": 1.2274, + "theoretical_loss": 3.59383201096651, + "tokens_seen": 1283915776 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003085857572718154, + "loss": 1.3252, + "theoretical_loss": 3.5938154829129143, + "tokens_seen": 1283981312 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030857572718154464, + "loss": 1.3654, + "theoretical_loss": 3.593798955939107, + "tokens_seen": 1284046848 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003085656970912738, + "loss": 1.3063, + "theoretical_loss": 3.5937824300449632, + "tokens_seen": 1284112384 + }, + { + "epoch": 0.39, + "learning_rate": 0.000308555667001003, + "loss": 1.2803, + "theoretical_loss": 3.5937659052303563, + "tokens_seen": 1284177920 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030854563691073224, + "loss": 1.2998, + "theoretical_loss": 3.5937493814951615, + "tokens_seen": 1284243456 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030853560682046137, + "loss": 1.2666, + "theoretical_loss": 3.5937328588392523, + "tokens_seen": 1284308992 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003085255767301906, + "loss": 1.2615, + "theoretical_loss": 3.593716337262504, + "tokens_seen": 1284374528 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003085155466399198, + "loss": 1.2352, + "theoretical_loss": 3.59369981676479, + "tokens_seen": 1284440064 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 429310, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3709691762924194, + "objective/train/theoretical_loss": 3.5936832973459865, + "objective/train/tokens_used": 1284505600, + "theoretical_loss": 3.5936832973459865, + "tokens_seen": 1284505600 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030850551654964896, + "loss": 1.3227, + "theoretical_loss": 3.5936832973459865, + "tokens_seen": 1284505600 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030849548645937814, + "loss": 1.1834, + "theoretical_loss": 3.593666779005966, + "tokens_seen": 1284571136 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003084854563691074, + "loss": 1.3304, + "theoretical_loss": 3.5936502617446044, + "tokens_seen": 1284636672 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003084754262788365, + "loss": 1.2728, + "theoretical_loss": 3.593633745561776, + "tokens_seen": 1284702208 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030846539618856574, + "loss": 1.2979, + "theoretical_loss": 3.593617230457355, + "tokens_seen": 1284767744 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030845536609829487, + "loss": 1.1935, + "theoretical_loss": 3.593600716431216, + "tokens_seen": 1284833280 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003084453360080241, + "loss": 1.1992, + "theoretical_loss": 3.593584203483234, + "tokens_seen": 1284898816 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003084353059177533, + "loss": 1.3562, + "theoretical_loss": 3.5935676916132833, + "tokens_seen": 1284964352 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030842527582748247, + "loss": 1.3047, + "theoretical_loss": 3.593551180821239, + "tokens_seen": 1285029888 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030841524573721165, + "loss": 1.1892, + "theoretical_loss": 3.593534671106975, + "tokens_seen": 1285095424 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030840521564694083, + "loss": 1.3259, + "theoretical_loss": 3.5935181624703665, + "tokens_seen": 1285160960 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030839518555667, + "loss": 1.2603, + "theoretical_loss": 3.5935016549112877, + "tokens_seen": 1285226496 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030838515546639924, + "loss": 1.3096, + "theoretical_loss": 3.5934851484296146, + "tokens_seen": 1285292032 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030837512537612837, + "loss": 1.273, + "theoretical_loss": 3.5934686430252203, + "tokens_seen": 1285357568 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003083650952858576, + "loss": 1.2602, + "theoretical_loss": 3.5934521386979807, + "tokens_seen": 1285423104 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030835506519558673, + "loss": 1.163, + "theoretical_loss": 3.5934356354477703, + "tokens_seen": 1285488640 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030834503510531597, + "loss": 1.2986, + "theoretical_loss": 3.5934191332744634, + "tokens_seen": 1285554176 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030833500501504515, + "loss": 1.1947, + "theoretical_loss": 3.593402632177936, + "tokens_seen": 1285619712 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030832497492477433, + "loss": 1.2699, + "theoretical_loss": 3.593386132158062, + "tokens_seen": 1285685248 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003083149448345035, + "loss": 1.1793, + "theoretical_loss": 3.5933696332147163, + "tokens_seen": 1285750784 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030830491474423275, + "loss": 1.2377, + "theoretical_loss": 3.593353135347774, + "tokens_seen": 1285816320 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003082948846539619, + "loss": 1.3934, + "theoretical_loss": 3.59333663855711, + "tokens_seen": 1285881856 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003082848545636911, + "loss": 1.2341, + "theoretical_loss": 3.5933201428425994, + "tokens_seen": 1285947392 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030827482447342024, + "loss": 1.3055, + "theoretical_loss": 3.5933036482041167, + "tokens_seen": 1286012928 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030826479438314947, + "loss": 1.2873, + "theoretical_loss": 3.593287154641538, + "tokens_seen": 1286078464 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030825476429287865, + "loss": 1.333, + "theoretical_loss": 3.593270662154737, + "tokens_seen": 1286144000 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030824473420260783, + "loss": 1.3203, + "theoretical_loss": 3.593254170743589, + "tokens_seen": 1286209536 + }, + { + "epoch": 0.39, + "learning_rate": 0.000308234704112337, + "loss": 1.3118, + "theoretical_loss": 3.5932376804079698, + "tokens_seen": 1286275072 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003082246740220662, + "loss": 1.2071, + "theoretical_loss": 3.5932211911477534, + "tokens_seen": 1286340608 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003082146439317954, + "loss": 1.3374, + "theoretical_loss": 3.593204702962816, + "tokens_seen": 1286406144 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003082046138415246, + "loss": 1.3881, + "theoretical_loss": 3.593188215853032, + "tokens_seen": 1286471680 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030819458375125374, + "loss": 1.2998, + "theoretical_loss": 3.593171729818276, + "tokens_seen": 1286537216 + }, + { + "epoch": 0.39, + "learning_rate": 0.000308184553660983, + "loss": 1.2453, + "theoretical_loss": 3.5931552448584245, + "tokens_seen": 1286602752 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003081745235707121, + "loss": 1.1767, + "theoretical_loss": 3.5931387609733516, + "tokens_seen": 1286668288 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030816449348044134, + "loss": 1.236, + "theoretical_loss": 3.5931222781629333, + "tokens_seen": 1286733824 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003081544633901705, + "loss": 1.2527, + "theoretical_loss": 3.593105796427044, + "tokens_seen": 1286799360 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003081444332998997, + "loss": 1.2456, + "theoretical_loss": 3.5930893157655595, + "tokens_seen": 1286864896 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003081344032096289, + "loss": 1.3505, + "theoretical_loss": 3.593072836178355, + "tokens_seen": 1286930432 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003081243731193581, + "loss": 1.2664, + "theoretical_loss": 3.593056357665305, + "tokens_seen": 1286995968 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030811434302908724, + "loss": 1.3231, + "theoretical_loss": 3.5930398802262857, + "tokens_seen": 1287061504 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003081043129388165, + "loss": 1.3958, + "theoretical_loss": 3.5930234038611726, + "tokens_seen": 1287127040 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003080942828485456, + "loss": 1.4131, + "theoretical_loss": 3.59300692856984, + "tokens_seen": 1287192576 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030808425275827484, + "loss": 1.3279, + "theoretical_loss": 3.5929904543521642, + "tokens_seen": 1287258112 + }, + { + "epoch": 0.39, + "learning_rate": 0.000308074222668004, + "loss": 1.2301, + "theoretical_loss": 3.59297398120802, + "tokens_seen": 1287323648 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003080641925777332, + "loss": 1.2612, + "theoretical_loss": 3.5929575091372827, + "tokens_seen": 1287389184 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003080541624874624, + "loss": 1.2689, + "theoretical_loss": 3.592941038139829, + "tokens_seen": 1287454720 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030804413239719157, + "loss": 1.2628, + "theoretical_loss": 3.5929245682155324, + "tokens_seen": 1287520256 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030803410230692075, + "loss": 1.2769, + "theoretical_loss": 3.5929080993642692, + "tokens_seen": 1287585792 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030802407221665, + "loss": 1.2298, + "theoretical_loss": 3.5928916315859154, + "tokens_seen": 1287651328 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003080140421263791, + "loss": 1.2634, + "theoretical_loss": 3.592875164880346, + "tokens_seen": 1287716864 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 430354, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2102304697036743, + "objective/train/theoretical_loss": 3.592858699247437, + "objective/train/tokens_used": 1287782400, + "theoretical_loss": 3.592858699247437, + "tokens_seen": 1287782400 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030800401203610834, + "loss": 1.2554, + "theoretical_loss": 3.592858699247437, + "tokens_seen": 1287782400 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003079939819458375, + "loss": 1.3197, + "theoretical_loss": 3.592842234687063, + "tokens_seen": 1287847936 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003079839518555667, + "loss": 1.2875, + "theoretical_loss": 3.5928257711991005, + "tokens_seen": 1287913472 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003079739217652959, + "loss": 1.2501, + "theoretical_loss": 3.5928093087834245, + "tokens_seen": 1287979008 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030796389167502507, + "loss": 1.2827, + "theoretical_loss": 3.592792847439911, + "tokens_seen": 1288044544 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030795386158475425, + "loss": 1.1762, + "theoretical_loss": 3.5927763871684353, + "tokens_seen": 1288110080 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003079438314944835, + "loss": 1.3557, + "theoretical_loss": 3.5927599279688733, + "tokens_seen": 1288175616 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003079338014042126, + "loss": 1.2631, + "theoretical_loss": 3.5927434698411003, + "tokens_seen": 1288241152 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030792377131394185, + "loss": 1.3645, + "theoretical_loss": 3.5927270127849926, + "tokens_seen": 1288306688 + }, + { + "epoch": 0.39, + "learning_rate": 0.000307913741223671, + "loss": 1.3138, + "theoretical_loss": 3.592710556800425, + "tokens_seen": 1288372224 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003079037111334002, + "loss": 1.4183, + "theoretical_loss": 3.5926941018872744, + "tokens_seen": 1288437760 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003078936810431294, + "loss": 1.1455, + "theoretical_loss": 3.5926776480454157, + "tokens_seen": 1288503296 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030788365095285857, + "loss": 1.268, + "theoretical_loss": 3.5926611952747245, + "tokens_seen": 1288568832 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030787362086258775, + "loss": 1.3279, + "theoretical_loss": 3.5926447435750775, + "tokens_seen": 1288634368 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030786359077231693, + "loss": 1.3347, + "theoretical_loss": 3.5926282929463498, + "tokens_seen": 1288699904 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003078535606820461, + "loss": 1.3093, + "theoretical_loss": 3.5926118433884175, + "tokens_seen": 1288765440 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030784353059177535, + "loss": 1.3014, + "theoretical_loss": 3.5925953949011564, + "tokens_seen": 1288830976 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003078335005015045, + "loss": 1.3383, + "theoretical_loss": 3.5925789474844425, + "tokens_seen": 1288896512 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003078234704112337, + "loss": 1.2671, + "theoretical_loss": 3.5925625011381515, + "tokens_seen": 1288962048 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003078134403209629, + "loss": 1.2178, + "theoretical_loss": 3.592546055862159, + "tokens_seen": 1289027584 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003078034102306921, + "loss": 1.4324, + "theoretical_loss": 3.5925296116563414, + "tokens_seen": 1289093120 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003077933801404213, + "loss": 1.2024, + "theoretical_loss": 3.5925131685205747, + "tokens_seen": 1289158656 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030778335005015044, + "loss": 1.2971, + "theoretical_loss": 3.592496726454735, + "tokens_seen": 1289224192 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030777331995987967, + "loss": 1.2471, + "theoretical_loss": 3.5924802854586977, + "tokens_seen": 1289289728 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030776328986960885, + "loss": 1.3045, + "theoretical_loss": 3.592463845532339, + "tokens_seen": 1289355264 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030775325977933803, + "loss": 1.2952, + "theoretical_loss": 3.592447406675536, + "tokens_seen": 1289420800 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003077432296890672, + "loss": 1.4216, + "theoretical_loss": 3.592430968888163, + "tokens_seen": 1289486336 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003077331995987964, + "loss": 1.3593, + "theoretical_loss": 3.592414532170097, + "tokens_seen": 1289551872 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003077231695085256, + "loss": 1.3489, + "theoretical_loss": 3.5923980965212143, + "tokens_seen": 1289617408 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003077131394182548, + "loss": 1.3786, + "theoretical_loss": 3.5923816619413906, + "tokens_seen": 1289682944 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030770310932798394, + "loss": 1.3374, + "theoretical_loss": 3.5923652284305025, + "tokens_seen": 1289748480 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003076930792377132, + "loss": 1.2505, + "theoretical_loss": 3.5923487959884257, + "tokens_seen": 1289814016 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003076830491474423, + "loss": 1.4916, + "theoretical_loss": 3.5923323646150367, + "tokens_seen": 1289879552 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030767301905717154, + "loss": 1.4298, + "theoretical_loss": 3.5923159343102116, + "tokens_seen": 1289945088 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003076629889669007, + "loss": 1.2706, + "theoretical_loss": 3.592299505073826, + "tokens_seen": 1290010624 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003076529588766299, + "loss": 1.2535, + "theoretical_loss": 3.5922830769057574, + "tokens_seen": 1290076160 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003076429287863591, + "loss": 1.4213, + "theoretical_loss": 3.592266649805881, + "tokens_seen": 1290141696 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003076328986960883, + "loss": 1.2916, + "theoretical_loss": 3.592250223774074, + "tokens_seen": 1290207232 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030762286860581744, + "loss": 1.4197, + "theoretical_loss": 3.592233798810212, + "tokens_seen": 1290272768 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003076128385155467, + "loss": 1.3887, + "theoretical_loss": 3.592217374914171, + "tokens_seen": 1290338304 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003076028084252758, + "loss": 1.3697, + "theoretical_loss": 3.5922009520858285, + "tokens_seen": 1290403840 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030759277833500504, + "loss": 1.3917, + "theoretical_loss": 3.59218453032506, + "tokens_seen": 1290469376 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003075827482447342, + "loss": 1.4371, + "theoretical_loss": 3.592168109631742, + "tokens_seen": 1290534912 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003075727181544634, + "loss": 1.4665, + "theoretical_loss": 3.5921516900057515, + "tokens_seen": 1290600448 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003075626880641926, + "loss": 1.4437, + "theoretical_loss": 3.5921352714469643, + "tokens_seen": 1290665984 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030755265797392177, + "loss": 1.2447, + "theoretical_loss": 3.592118853955257, + "tokens_seen": 1290731520 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030754262788365095, + "loss": 1.3459, + "theoretical_loss": 3.592102437530506, + "tokens_seen": 1290797056 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003075325977933802, + "loss": 1.2853, + "theoretical_loss": 3.5920860221725883, + "tokens_seen": 1290862592 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003075225677031093, + "loss": 1.2752, + "theoretical_loss": 3.5920696078813794, + "tokens_seen": 1290928128 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030751253761283854, + "loss": 1.2847, + "theoretical_loss": 3.592053194656757, + "tokens_seen": 1290993664 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 431564, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3212494850158691, + "objective/train/theoretical_loss": 3.592036782498597, + "objective/train/tokens_used": 1291059200, + "theoretical_loss": 3.592036782498597, + "tokens_seen": 1291059200 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003075025075225677, + "loss": 1.3587, + "theoretical_loss": 3.592036782498597, + "tokens_seen": 1291059200 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003074924774322969, + "loss": 1.3234, + "theoretical_loss": 3.5920203714067758, + "tokens_seen": 1291124736 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003074824473420261, + "loss": 1.3849, + "theoretical_loss": 3.5920039613811703, + "tokens_seen": 1291190272 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030747241725175527, + "loss": 1.3179, + "theoretical_loss": 3.5919875524216573, + "tokens_seen": 1291255808 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030746238716148445, + "loss": 1.285, + "theoretical_loss": 3.591971144528113, + "tokens_seen": 1291321344 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003074523570712137, + "loss": 1.2866, + "theoretical_loss": 3.591954737700415, + "tokens_seen": 1291386880 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003074423269809428, + "loss": 1.292, + "theoretical_loss": 3.5919383319384384, + "tokens_seen": 1291452416 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030743229689067205, + "loss": 1.2513, + "theoretical_loss": 3.591921927242061, + "tokens_seen": 1291517952 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003074222668004012, + "loss": 1.2893, + "theoretical_loss": 3.5919055236111594, + "tokens_seen": 1291583488 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003074122367101304, + "loss": 1.2857, + "theoretical_loss": 3.5918891210456105, + "tokens_seen": 1291649024 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003074022066198596, + "loss": 1.3249, + "theoretical_loss": 3.5918727195452904, + "tokens_seen": 1291714560 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030739217652958877, + "loss": 1.3903, + "theoretical_loss": 3.5918563191100765, + "tokens_seen": 1291780096 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030738214643931795, + "loss": 1.4224, + "theoretical_loss": 3.5918399197398454, + "tokens_seen": 1291845632 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030737211634904713, + "loss": 1.3117, + "theoretical_loss": 3.591823521434474, + "tokens_seen": 1291911168 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003073620862587763, + "loss": 1.2206, + "theoretical_loss": 3.5918071241938385, + "tokens_seen": 1291976704 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030735205616850555, + "loss": 1.4279, + "theoretical_loss": 3.5917907280178167, + "tokens_seen": 1292042240 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003073420260782347, + "loss": 1.281, + "theoretical_loss": 3.5917743329062852, + "tokens_seen": 1292107776 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003073319959879639, + "loss": 1.3642, + "theoretical_loss": 3.5917579388591205, + "tokens_seen": 1292173312 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003073219658976931, + "loss": 1.2913, + "theoretical_loss": 3.5917415458762, + "tokens_seen": 1292238848 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003073119358074223, + "loss": 1.3903, + "theoretical_loss": 3.5917251539574004, + "tokens_seen": 1292304384 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030730190571715146, + "loss": 1.3111, + "theoretical_loss": 3.5917087631025986, + "tokens_seen": 1292369920 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030729187562688064, + "loss": 1.2758, + "theoretical_loss": 3.591692373311672, + "tokens_seen": 1292435456 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003072818455366098, + "loss": 1.2321, + "theoretical_loss": 3.591675984584497, + "tokens_seen": 1292500992 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030727181544633905, + "loss": 1.3749, + "theoretical_loss": 3.591659596920951, + "tokens_seen": 1292566528 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003072617853560682, + "loss": 1.3247, + "theoretical_loss": 3.5916432103209113, + "tokens_seen": 1292632064 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003072517552657974, + "loss": 1.2838, + "theoretical_loss": 3.591626824784255, + "tokens_seen": 1292697600 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030724172517552654, + "loss": 1.1932, + "theoretical_loss": 3.5916104403108577, + "tokens_seen": 1292763136 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003072316950852558, + "loss": 1.4322, + "theoretical_loss": 3.5915940569005986, + "tokens_seen": 1292828672 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030722166499498496, + "loss": 1.2434, + "theoretical_loss": 3.5915776745533536, + "tokens_seen": 1292894208 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030721163490471414, + "loss": 1.3761, + "theoretical_loss": 3.5915612932689998, + "tokens_seen": 1292959744 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003072016048144433, + "loss": 1.3479, + "theoretical_loss": 3.5915449130474153, + "tokens_seen": 1293025280 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003071915747241725, + "loss": 1.2866, + "theoretical_loss": 3.591528533888476, + "tokens_seen": 1293090816 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003071815446339017, + "loss": 1.3271, + "theoretical_loss": 3.5915121557920604, + "tokens_seen": 1293156352 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003071715145436309, + "loss": 1.3435, + "theoretical_loss": 3.591495778758045, + "tokens_seen": 1293221888 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030716148445336005, + "loss": 1.3484, + "theoretical_loss": 3.591479402786307, + "tokens_seen": 1293287424 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003071514543630893, + "loss": 1.387, + "theoretical_loss": 3.591463027876724, + "tokens_seen": 1293352960 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030714142427281846, + "loss": 1.2777, + "theoretical_loss": 3.5914466540291734, + "tokens_seen": 1293418496 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030713139418254764, + "loss": 1.3496, + "theoretical_loss": 3.5914302812435315, + "tokens_seen": 1293484032 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003071213640922768, + "loss": 1.5035, + "theoretical_loss": 3.591413909519677, + "tokens_seen": 1293549568 + }, + { + "epoch": 0.39, + "learning_rate": 0.000307111334002006, + "loss": 1.3221, + "theoretical_loss": 3.5913975388574864, + "tokens_seen": 1293615104 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003071013039117352, + "loss": 1.3467, + "theoretical_loss": 3.591381169256838, + "tokens_seen": 1293680640 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003070912738214644, + "loss": 1.2892, + "theoretical_loss": 3.5913648007176078, + "tokens_seen": 1293746176 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030708124373119355, + "loss": 1.3963, + "theoretical_loss": 3.5913484332396743, + "tokens_seen": 1293811712 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003070712136409228, + "loss": 1.2937, + "theoretical_loss": 3.5913320668229143, + "tokens_seen": 1293877248 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003070611835506519, + "loss": 1.3377, + "theoretical_loss": 3.5913157014672055, + "tokens_seen": 1293942784 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030705115346038115, + "loss": 1.3043, + "theoretical_loss": 3.5912993371724258, + "tokens_seen": 1294008320 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003070411233701104, + "loss": 1.2173, + "theoretical_loss": 3.591282973938452, + "tokens_seen": 1294073856 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003070310932798395, + "loss": 1.2837, + "theoretical_loss": 3.591266611765162, + "tokens_seen": 1294139392 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030702106318956874, + "loss": 1.3357, + "theoretical_loss": 3.5912502506524335, + "tokens_seen": 1294204928 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003070110330992979, + "loss": 1.4181, + "theoretical_loss": 3.5912338906001433, + "tokens_seen": 1294270464 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 432831, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.316493272781372, + "objective/train/theoretical_loss": 3.5912175316081694, + "objective/train/tokens_used": 1294336000, + "theoretical_loss": 3.5912175316081694, + "tokens_seen": 1294336000 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003070010030090271, + "loss": 1.3048, + "theoretical_loss": 3.5912175316081694, + "tokens_seen": 1294336000 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003069909729187563, + "loss": 1.3473, + "theoretical_loss": 3.59120117367639, + "tokens_seen": 1294401536 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030698094282848547, + "loss": 1.4015, + "theoretical_loss": 3.591184816804682, + "tokens_seen": 1294467072 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030697091273821465, + "loss": 1.3886, + "theoretical_loss": 3.5911684609929235, + "tokens_seen": 1294532608 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003069608826479439, + "loss": 1.3866, + "theoretical_loss": 3.5911521062409912, + "tokens_seen": 1294598144 + }, + { + "epoch": 0.39, + "learning_rate": 0.000306950852557673, + "loss": 1.2515, + "theoretical_loss": 3.591135752548764, + "tokens_seen": 1294663680 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030694082246740225, + "loss": 1.4092, + "theoretical_loss": 3.591119399916119, + "tokens_seen": 1294729216 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003069307923771314, + "loss": 1.3618, + "theoretical_loss": 3.5911030483429345, + "tokens_seen": 1294794752 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003069207622868606, + "loss": 1.3411, + "theoretical_loss": 3.5910866978290867, + "tokens_seen": 1294860288 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003069107321965898, + "loss": 1.3499, + "theoretical_loss": 3.5910703483744553, + "tokens_seen": 1294925824 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030690070210631897, + "loss": 1.4194, + "theoretical_loss": 3.591053999978917, + "tokens_seen": 1294991360 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030689067201604815, + "loss": 1.2884, + "theoretical_loss": 3.591037652642349, + "tokens_seen": 1295056896 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030688064192577733, + "loss": 1.5124, + "theoretical_loss": 3.591021306364631, + "tokens_seen": 1295122432 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003068706118355065, + "loss": 1.3213, + "theoretical_loss": 3.5910049611456394, + "tokens_seen": 1295187968 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030686058174523575, + "loss": 1.1959, + "theoretical_loss": 3.590988616985252, + "tokens_seen": 1295253504 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003068505516549649, + "loss": 1.2745, + "theoretical_loss": 3.5909722738833474, + "tokens_seen": 1295319040 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003068405215646941, + "loss": 1.2086, + "theoretical_loss": 3.590955931839803, + "tokens_seen": 1295384576 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003068304914744233, + "loss": 1.4578, + "theoretical_loss": 3.5909395908544974, + "tokens_seen": 1295450112 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003068204613841525, + "loss": 1.3331, + "theoretical_loss": 3.5909232509273075, + "tokens_seen": 1295515648 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030681043129388166, + "loss": 1.3378, + "theoretical_loss": 3.590906912058112, + "tokens_seen": 1295581184 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030680040120361084, + "loss": 1.5238, + "theoretical_loss": 3.5908905742467887, + "tokens_seen": 1295646720 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030679037111334, + "loss": 1.2877, + "theoretical_loss": 3.5908742374932157, + "tokens_seen": 1295712256 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030678034102306925, + "loss": 1.4311, + "theoretical_loss": 3.590857901797271, + "tokens_seen": 1295777792 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003067703109327984, + "loss": 1.4698, + "theoretical_loss": 3.590841567158832, + "tokens_seen": 1295843328 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003067602808425276, + "loss": 1.3298, + "theoretical_loss": 3.590825233577778, + "tokens_seen": 1295908864 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030675025075225674, + "loss": 1.3754, + "theoretical_loss": 3.590808901053986, + "tokens_seen": 1295974400 + }, + { + "epoch": 0.39, + "learning_rate": 0.000306740220661986, + "loss": 1.4892, + "theoretical_loss": 3.590792569587335, + "tokens_seen": 1296039936 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030673019057171516, + "loss": 1.3543, + "theoretical_loss": 3.5907762391777025, + "tokens_seen": 1296105472 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030672016048144434, + "loss": 1.3439, + "theoretical_loss": 3.5907599098249667, + "tokens_seen": 1296171008 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003067101303911735, + "loss": 1.3151, + "theoretical_loss": 3.5907435815290056, + "tokens_seen": 1296236544 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003067001003009027, + "loss": 1.2613, + "theoretical_loss": 3.590727254289698, + "tokens_seen": 1296302080 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003066900702106319, + "loss": 1.2736, + "theoretical_loss": 3.5907109281069216, + "tokens_seen": 1296367616 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003066800401203611, + "loss": 1.3277, + "theoretical_loss": 3.5906946029805544, + "tokens_seen": 1296433152 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030667001003009025, + "loss": 1.2675, + "theoretical_loss": 3.5906782789104756, + "tokens_seen": 1296498688 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003066599799398195, + "loss": 1.2268, + "theoretical_loss": 3.5906619558965627, + "tokens_seen": 1296564224 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030664994984954866, + "loss": 1.2647, + "theoretical_loss": 3.5906456339386943, + "tokens_seen": 1296629760 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030663991975927784, + "loss": 1.315, + "theoretical_loss": 3.5906293130367484, + "tokens_seen": 1296695296 + }, + { + "epoch": 0.39, + "learning_rate": 0.000306629889669007, + "loss": 1.2205, + "theoretical_loss": 3.5906129931906037, + "tokens_seen": 1296760832 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003066198595787362, + "loss": 1.3153, + "theoretical_loss": 3.590596674400138, + "tokens_seen": 1296826368 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003066098294884654, + "loss": 1.4728, + "theoretical_loss": 3.590580356665231, + "tokens_seen": 1296891904 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003065997993981946, + "loss": 1.2939, + "theoretical_loss": 3.5905640399857583, + "tokens_seen": 1296957440 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030658976930792375, + "loss": 1.305, + "theoretical_loss": 3.5905477243616017, + "tokens_seen": 1297022976 + }, + { + "epoch": 0.39, + "learning_rate": 0.000306579739217653, + "loss": 1.3644, + "theoretical_loss": 3.5905314097926375, + "tokens_seen": 1297088512 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003065697091273821, + "loss": 1.4967, + "theoretical_loss": 3.590515096278745, + "tokens_seen": 1297154048 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030655967903711135, + "loss": 1.336, + "theoretical_loss": 3.5904987838198017, + "tokens_seen": 1297219584 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030654964894684053, + "loss": 1.3796, + "theoretical_loss": 3.5904824724156876, + "tokens_seen": 1297285120 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003065396188565697, + "loss": 1.3601, + "theoretical_loss": 3.5904661620662797, + "tokens_seen": 1297350656 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003065295887662989, + "loss": 1.2401, + "theoretical_loss": 3.5904498527714575, + "tokens_seen": 1297416192 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003065195586760281, + "loss": 1.3236, + "theoretical_loss": 3.590433544531099, + "tokens_seen": 1297481728 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030650952858575725, + "loss": 1.4685, + "theoretical_loss": 3.590417237345083, + "tokens_seen": 1297547264 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 433820, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3087390661239624, + "objective/train/theoretical_loss": 3.5904009312132885, + "objective/train/tokens_used": 1297612800, + "theoretical_loss": 3.5904009312132885, + "tokens_seen": 1297612800 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003064994984954865, + "loss": 1.3057, + "theoretical_loss": 3.5904009312132885, + "tokens_seen": 1297612800 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003064894684052156, + "loss": 1.4843, + "theoretical_loss": 3.5903846261355934, + "tokens_seen": 1297678336 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030647943831494485, + "loss": 1.3186, + "theoretical_loss": 3.5903683221118765, + "tokens_seen": 1297743872 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030646940822467403, + "loss": 1.2322, + "theoretical_loss": 3.5903520191420166, + "tokens_seen": 1297809408 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003064593781344032, + "loss": 1.2469, + "theoretical_loss": 3.5903357172258925, + "tokens_seen": 1297874944 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003064493480441324, + "loss": 1.2537, + "theoretical_loss": 3.590319416363383, + "tokens_seen": 1297940480 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003064393179538616, + "loss": 1.3055, + "theoretical_loss": 3.5903031165543657, + "tokens_seen": 1298006016 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030642928786359076, + "loss": 1.1989, + "theoretical_loss": 3.590286817798721, + "tokens_seen": 1298071552 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030641925777332, + "loss": 1.3101, + "theoretical_loss": 3.5902705200963263, + "tokens_seen": 1298137088 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003064092276830491, + "loss": 1.4598, + "theoretical_loss": 3.5902542234470616, + "tokens_seen": 1298202624 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030639919759277835, + "loss": 1.2659, + "theoretical_loss": 3.5902379278508043, + "tokens_seen": 1298268160 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003063891675025075, + "loss": 1.4166, + "theoretical_loss": 3.590221633307434, + "tokens_seen": 1298333696 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003063791374122367, + "loss": 1.324, + "theoretical_loss": 3.59020533981683, + "tokens_seen": 1298399232 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003063691073219659, + "loss": 1.3421, + "theoretical_loss": 3.59018904737887, + "tokens_seen": 1298464768 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003063590772316951, + "loss": 1.3335, + "theoretical_loss": 3.590172755993433, + "tokens_seen": 1298530304 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030634904714142426, + "loss": 1.3884, + "theoretical_loss": 3.5901564656603995, + "tokens_seen": 1298595840 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003063390170511535, + "loss": 1.2459, + "theoretical_loss": 3.5901401763796468, + "tokens_seen": 1298661376 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003063289869608826, + "loss": 1.2904, + "theoretical_loss": 3.5901238881510538, + "tokens_seen": 1298726912 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030631895687061186, + "loss": 1.4151, + "theoretical_loss": 3.5901076009745, + "tokens_seen": 1298792448 + }, + { + "epoch": 0.39, + "learning_rate": 0.000306308926780341, + "loss": 1.3241, + "theoretical_loss": 3.5900913148498645, + "tokens_seen": 1298857984 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003062988966900702, + "loss": 1.3759, + "theoretical_loss": 3.5900750297770267, + "tokens_seen": 1298923520 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030628886659979945, + "loss": 1.2649, + "theoretical_loss": 3.590058745755864, + "tokens_seen": 1298989056 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003062788365095286, + "loss": 1.3169, + "theoretical_loss": 3.590042462786257, + "tokens_seen": 1299054592 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003062688064192578, + "loss": 1.3496, + "theoretical_loss": 3.590026180868084, + "tokens_seen": 1299120128 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030625877632898694, + "loss": 1.3791, + "theoretical_loss": 3.5900099000012244, + "tokens_seen": 1299185664 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003062487462387162, + "loss": 1.2699, + "theoretical_loss": 3.589993620185557, + "tokens_seen": 1299251200 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030623871614844536, + "loss": 1.3108, + "theoretical_loss": 3.589977341420961, + "tokens_seen": 1299316736 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030622868605817454, + "loss": 1.3048, + "theoretical_loss": 3.5899610637073156, + "tokens_seen": 1299382272 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003062186559679037, + "loss": 1.3417, + "theoretical_loss": 3.5899447870445, + "tokens_seen": 1299447808 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003062086258776329, + "loss": 1.345, + "theoretical_loss": 3.589928511432393, + "tokens_seen": 1299513344 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003061985957873621, + "loss": 1.4236, + "theoretical_loss": 3.5899122368708745, + "tokens_seen": 1299578880 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003061885656970913, + "loss": 1.2925, + "theoretical_loss": 3.589895963359823, + "tokens_seen": 1299644416 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030617853560682045, + "loss": 1.4994, + "theoretical_loss": 3.589879690899118, + "tokens_seen": 1299709952 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003061685055165497, + "loss": 1.4431, + "theoretical_loss": 3.5898634194886387, + "tokens_seen": 1299775488 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030615847542627886, + "loss": 1.3017, + "theoretical_loss": 3.589847149128265, + "tokens_seen": 1299841024 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030614844533600804, + "loss": 1.3352, + "theoretical_loss": 3.5898308798178755, + "tokens_seen": 1299906560 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003061384152457372, + "loss": 1.3236, + "theoretical_loss": 3.5898146115573493, + "tokens_seen": 1299972096 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003061283851554664, + "loss": 1.1947, + "theoretical_loss": 3.5897983443465664, + "tokens_seen": 1300037632 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003061183550651956, + "loss": 1.512, + "theoretical_loss": 3.5897820781854053, + "tokens_seen": 1300103168 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003061083249749248, + "loss": 1.3709, + "theoretical_loss": 3.589765813073746, + "tokens_seen": 1300168704 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030609829488465395, + "loss": 1.2801, + "theoretical_loss": 3.5897495490114686, + "tokens_seen": 1300234240 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003060882647943832, + "loss": 1.2501, + "theoretical_loss": 3.5897332859984505, + "tokens_seen": 1300299776 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003060782347041123, + "loss": 1.228, + "theoretical_loss": 3.5897170240345733, + "tokens_seen": 1300365312 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030606820461384155, + "loss": 1.2786, + "theoretical_loss": 3.589700763119715, + "tokens_seen": 1300430848 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030605817452357073, + "loss": 1.3377, + "theoretical_loss": 3.589684503253755, + "tokens_seen": 1300496384 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003060481444332999, + "loss": 1.1492, + "theoretical_loss": 3.5896682444365746, + "tokens_seen": 1300561920 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003060381143430291, + "loss": 1.2116, + "theoretical_loss": 3.589651986668051, + "tokens_seen": 1300627456 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003060280842527583, + "loss": 1.2617, + "theoretical_loss": 3.5896357299480655, + "tokens_seen": 1300692992 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030601805416248745, + "loss": 1.3252, + "theoretical_loss": 3.5896194742764966, + "tokens_seen": 1300758528 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003060080240722167, + "loss": 1.3155, + "theoretical_loss": 3.5896032196532244, + "tokens_seen": 1300824064 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 435014, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1333084106445312, + "objective/train/theoretical_loss": 3.589586966078128, + "objective/train/tokens_used": 1300889600, + "theoretical_loss": 3.589586966078128, + "tokens_seen": 1300889600 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003059979939819458, + "loss": 1.2691, + "theoretical_loss": 3.589586966078128, + "tokens_seen": 1300889600 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030598796389167505, + "loss": 1.3249, + "theoretical_loss": 3.5895707135510873, + "tokens_seen": 1300955136 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030597793380140423, + "loss": 1.3334, + "theoretical_loss": 3.589554462071982, + "tokens_seen": 1301020672 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003059679037111334, + "loss": 1.2614, + "theoretical_loss": 3.5895382116406918, + "tokens_seen": 1301086208 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003059578736208626, + "loss": 1.3113, + "theoretical_loss": 3.5895219622570957, + "tokens_seen": 1301151744 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003059478435305918, + "loss": 1.2196, + "theoretical_loss": 3.589505713921074, + "tokens_seen": 1301217280 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030593781344032096, + "loss": 1.3026, + "theoretical_loss": 3.5894894666325072, + "tokens_seen": 1301282816 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003059277833500502, + "loss": 1.4026, + "theoretical_loss": 3.5894732203912736, + "tokens_seen": 1301348352 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003059177532597793, + "loss": 1.2572, + "theoretical_loss": 3.5894569751972534, + "tokens_seen": 1301413888 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030590772316950855, + "loss": 1.3602, + "theoretical_loss": 3.5894407310503267, + "tokens_seen": 1301479424 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003058976930792377, + "loss": 1.4652, + "theoretical_loss": 3.5894244879503727, + "tokens_seen": 1301544960 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003058876629889669, + "loss": 1.298, + "theoretical_loss": 3.589408245897272, + "tokens_seen": 1301610496 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003058776328986961, + "loss": 1.2614, + "theoretical_loss": 3.589392004890904, + "tokens_seen": 1301676032 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003058676028084253, + "loss": 1.3273, + "theoretical_loss": 3.589375764931148, + "tokens_seen": 1301741568 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030585757271815446, + "loss": 1.3709, + "theoretical_loss": 3.5893595260178843, + "tokens_seen": 1301807104 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003058475426278837, + "loss": 1.4016, + "theoretical_loss": 3.5893432881509937, + "tokens_seen": 1301872640 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003058375125376128, + "loss": 1.4428, + "theoretical_loss": 3.589327051330355, + "tokens_seen": 1301938176 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030582748244734206, + "loss": 1.3435, + "theoretical_loss": 3.589310815555848, + "tokens_seen": 1302003712 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003058174523570712, + "loss": 1.3237, + "theoretical_loss": 3.589294580827354, + "tokens_seen": 1302069248 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003058074222668004, + "loss": 1.2978, + "theoretical_loss": 3.5892783471447514, + "tokens_seen": 1302134784 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003057973921765296, + "loss": 1.2774, + "theoretical_loss": 3.5892621145079207, + "tokens_seen": 1302200320 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003057873620862588, + "loss": 1.1844, + "theoretical_loss": 3.5892458829167424, + "tokens_seen": 1302265856 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030577733199598796, + "loss": 1.3015, + "theoretical_loss": 3.589229652371096, + "tokens_seen": 1302331392 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030576730190571714, + "loss": 1.2538, + "theoretical_loss": 3.589213422870862, + "tokens_seen": 1302396928 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003057572718154463, + "loss": 1.278, + "theoretical_loss": 3.5891971944159193, + "tokens_seen": 1302462464 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030574724172517556, + "loss": 1.3797, + "theoretical_loss": 3.5891809670061496, + "tokens_seen": 1302528000 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003057372116349047, + "loss": 1.2107, + "theoretical_loss": 3.589164740641432, + "tokens_seen": 1302593536 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003057271815446339, + "loss": 1.3888, + "theoretical_loss": 3.589148515321647, + "tokens_seen": 1302659072 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030571715145436305, + "loss": 1.2363, + "theoretical_loss": 3.589132291046675, + "tokens_seen": 1302724608 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003057071213640923, + "loss": 1.3237, + "theoretical_loss": 3.589116067816395, + "tokens_seen": 1302790144 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030569709127382147, + "loss": 1.3672, + "theoretical_loss": 3.589099845630688, + "tokens_seen": 1302855680 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030568706118355065, + "loss": 1.2943, + "theoretical_loss": 3.589083624489435, + "tokens_seen": 1302921216 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030567703109327983, + "loss": 1.2731, + "theoretical_loss": 3.589067404392515, + "tokens_seen": 1302986752 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030566700100300906, + "loss": 1.3525, + "theoretical_loss": 3.589051185339809, + "tokens_seen": 1303052288 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003056569709127382, + "loss": 1.4073, + "theoretical_loss": 3.5890349673311963, + "tokens_seen": 1303117824 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003056469408224674, + "loss": 1.3301, + "theoretical_loss": 3.5890187503665576, + "tokens_seen": 1303183360 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030563691073219655, + "loss": 1.4596, + "theoretical_loss": 3.589002534445774, + "tokens_seen": 1303248896 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003056268806419258, + "loss": 1.3186, + "theoretical_loss": 3.588986319568725, + "tokens_seen": 1303314432 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030561685055165497, + "loss": 1.2648, + "theoretical_loss": 3.5889701057352914, + "tokens_seen": 1303379968 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030560682046138415, + "loss": 1.36, + "theoretical_loss": 3.5889538929453533, + "tokens_seen": 1303445504 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030559679037111333, + "loss": 1.361, + "theoretical_loss": 3.5889376811987908, + "tokens_seen": 1303511040 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003055867602808425, + "loss": 1.3669, + "theoretical_loss": 3.588921470495485, + "tokens_seen": 1303576576 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003055767301905717, + "loss": 1.3133, + "theoretical_loss": 3.5889052608353156, + "tokens_seen": 1303642112 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030556670010030093, + "loss": 1.2407, + "theoretical_loss": 3.588889052218163, + "tokens_seen": 1303707648 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030555667001003006, + "loss": 1.3245, + "theoretical_loss": 3.588872844643909, + "tokens_seen": 1303773184 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003055466399197593, + "loss": 1.3632, + "theoretical_loss": 3.588856638112432, + "tokens_seen": 1303838720 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003055366098294885, + "loss": 1.3796, + "theoretical_loss": 3.5888404326236145, + "tokens_seen": 1303904256 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030552657973921765, + "loss": 1.3602, + "theoretical_loss": 3.5888242281773355, + "tokens_seen": 1303969792 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003055165496489469, + "loss": 1.3736, + "theoretical_loss": 3.588808024773477, + "tokens_seen": 1304035328 + }, + { + "epoch": 0.4, + "learning_rate": 0.000305506519558676, + "loss": 1.3259, + "theoretical_loss": 3.5887918224119177, + "tokens_seen": 1304100864 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 435857, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4537816047668457, + "objective/train/theoretical_loss": 3.58877562109254, + "objective/train/tokens_used": 1304166400, + "theoretical_loss": 3.58877562109254, + "tokens_seen": 1304166400 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030549648946840525, + "loss": 1.4435, + "theoretical_loss": 3.58877562109254, + "tokens_seen": 1304166400 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030548645937813443, + "loss": 1.2344, + "theoretical_loss": 3.588759420815223, + "tokens_seen": 1304231936 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003054764292878636, + "loss": 1.3426, + "theoretical_loss": 3.5887432215798487, + "tokens_seen": 1304297472 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003054663991975928, + "loss": 1.3554, + "theoretical_loss": 3.588727023386297, + "tokens_seen": 1304363008 + }, + { + "epoch": 0.4, + "learning_rate": 0.000305456369107322, + "loss": 1.3788, + "theoretical_loss": 3.588710826234448, + "tokens_seen": 1304428544 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030544633901705116, + "loss": 1.189, + "theoretical_loss": 3.5886946301241833, + "tokens_seen": 1304494080 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003054363089267804, + "loss": 1.3304, + "theoretical_loss": 3.5886784350553835, + "tokens_seen": 1304559616 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003054262788365095, + "loss": 1.3712, + "theoretical_loss": 3.5886622410279294, + "tokens_seen": 1304625152 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030541624874623875, + "loss": 1.3547, + "theoretical_loss": 3.588646048041701, + "tokens_seen": 1304690688 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003054062186559679, + "loss": 1.3173, + "theoretical_loss": 3.5886298560965795, + "tokens_seen": 1304756224 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003053961885656971, + "loss": 1.3763, + "theoretical_loss": 3.5886136651924456, + "tokens_seen": 1304821760 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003053861584754263, + "loss": 1.2922, + "theoretical_loss": 3.58859747532918, + "tokens_seen": 1304887296 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003053761283851555, + "loss": 1.3526, + "theoretical_loss": 3.588581286506664, + "tokens_seen": 1304952832 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030536609829488466, + "loss": 1.4287, + "theoretical_loss": 3.5885650987247777, + "tokens_seen": 1305018368 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003053560682046139, + "loss": 1.4507, + "theoretical_loss": 3.588548911983403, + "tokens_seen": 1305083904 + }, + { + "epoch": 0.4, + "learning_rate": 0.000305346038114343, + "loss": 1.4477, + "theoretical_loss": 3.58853272628242, + "tokens_seen": 1305149440 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030533600802407226, + "loss": 1.4256, + "theoretical_loss": 3.5885165416217095, + "tokens_seen": 1305214976 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003053259779338014, + "loss": 1.2061, + "theoretical_loss": 3.588500358001153, + "tokens_seen": 1305280512 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003053159478435306, + "loss": 1.2464, + "theoretical_loss": 3.5884841754206303, + "tokens_seen": 1305346048 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003053059177532598, + "loss": 1.3825, + "theoretical_loss": 3.588467993880024, + "tokens_seen": 1305411584 + }, + { + "epoch": 0.4, + "learning_rate": 0.000305295887662989, + "loss": 1.398, + "theoretical_loss": 3.588451813379214, + "tokens_seen": 1305477120 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030528585757271816, + "loss": 1.1921, + "theoretical_loss": 3.588435633918081, + "tokens_seen": 1305542656 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030527582748244734, + "loss": 1.3435, + "theoretical_loss": 3.5884194554965068, + "tokens_seen": 1305608192 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003052657973921765, + "loss": 1.3903, + "theoretical_loss": 3.588403278114372, + "tokens_seen": 1305673728 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030525576730190576, + "loss": 1.3338, + "theoretical_loss": 3.5883871017715583, + "tokens_seen": 1305739264 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003052457372116349, + "loss": 1.3321, + "theoretical_loss": 3.5883709264679458, + "tokens_seen": 1305804800 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003052357071213641, + "loss": 1.3579, + "theoretical_loss": 3.588354752203416, + "tokens_seen": 1305870336 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030522567703109325, + "loss": 1.4211, + "theoretical_loss": 3.5883385789778504, + "tokens_seen": 1305935872 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003052156469408225, + "loss": 1.3059, + "theoretical_loss": 3.5883224067911295, + "tokens_seen": 1306001408 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030520561685055167, + "loss": 1.3625, + "theoretical_loss": 3.5883062356431346, + "tokens_seen": 1306066944 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030519558676028085, + "loss": 1.352, + "theoretical_loss": 3.588290065533747, + "tokens_seen": 1306132480 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030518555667001003, + "loss": 1.3434, + "theoretical_loss": 3.5882738964628484, + "tokens_seen": 1306198016 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030517552657973926, + "loss": 1.3455, + "theoretical_loss": 3.588257728430319, + "tokens_seen": 1306263552 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003051654964894684, + "loss": 1.3134, + "theoretical_loss": 3.588241561436041, + "tokens_seen": 1306329088 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003051554663991976, + "loss": 1.3807, + "theoretical_loss": 3.5882253954798946, + "tokens_seen": 1306394624 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030514543630892675, + "loss": 1.3759, + "theoretical_loss": 3.5882092305617617, + "tokens_seen": 1306460160 + }, + { + "epoch": 0.4, + "learning_rate": 0.000305135406218656, + "loss": 1.3779, + "theoretical_loss": 3.588193066681524, + "tokens_seen": 1306525696 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030512537612838517, + "loss": 1.3751, + "theoretical_loss": 3.588176903839062, + "tokens_seen": 1306591232 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030511534603811435, + "loss": 1.3596, + "theoretical_loss": 3.5881607420342574, + "tokens_seen": 1306656768 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030510531594784353, + "loss": 1.3326, + "theoretical_loss": 3.588144581266991, + "tokens_seen": 1306722304 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003050952858575727, + "loss": 1.2828, + "theoretical_loss": 3.588128421537145, + "tokens_seen": 1306787840 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003050852557673019, + "loss": 1.3548, + "theoretical_loss": 3.5881122628446005, + "tokens_seen": 1306853376 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030507522567703113, + "loss": 1.336, + "theoretical_loss": 3.588096105189239, + "tokens_seen": 1306918912 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030506519558676026, + "loss": 1.4197, + "theoretical_loss": 3.5880799485709414, + "tokens_seen": 1306984448 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003050551654964895, + "loss": 1.2481, + "theoretical_loss": 3.5880637929895896, + "tokens_seen": 1307049984 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003050451354062186, + "loss": 1.3495, + "theoretical_loss": 3.588047638445065, + "tokens_seen": 1307115520 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030503510531594785, + "loss": 1.2848, + "theoretical_loss": 3.5880314849372485, + "tokens_seen": 1307181056 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030502507522567703, + "loss": 1.3379, + "theoretical_loss": 3.588015332466023, + "tokens_seen": 1307246592 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003050150451354062, + "loss": 1.3175, + "theoretical_loss": 3.5879991810312686, + "tokens_seen": 1307312128 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003050050150451354, + "loss": 1.1974, + "theoretical_loss": 3.5879830306328673, + "tokens_seen": 1307377664 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 437174, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1535789966583252, + "objective/train/theoretical_loss": 3.587966881270701, + "objective/train/tokens_used": 1307443200, + "theoretical_loss": 3.587966881270701, + "tokens_seen": 1307443200 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030499498495486463, + "loss": 1.2478, + "theoretical_loss": 3.587966881270701, + "tokens_seen": 1307443200 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030498495486459376, + "loss": 1.3868, + "theoretical_loss": 3.587950732944651, + "tokens_seen": 1307508736 + }, + { + "epoch": 0.4, + "learning_rate": 0.000304974924774323, + "loss": 1.3465, + "theoretical_loss": 3.5879345856545988, + "tokens_seen": 1307574272 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003049648946840521, + "loss": 1.2482, + "theoretical_loss": 3.5879184394004264, + "tokens_seen": 1307639808 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030495486459378136, + "loss": 1.2979, + "theoretical_loss": 3.587902294182015, + "tokens_seen": 1307705344 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030494483450351054, + "loss": 1.3897, + "theoretical_loss": 3.587886149999246, + "tokens_seen": 1307770880 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003049348044132397, + "loss": 1.3445, + "theoretical_loss": 3.587870006852002, + "tokens_seen": 1307836416 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003049247743229689, + "loss": 1.2829, + "theoretical_loss": 3.587853864740164, + "tokens_seen": 1307901952 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003049147442326981, + "loss": 1.3081, + "theoretical_loss": 3.587837723663614, + "tokens_seen": 1307967488 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030490471414242726, + "loss": 1.2974, + "theoretical_loss": 3.587821583622233, + "tokens_seen": 1308033024 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003048946840521565, + "loss": 1.2729, + "theoretical_loss": 3.5878054446159044, + "tokens_seen": 1308098560 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003048846539618856, + "loss": 1.4383, + "theoretical_loss": 3.587789306644509, + "tokens_seen": 1308164096 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030487462387161486, + "loss": 1.2451, + "theoretical_loss": 3.5877731697079276, + "tokens_seen": 1308229632 + }, + { + "epoch": 0.4, + "learning_rate": 0.000304864593781344, + "loss": 1.3496, + "theoretical_loss": 3.587757033806043, + "tokens_seen": 1308295168 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003048545636910732, + "loss": 1.2567, + "theoretical_loss": 3.587740898938738, + "tokens_seen": 1308360704 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003048445336008024, + "loss": 1.314, + "theoretical_loss": 3.5877247651058926, + "tokens_seen": 1308426240 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003048345035105316, + "loss": 1.4598, + "theoretical_loss": 3.5877086323073897, + "tokens_seen": 1308491776 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030482447342026076, + "loss": 1.3175, + "theoretical_loss": 3.587692500543111, + "tokens_seen": 1308557312 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030481444332999, + "loss": 1.3255, + "theoretical_loss": 3.5876763698129386, + "tokens_seen": 1308622848 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003048044132397192, + "loss": 1.3518, + "theoretical_loss": 3.5876602401167537, + "tokens_seen": 1308688384 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030479438314944836, + "loss": 1.2858, + "theoretical_loss": 3.5876441114544395, + "tokens_seen": 1308753920 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030478435305917754, + "loss": 1.2881, + "theoretical_loss": 3.587627983825876, + "tokens_seen": 1308819456 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003047743229689067, + "loss": 1.3541, + "theoretical_loss": 3.587611857230948, + "tokens_seen": 1308884992 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030476429287863596, + "loss": 1.2821, + "theoretical_loss": 3.587595731669535, + "tokens_seen": 1308950528 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003047542627883651, + "loss": 1.3095, + "theoretical_loss": 3.58757960714152, + "tokens_seen": 1309016064 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003047442326980943, + "loss": 1.3468, + "theoretical_loss": 3.587563483646785, + "tokens_seen": 1309081600 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030473420260782345, + "loss": 1.2529, + "theoretical_loss": 3.5875473611852122, + "tokens_seen": 1309147136 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003047241725175527, + "loss": 1.2705, + "theoretical_loss": 3.5875312397566836, + "tokens_seen": 1309212672 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030471414242728187, + "loss": 1.3123, + "theoretical_loss": 3.5875151193610817, + "tokens_seen": 1309278208 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030470411233701105, + "loss": 1.3833, + "theoretical_loss": 3.587498999998287, + "tokens_seen": 1309343744 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030469408224674023, + "loss": 1.2727, + "theoretical_loss": 3.5874828816681834, + "tokens_seen": 1309409280 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030468405215646946, + "loss": 1.3608, + "theoretical_loss": 3.5874667643706526, + "tokens_seen": 1309474816 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003046740220661986, + "loss": 1.3931, + "theoretical_loss": 3.587450648105577, + "tokens_seen": 1309540352 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003046639919759278, + "loss": 1.2361, + "theoretical_loss": 3.5874345328728374, + "tokens_seen": 1309605888 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030465396188565695, + "loss": 1.4424, + "theoretical_loss": 3.5874184186723177, + "tokens_seen": 1309671424 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003046439317953862, + "loss": 1.2984, + "theoretical_loss": 3.5874023055038995, + "tokens_seen": 1309736960 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030463390170511537, + "loss": 1.3313, + "theoretical_loss": 3.5873861933674647, + "tokens_seen": 1309802496 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030462387161484455, + "loss": 1.2376, + "theoretical_loss": 3.5873700822628964, + "tokens_seen": 1309868032 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030461384152457373, + "loss": 1.3783, + "theoretical_loss": 3.5873539721900762, + "tokens_seen": 1309933568 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003046038114343029, + "loss": 1.3653, + "theoretical_loss": 3.5873378631488864, + "tokens_seen": 1309999104 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003045937813440321, + "loss": 1.4211, + "theoretical_loss": 3.58732175513921, + "tokens_seen": 1310064640 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030458375125376133, + "loss": 1.3155, + "theoretical_loss": 3.5873056481609282, + "tokens_seen": 1310130176 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030457372116349046, + "loss": 1.3697, + "theoretical_loss": 3.5872895422139246, + "tokens_seen": 1310195712 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003045636910732197, + "loss": 1.3907, + "theoretical_loss": 3.587273437298081, + "tokens_seen": 1310261248 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003045536609829488, + "loss": 1.3411, + "theoretical_loss": 3.5872573334132802, + "tokens_seen": 1310326784 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030454363089267805, + "loss": 1.3078, + "theoretical_loss": 3.587241230559404, + "tokens_seen": 1310392320 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030453360080240723, + "loss": 1.3269, + "theoretical_loss": 3.5872251287363355, + "tokens_seen": 1310457856 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003045235707121364, + "loss": 1.2602, + "theoretical_loss": 3.587209027943956, + "tokens_seen": 1310523392 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003045135406218656, + "loss": 1.2641, + "theoretical_loss": 3.5871929281821497, + "tokens_seen": 1310588928 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030450351053159483, + "loss": 1.2577, + "theoretical_loss": 3.5871768294507973, + "tokens_seen": 1310654464 + }, + { + "debugging/length": 576.375, + "debugging/num_segments": 8, + "debugging/score": 0.05779437402971244, + "debugging/score_std": 0.03719063426069438, + "epoch": 0.4, + "objective/train/docs_used": 437926, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.244199275970459, + "objective/train/theoretical_loss": 3.5871607317497833, + "objective/train/tokens_used": 1310720000, + "theoretical_loss": 3.5871607317497833, + "tokens_seen": 1310720000 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030449348044132396, + "loss": 1.2613, + "theoretical_loss": 3.5871607317497833, + "tokens_seen": 1310720000 + } + ], + "max_steps": 50354, + "num_train_epochs": 9223372036854775807, + "total_flos": 6.6890760192e+17, + "trial_name": null, + "trial_params": null +}