Training in progress, step 4350

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:badb32843897978102b35b7bbcd3f3463077dd48911a93591713c6836235fdfc
+oid sha256:002a23ea4ec33d68169cf04be99ea953375df47ab38dedc2e7ebb6618e79408a
 size 1140991056

train.log

@@ -8595,3 +8595,128 @@ Time to load cpu_adam op: 2.2494730949401855 seconds
 [Rank 0] Trainer log: {'loss': 0.7007, 'grad_norm': 5.15255069732666, 'learning_rate': 6.292576566887787e-07}[Rank 2] Trainer log: {'loss': 0.7007, 'grad_norm': 5.15255069732666, 'learning_rate': 6.292576566887787e-07}
 
 {'loss': 0.7007, 'grad_norm': 5.15255069732666, 'learning_rate': 6.292576566887787e-07, 'epoch': 0.89}
+[Rank 0] Trainer log: {'loss': 0.7336, 'grad_norm': 8.90782642364502, 'learning_rate': 6.268790568208116e-07}[Rank 1] Trainer log: {'loss': 0.7336, 'grad_norm': 8.90782642364502, 'learning_rate': 6.268790568208116e-07}[Rank 3] Trainer log: {'loss': 0.7336, 'grad_norm': 8.90782642364502, 'learning_rate': 6.268790568208116e-07}
+
+
+[Rank 2] Trainer log: {'loss': 0.7336, 'grad_norm': 8.90782642364502, 'learning_rate': 6.268790568208116e-07}
+{'loss': 0.7336, 'grad_norm': 8.90782642364502, 'learning_rate': 6.268790568208116e-07, 'epoch': 0.89}
+[Rank 3] Trainer log: {'loss': 0.9483, 'grad_norm': 5.516266345977783, 'learning_rate': 6.24504815557967e-07}[Rank 0] Trainer log: {'loss': 0.9483, 'grad_norm': 5.516266345977783, 'learning_rate': 6.24504815557967e-07}[Rank 1] Trainer log: {'loss': 0.9483, 'grad_norm': 5.516266345977783, 'learning_rate': 6.24504815557967e-07}
+
+
+[Rank 2] Trainer log: {'loss': 0.9483, 'grad_norm': 5.516266345977783, 'learning_rate': 6.24504815557967e-07}
+{'loss': 0.9483, 'grad_norm': 5.516266345977783, 'learning_rate': 6.24504815557967e-07, 'epoch': 0.89}
+[Rank 1] Trainer log: {'loss': 0.9115, 'grad_norm': 5.219529628753662, 'learning_rate': 6.221349340042937e-07}[Rank 3] Trainer log: {'loss': 0.9115, 'grad_norm': 5.219529628753662, 'learning_rate': 6.221349340042937e-07}
+[Rank 2] Trainer log: {'loss': 0.9115, 'grad_norm': 5.219529628753662, 'learning_rate': 6.221349340042937e-07}
+
+[Rank 0] Trainer log: {'loss': 0.9115, 'grad_norm': 5.219529628753662, 'learning_rate': 6.221349340042937e-07}
+{'loss': 0.9115, 'grad_norm': 5.219529628753662, 'learning_rate': 6.221349340042937e-07, 'epoch': 0.89}
+[Rank 0] Trainer log: {'loss': 0.799, 'grad_norm': 3.442587375640869, 'learning_rate': 6.197694132618115e-07}[Rank 1] Trainer log: {'loss': 0.799, 'grad_norm': 3.442587375640869, 'learning_rate': 6.197694132618115e-07}
+[Rank 3] Trainer log: {'loss': 0.799, 'grad_norm': 3.442587375640869, 'learning_rate': 6.197694132618115e-07}
+
+[Rank 2] Trainer log: {'loss': 0.799, 'grad_norm': 3.442587375640869, 'learning_rate': 6.197694132618115e-07}
+{'loss': 0.799, 'grad_norm': 3.442587375640869, 'learning_rate': 6.197694132618115e-07, 'epoch': 0.89}
+[Rank 3] Trainer log: {'loss': 0.6708, 'grad_norm': 7.776933193206787, 'learning_rate': 6.174082544305149e-07}[Rank 1] Trainer log: {'loss': 0.6708, 'grad_norm': 7.776933193206787, 'learning_rate': 6.174082544305149e-07}[Rank 2] Trainer log: {'loss': 0.6708, 'grad_norm': 7.776933193206787, 'learning_rate': 6.174082544305149e-07}
+
+
+[Rank 0] Trainer log: {'loss': 0.6708, 'grad_norm': 7.776933193206787, 'learning_rate': 6.174082544305149e-07}
+{'loss': 0.6708, 'grad_norm': 7.776933193206787, 'learning_rate': 6.174082544305149e-07, 'epoch': 0.89}
+[Rank 3] Trainer log: {'loss': 0.8275, 'grad_norm': 4.444672107696533, 'learning_rate': 6.1505145860837e-07}
+[Rank 0] Trainer log: {'loss': 0.8275, 'grad_norm': 4.444672107696533, 'learning_rate': 6.1505145860837e-07}[Rank 2] Trainer log: {'loss': 0.8275, 'grad_norm': 4.444672107696533, 'learning_rate': 6.1505145860837e-07}
+[Rank 1] Trainer log: {'loss': 0.8275, 'grad_norm': 4.444672107696533, 'learning_rate': 6.1505145860837e-07}
+
+{'loss': 0.8275, 'grad_norm': 4.444672107696533, 'learning_rate': 6.1505145860837e-07, 'epoch': 0.89}
+[Rank 3] Trainer log: {'loss': 0.8794, 'grad_norm': 4.909470558166504, 'learning_rate': 6.126990268913091e-07}
+[Rank 1] Trainer log: {'loss': 0.8794, 'grad_norm': 4.909470558166504, 'learning_rate': 6.126990268913091e-07}[Rank 0] Trainer log: {'loss': 0.8794, 'grad_norm': 4.909470558166504, 'learning_rate': 6.126990268913091e-07}
+[Rank 2] Trainer log: {'loss': 0.8794, 'grad_norm': 4.909470558166504, 'learning_rate': 6.126990268913091e-07}
+
+{'loss': 0.8794, 'grad_norm': 4.909470558166504, 'learning_rate': 6.126990268913091e-07, 'epoch': 0.89}
+[Rank 1] Trainer log: {'loss': 0.691, 'grad_norm': 6.546082019805908, 'learning_rate': 6.103509603732416e-07}[Rank 0] Trainer log: {'loss': 0.691, 'grad_norm': 6.546082019805908, 'learning_rate': 6.103509603732416e-07}
+[Rank 3] Trainer log: {'loss': 0.691, 'grad_norm': 6.546082019805908, 'learning_rate': 6.103509603732416e-07}
+
+[Rank 2] Trainer log: {'loss': 0.691, 'grad_norm': 6.546082019805908, 'learning_rate': 6.103509603732416e-07}
+{'loss': 0.691, 'grad_norm': 6.546082019805908, 'learning_rate': 6.103509603732416e-07, 'epoch': 0.89}
+[Rank 3] Trainer log: {'loss': 0.921, 'grad_norm': 5.066746234893799, 'learning_rate': 6.080072601460451e-07}
+[Rank 1] Trainer log: {'loss': 0.921, 'grad_norm': 5.066746234893799, 'learning_rate': 6.080072601460451e-07}
+[Rank 2] Trainer log: {'loss': 0.921, 'grad_norm': 5.066746234893799, 'learning_rate': 6.080072601460451e-07}
+[Rank 0] Trainer log: {'loss': 0.921, 'grad_norm': 5.066746234893799, 'learning_rate': 6.080072601460451e-07}
+{'loss': 0.921, 'grad_norm': 5.066746234893799, 'learning_rate': 6.080072601460451e-07, 'epoch': 0.89}
+[Rank 2] Trainer log: {'loss': 0.7493, 'grad_norm': 9.553936958312988, 'learning_rate': 6.056679272995647e-07}[Rank 3] Trainer log: {'loss': 0.7493, 'grad_norm': 9.553936958312988, 'learning_rate': 6.056679272995647e-07}[Rank 1] Trainer log: {'loss': 0.7493, 'grad_norm': 9.553936958312988, 'learning_rate': 6.056679272995647e-07}
+
+
+[Rank 0] Trainer log: {'loss': 0.7493, 'grad_norm': 9.553936958312988, 'learning_rate': 6.056679272995647e-07}
+{'loss': 0.7493, 'grad_norm': 9.553936958312988, 'learning_rate': 6.056679272995647e-07, 'epoch': 0.89}
+[Rank 0] Trainer log: {'loss': 0.7182, 'grad_norm': 8.36179256439209, 'learning_rate': 6.033329629216189e-07}[Rank 3] Trainer log: {'loss': 0.7182, 'grad_norm': 8.36179256439209, 'learning_rate': 6.033329629216189e-07}[Rank 1] Trainer log: {'loss': 0.7182, 'grad_norm': 8.36179256439209, 'learning_rate': 6.033329629216189e-07}
+
+
+[Rank 2] Trainer log: {'loss': 0.7182, 'grad_norm': 8.36179256439209, 'learning_rate': 6.033329629216189e-07}
+{'loss': 0.7182, 'grad_norm': 8.36179256439209, 'learning_rate': 6.033329629216189e-07, 'epoch': 0.89}
+[Rank 3] Trainer log: {'loss': 0.8551, 'grad_norm': 2.233321189880371, 'learning_rate': 6.010023680979893e-07}[Rank 1] Trainer log: {'loss': 0.8551, 'grad_norm': 2.233321189880371, 'learning_rate': 6.010023680979893e-07}
+[Rank 2] Trainer log: {'loss': 0.8551, 'grad_norm': 2.233321189880371, 'learning_rate': 6.010023680979893e-07}
+
+[Rank 0] Trainer log: {'loss': 0.8551, 'grad_norm': 2.233321189880371, 'learning_rate': 6.010023680979893e-07}
+{'loss': 0.8551, 'grad_norm': 2.233321189880371, 'learning_rate': 6.010023680979893e-07, 'epoch': 0.89}
+[Rank 3] Trainer log: {'loss': 0.9446, 'grad_norm': 5.3710737228393555, 'learning_rate': 5.986761439124289e-07}
+[Rank 2] Trainer log: {'loss': 0.9446, 'grad_norm': 5.3710737228393555, 'learning_rate': 5.986761439124289e-07}
+[Rank 0] Trainer log: {'loss': 0.9446, 'grad_norm': 5.3710737228393555, 'learning_rate': 5.986761439124289e-07}[Rank 1] Trainer log: {'loss': 0.9446, 'grad_norm': 5.3710737228393555, 'learning_rate': 5.986761439124289e-07}
+
+{'loss': 0.9446, 'grad_norm': 5.3710737228393555, 'learning_rate': 5.986761439124289e-07, 'epoch': 0.89}
+[Rank 3] Trainer log: {'loss': 0.8222, 'grad_norm': 5.1239848136901855, 'learning_rate': 5.963542914466569e-07}
+[Rank 0] Trainer log: {'loss': 0.8222, 'grad_norm': 5.1239848136901855, 'learning_rate': 5.963542914466569e-07}[Rank 1] Trainer log: {'loss': 0.8222, 'grad_norm': 5.1239848136901855, 'learning_rate': 5.963542914466569e-07}
+
+[Rank 2] Trainer log: {'loss': 0.8222, 'grad_norm': 5.1239848136901855, 'learning_rate': 5.963542914466569e-07}
+{'loss': 0.8222, 'grad_norm': 5.1239848136901855, 'learning_rate': 5.963542914466569e-07, 'epoch': 0.89}
+[Rank 0] Trainer log: {'loss': 0.9728, 'grad_norm': 2.581252336502075, 'learning_rate': 5.94036811780363e-07}[Rank 3] Trainer log: {'loss': 0.9728, 'grad_norm': 2.581252336502075, 'learning_rate': 5.94036811780363e-07}[Rank 1] Trainer log: {'loss': 0.9728, 'grad_norm': 2.581252336502075, 'learning_rate': 5.94036811780363e-07}
+
+[Rank 2] Trainer log: {'loss': 0.9728, 'grad_norm': 2.581252336502075, 'learning_rate': 5.94036811780363e-07}
+
+{'loss': 0.9728, 'grad_norm': 2.581252336502075, 'learning_rate': 5.94036811780363e-07, 'epoch': 0.9}
+[Rank 1] Trainer log: {'loss': 0.8425, 'grad_norm': 2.5363917350769043, 'learning_rate': 5.917237059911963e-07}[Rank 3] Trainer log: {'loss': 0.8425, 'grad_norm': 2.5363917350769043, 'learning_rate': 5.917237059911963e-07}[Rank 0] Trainer log: {'loss': 0.8425, 'grad_norm': 2.5363917350769043, 'learning_rate': 5.917237059911963e-07}
+
+
+[Rank 2] Trainer log: {'loss': 0.8425, 'grad_norm': 2.5363917350769043, 'learning_rate': 5.917237059911963e-07}
+{'loss': 0.8425, 'grad_norm': 2.5363917350769043, 'learning_rate': 5.917237059911963e-07, 'epoch': 0.9}
+[Rank 3] Trainer log: {'loss': 0.6119, 'grad_norm': 8.582449913024902, 'learning_rate': 5.894149751547806e-07}[Rank 1] Trainer log: {'loss': 0.6119, 'grad_norm': 8.582449913024902, 'learning_rate': 5.894149751547806e-07}
+
+[Rank 0] Trainer log: {'loss': 0.6119, 'grad_norm': 8.582449913024902, 'learning_rate': 5.894149751547806e-07}[Rank 2] Trainer log: {'loss': 0.6119, 'grad_norm': 8.582449913024902, 'learning_rate': 5.894149751547806e-07}
+
+{'loss': 0.6119, 'grad_norm': 8.582449913024902, 'learning_rate': 5.894149751547806e-07, 'epoch': 0.9}
+[Rank 3] Trainer log: {'loss': 0.9873, 'grad_norm': 9.198247909545898, 'learning_rate': 5.871106203447019e-07}[Rank 2] Trainer log: {'loss': 0.9873, 'grad_norm': 9.198247909545898, 'learning_rate': 5.871106203447019e-07}
+
+[Rank 0] Trainer log: {'loss': 0.9873, 'grad_norm': 9.198247909545898, 'learning_rate': 5.871106203447019e-07}[Rank 1] Trainer log: {'loss': 0.9873, 'grad_norm': 9.198247909545898, 'learning_rate': 5.871106203447019e-07}
+
+{'loss': 0.9873, 'grad_norm': 9.198247909545898, 'learning_rate': 5.871106203447019e-07, 'epoch': 0.9}
+[Rank 1] Trainer log: {'loss': 0.8318, 'grad_norm': 4.501786231994629, 'learning_rate': 5.848106426325095e-07}[Rank 3] Trainer log: {'loss': 0.8318, 'grad_norm': 4.501786231994629, 'learning_rate': 5.848106426325095e-07}
+[Rank 2] Trainer log: {'loss': 0.8318, 'grad_norm': 4.501786231994629, 'learning_rate': 5.848106426325095e-07}
+
+[Rank 0] Trainer log: {'loss': 0.8318, 'grad_norm': 4.501786231994629, 'learning_rate': 5.848106426325095e-07}
+{'loss': 0.8318, 'grad_norm': 4.501786231994629, 'learning_rate': 5.848106426325095e-07, 'epoch': 0.9}
+[Rank 3] Trainer log: {'loss': 0.7579, 'grad_norm': 4.943948268890381, 'learning_rate': 5.825150430877157e-07}[Rank 0] Trainer log: {'loss': 0.7579, 'grad_norm': 4.943948268890381, 'learning_rate': 5.825150430877157e-07}[Rank 1] Trainer log: {'loss': 0.7579, 'grad_norm': 4.943948268890381, 'learning_rate': 5.825150430877157e-07}
+
+
+[Rank 2] Trainer log: {'loss': 0.7579, 'grad_norm': 4.943948268890381, 'learning_rate': 5.825150430877157e-07}
+{'loss': 0.7579, 'grad_norm': 4.943948268890381, 'learning_rate': 5.825150430877157e-07, 'epoch': 0.9}
+[Rank 2] Trainer log: {'loss': 1.0215, 'grad_norm': 2.7522549629211426, 'learning_rate': 5.802238227778045e-07}[Rank 3] Trainer log: {'loss': 1.0215, 'grad_norm': 2.7522549629211426, 'learning_rate': 5.802238227778045e-07}
+
+[Rank 1] Trainer log: {'loss': 1.0215, 'grad_norm': 2.7522549629211426, 'learning_rate': 5.802238227778045e-07}
+[Rank 0] Trainer log: {'loss': 1.0215, 'grad_norm': 2.7522549629211426, 'learning_rate': 5.802238227778045e-07}
+{'loss': 1.0215, 'grad_norm': 2.7522549629211426, 'learning_rate': 5.802238227778045e-07, 'epoch': 0.9}
+[Rank 0] Trainer log: {'loss': 0.738, 'grad_norm': 4.653674125671387, 'learning_rate': 5.779369827682158e-07}[Rank 1] Trainer log: {'loss': 0.738, 'grad_norm': 4.653674125671387, 'learning_rate': 5.779369827682158e-07}
+[Rank 3] Trainer log: {'loss': 0.738, 'grad_norm': 4.653674125671387, 'learning_rate': 5.779369827682158e-07}
+
+[Rank 2] Trainer log: {'loss': 0.738, 'grad_norm': 4.653674125671387, 'learning_rate': 5.779369827682158e-07}
+{'loss': 0.738, 'grad_norm': 4.653674125671387, 'learning_rate': 5.779369827682158e-07, 'epoch': 0.9}
+[Rank 3] Trainer log: {'loss': 0.7365, 'grad_norm': 4.448984622955322, 'learning_rate': 5.756545241223554e-07}[Rank 0] Trainer log: {'loss': 0.7365, 'grad_norm': 4.448984622955322, 'learning_rate': 5.756545241223554e-07}
+
+[Rank 2] Trainer log: {'loss': 0.7365, 'grad_norm': 4.448984622955322, 'learning_rate': 5.756545241223554e-07}
+[Rank 1] Trainer log: {'loss': 0.7365, 'grad_norm': 4.448984622955322, 'learning_rate': 5.756545241223554e-07}
+{'loss': 0.7365, 'grad_norm': 4.448984622955322, 'learning_rate': 5.756545241223554e-07, 'epoch': 0.9}
+[Rank 2] Trainer log: {'loss': 0.9334, 'grad_norm': 3.2630815505981445, 'learning_rate': 5.733764479015935e-07}
+[Rank 0] Trainer log: {'loss': 0.9334, 'grad_norm': 3.2630815505981445, 'learning_rate': 5.733764479015935e-07}
+[Rank 1] Trainer log: {'loss': 0.9334, 'grad_norm': 3.2630815505981445, 'learning_rate': 5.733764479015935e-07}
+[Rank 3] Trainer log: {'loss': 0.9334, 'grad_norm': 3.2630815505981445, 'learning_rate': 5.733764479015935e-07}
+{'loss': 0.9334, 'grad_norm': 3.2630815505981445, 'learning_rate': 5.733764479015935e-07, 'epoch': 0.9}
+[Rank 2] Trainer log: {'loss': 0.6217, 'grad_norm': 7.506957530975342, 'learning_rate': 5.711027551652593e-07}
+[Rank 3] Trainer log: {'loss': 0.6217, 'grad_norm': 7.506957530975342, 'learning_rate': 5.711027551652593e-07}[Rank 0] Trainer log: {'loss': 0.6217, 'grad_norm': 7.506957530975342, 'learning_rate': 5.711027551652593e-07}
+
+[Rank 1] Trainer log: {'loss': 0.6217, 'grad_norm': 7.506957530975342, 'learning_rate': 5.711027551652593e-07}
+{'loss': 0.6217, 'grad_norm': 7.506957530975342, 'learning_rate': 5.711027551652593e-07, 'epoch': 0.9}