Training in progress, step 2050

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:208a41e4cc54a86bdd53676765ee523c73e15be2398cb334acf3c3c1c346f887
+oid sha256:29a81378ab1f78803e81097b916a079f8e851fd7ab413fde0729ae1de1e9207a
 size 1204780872

train.log

@@ -17028,3 +17028,388 @@ tensor(-0.0011, device='cuda:0', grad_fn=<AddBackward0>) tensor(-0.0011, device=
 {'train/tv_loss': None, 'train/lm_loss': 0.32293994426727296, 'train/info_loss': 0.12928418815135956, 'train/ref_loss': None, 'train/uncertainty_loss': -0.00011226574424654246, 'train/video_loss': 0.1291719228029251, 'train/total_loss': 0.4521118998527527}
 tensor(-0.0012, device='cuda:1', grad_fn=<AddBackward0>) tensor(-0.0012, device='cuda:1', grad_fn=<MulBackward0>)
 tensor(0.2072, device='cuda:2', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:2', grad_fn=<MulBackward0>)
+[Rank 0] Trainer log: {'loss': 0.4444, 'grad_norm': 12.160444259643555, 'learning_rate': 5.866149411468177e-06}[Rank 3] Trainer log: {'loss': 0.4444, 'grad_norm': 12.160444259643555, 'learning_rate': 5.866149411468177e-06}[Rank 1] Trainer log: {'loss': 0.4444, 'grad_norm': 12.160444259643555, 'learning_rate': 5.866149411468177e-06}
+
+
+[Rank 2] Trainer log: {'loss': 0.4444, 'grad_norm': 12.160444259643555, 'learning_rate': 5.866149411468177e-06}
+{'loss': 0.4444, 'grad_norm': 12.160444259643555, 'learning_rate': 5.866149411468177e-06, 'epoch': 0.65}
+tensor(-0.0014, device='cuda:1', grad_fn=<AddBackward0>) tensor(-0.0014, device='cuda:1', grad_fn=<MulBackward0>)
+tensor(-0.0010, device='cuda:3', grad_fn=<AddBackward0>) tensor(-0.0010, device='cuda:3', grad_fn=<MulBackward0>)
+tensor(-0.0008, device='cuda:2', grad_fn=<AddBackward0>) tensor(-0.0008, device='cuda:2', grad_fn=<MulBackward0>)
+tensor(-0.0007, device='cuda:0', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:0', grad_fn=<MulBackward0>)
+{'train/tv_loss': 0.00020880883093923332, 'train/lm_loss': 2.4817834491841496e-05, 'train/info_loss': 1.8834713046089746e-05, 'train/ref_loss': 0.18321365118026733, 'train/uncertainty_loss': -6.815286469645798e-05, 'train/video_loss': 0.1848347932100296, 'train/total_loss': 0.18485960364341736}
+tensor(-0.0012, device='cuda:3', grad_fn=<AddBackward0>) tensor(-0.0012, device='cuda:3', grad_fn=<MulBackward0>)
+tensor(-0.0010, device='cuda:0', grad_fn=<AddBackward0>) tensor(-0.0010, device='cuda:0', grad_fn=<MulBackward0>)
+{'train/tv_loss': None, 'train/lm_loss': 0.2403105974197388, 'train/info_loss': 0.2913327217102051, 'train/ref_loss': None, 'train/uncertainty_loss': -0.00010404838249087334, 'train/video_loss': 0.29122868180274963, 'train/total_loss': 0.531539261341095}
+tensor(0.0252, device='cuda:1', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:1', grad_fn=<MulBackward0>)
+tensor(0.0596, device='cuda:2', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:2', grad_fn=<MulBackward0>)
+[Rank 3] Trainer log: {'loss': 0.3638, 'grad_norm': 1.8527789115905762, 'learning_rate': 5.8564383629784175e-06}[Rank 1] Trainer log: {'loss': 0.3638, 'grad_norm': 1.8527789115905762, 'learning_rate': 5.8564383629784175e-06}
+
+[Rank 0] Trainer log: {'loss': 0.3638, 'grad_norm': 1.8527789115905762, 'learning_rate': 5.8564383629784175e-06}
+[Rank 2] Trainer log: {'loss': 0.3638, 'grad_norm': 1.8527789115905762, 'learning_rate': 5.8564383629784175e-06}
+{'loss': 0.3638, 'grad_norm': 1.8527789115905762, 'learning_rate': 5.8564383629784175e-06, 'epoch': 0.65}
+tensor(-0.0013, device='cuda:0', grad_fn=<AddBackward0>) tensor(-0.0013, device='cuda:0', grad_fn=<MulBackward0>)
+{'train/tv_loss': None, 'train/lm_loss': 0.38158850669860844, 'train/info_loss': 0.06055440008640289, 'train/ref_loss': None, 'train/uncertainty_loss': -0.0001311479019932449, 'train/video_loss': 0.06042325124144554, 'train/total_loss': 0.4420117735862732}
+tensor(-0.0014, device='cuda:3', grad_fn=<AddBackward0>) tensor(-0.0014, device='cuda:3', grad_fn=<MulBackward0>)
+tensor(-0.0012, device='cuda:1', grad_fn=<AddBackward0>) tensor(-0.0012, device='cuda:1', grad_fn=<MulBackward0>)
+tensor(-0.0007, device='cuda:2', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:2', grad_fn=<MulBackward0>)
+tensor(-0.0010, device='cuda:1', grad_fn=<AddBackward0>) tensor(-0.0010, device='cuda:1', grad_fn=<MulBackward0>)
+tensor(-0.0011, device='cuda:3', grad_fn=<AddBackward0>) tensor(-0.0011, device='cuda:3', grad_fn=<MulBackward0>)
+tensor(-0.0012, device='cuda:0', grad_fn=<AddBackward0>) tensor(-0.0012, device='cuda:0', grad_fn=<MulBackward0>)
+{'train/tv_loss': None, 'train/lm_loss': 0.2511230230331421, 'train/info_loss': 0.2582288980484009, 'train/ref_loss': None, 'train/uncertainty_loss': -0.00012226408580318093, 'train/video_loss': 0.25810661911964417, 'train/total_loss': 0.5092296600341797}
+tensor(-0.0008, device='cuda:2', grad_fn=<AddBackward0>) tensor(-0.0008, device='cuda:2', grad_fn=<MulBackward0>)
+[Rank 3] Trainer log: {'loss': 0.3692, 'grad_norm': 2.5602004528045654, 'learning_rate': 5.846732029718962e-06}[Rank 0] Trainer log: {'loss': 0.3692, 'grad_norm': 2.5602004528045654, 'learning_rate': 5.846732029718962e-06}
+[Rank 1] Trainer log: {'loss': 0.3692, 'grad_norm': 2.5602004528045654, 'learning_rate': 5.846732029718962e-06}
+
+[Rank 2] Trainer log: {'loss': 0.3692, 'grad_norm': 2.5602004528045654, 'learning_rate': 5.846732029718962e-06}
+{'loss': 0.3692, 'grad_norm': 2.5602004528045654, 'learning_rate': 5.846732029718962e-06, 'epoch': 0.65}
+tensor(-0.0013, device='cuda:1', grad_fn=<AddBackward0>) tensor(-0.0013, device='cuda:1', grad_fn=<MulBackward0>)
+tensor(-0.0013, device='cuda:0', grad_fn=<AddBackward0>) tensor(-0.0013, device='cuda:0', grad_fn=<MulBackward0>)
+{'train/tv_loss': None, 'train/lm_loss': 0.3681243896484375, 'train/info_loss': 0.07267985492944717, 'train/ref_loss': None, 'train/uncertainty_loss': -0.00012856742832809687, 'train/video_loss': 0.07255128771066666, 'train/total_loss': 0.44067567586898804}
+tensor(0.2176, device='cuda:3', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:3', grad_fn=<MulBackward0>)
+tensor(0.0752, device='cuda:2', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:2', grad_fn=<MulBackward0>)
+tensor(-0.0010, device='cuda:0', grad_fn=<AddBackward0>) tensor(-0.0010, device='cuda:0', grad_fn=<MulBackward0>)
+{'train/tv_loss': None, 'train/lm_loss': 0.29500226974487304, 'train/info_loss': 0.15204651653766632, 'train/ref_loss': None, 'train/uncertainty_loss': -0.00010267778998240829, 'train/video_loss': 0.15194383263587952, 'train/total_loss': 0.4469461143016815}
+tensor(-0.0011, device='cuda:1', grad_fn=<AddBackward0>) tensor(-0.0011, device='cuda:1', grad_fn=<MulBackward0>)
+tensor(-0.0007, device='cuda:3', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:3', grad_fn=<MulBackward0>)
+tensor(-0.0007, device='cuda:2', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:2', grad_fn=<MulBackward0>)
+[Rank 0] Trainer log: {'loss': 0.3501, 'grad_norm': 5.391907691955566, 'learning_rate': 5.837030422735281e-06}[Rank 2] Trainer log: {'loss': 0.3501, 'grad_norm': 5.391907691955566, 'learning_rate': 5.837030422735281e-06}[Rank 1] Trainer log: {'loss': 0.3501, 'grad_norm': 5.391907691955566, 'learning_rate': 5.837030422735281e-06}
+
+
+[Rank 3] Trainer log: {'loss': 0.3501, 'grad_norm': 5.391907691955566, 'learning_rate': 5.837030422735281e-06}
+{'loss': 0.3501, 'grad_norm': 5.391907691955566, 'learning_rate': 5.837030422735281e-06, 'epoch': 0.65}
+tensor(-0.0011, device='cuda:3', grad_fn=<AddBackward0>) tensor(-0.0011, device='cuda:3', grad_fn=<MulBackward0>)
+tensor(-0.0011, device='cuda:0', grad_fn=<AddBackward0>) tensor(-0.0011, device='cuda:0', grad_fn=<MulBackward0>)
+{'train/tv_loss': None, 'train/lm_loss': 0.27378618717193604, 'train/info_loss': 0.18530026078224182, 'train/ref_loss': None, 'train/uncertainty_loss': -0.0001140484819188714, 'train/video_loss': 0.1851862072944641, 'train/total_loss': 0.45897239446640015}
+tensor(-0.0007, device='cuda:2', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:2', grad_fn=<MulBackward0>)
+tensor(-0.0007, device='cuda:1', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:1', grad_fn=<MulBackward0>)
+tensor(-0.0011, device='cuda:1', grad_fn=<AddBackward0>) tensor(-0.0011, device='cuda:1', grad_fn=<MulBackward0>)
+tensor(0.2328, device='cuda:2', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:2', grad_fn=<MulBackward0>)
+tensor(0.1607, device='cuda:0', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:0', grad_fn=<MulBackward0>)
+{'train/tv_loss': 0.00021110486704856157, 'train/lm_loss': 0.00011288317618891598, 'train/info_loss': 2.6761768822325394e-05, 'train/ref_loss': 0.3208291828632355, 'train/uncertainty_loss': 0.016073283553123475, 'train/video_loss': 0.33861806988716125, 'train/total_loss': 0.33873096108436584}
+tensor(0.0314, device='cuda:3', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:3', grad_fn=<MulBackward0>)
+[Rank 0] Trainer log: {'loss': 0.3379, 'grad_norm': 7.8567070960998535, 'learning_rate': 5.827333553067473e-06}[Rank 1] Trainer log: {'loss': 0.3379, 'grad_norm': 7.8567070960998535, 'learning_rate': 5.827333553067473e-06}
+
+[Rank 2] Trainer log: {'loss': 0.3379, 'grad_norm': 7.8567070960998535, 'learning_rate': 5.827333553067473e-06}
+[Rank 3] Trainer log: {'loss': 0.3379, 'grad_norm': 7.8567070960998535, 'learning_rate': 5.827333553067473e-06}
+{'loss': 0.3379, 'grad_norm': 7.8567070960998535, 'learning_rate': 5.827333553067473e-06, 'epoch': 0.66}
+tensor(-0.0012, device='cuda:1', grad_fn=<AddBackward0>) tensor(-0.0012, device='cuda:1', grad_fn=<MulBackward0>)
+tensor(-0.0010, device='cuda:2', grad_fn=<AddBackward0>) tensor(-0.0010, device='cuda:2', grad_fn=<MulBackward0>)
+tensor(0.0031, device='cuda:0', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:0', grad_fn=<MulBackward0>)
+{'train/tv_loss': 0.00014839788200333716, 'train/lm_loss': 2.1766431746073068e-05, 'train/info_loss': 1.794067611626815e-05, 'train/ref_loss': 0.21624045073986053, 'train/uncertainty_loss': 0.0003052026499062777, 'train/video_loss': 0.21775078773498535, 'train/total_loss': 0.21777255833148956}
+tensor(-0.0007, device='cuda:3', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:3', grad_fn=<MulBackward0>)
+tensor(-0.0012, device='cuda:3', grad_fn=<AddBackward0>) tensor(-0.0012, device='cuda:3', grad_fn=<MulBackward0>)
+tensor(-0.0007, device='cuda:1', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:1', grad_fn=<MulBackward0>)
+tensor(0.0286, device='cuda:2', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:2', grad_fn=<MulBackward0>)
+tensor(-0.0007, device='cuda:0', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:0', grad_fn=<MulBackward0>)
+{'train/tv_loss': 0.00024025768507272006, 'train/lm_loss': 3.1874023261480036e-05, 'train/info_loss': 1.9728748156921938e-05, 'train/ref_loss': 0.1827160120010376, 'train/uncertainty_loss': -6.815321394242347e-05, 'train/video_loss': 0.18458963930606842, 'train/total_loss': 0.18462151288986206}
+[Rank 1] Trainer log: {'loss': 0.2946, 'grad_norm': 7.998960971832275, 'learning_rate': 5.817641431750234e-06}
+[Rank 3] Trainer log: {'loss': 0.2946, 'grad_norm': 7.998960971832275, 'learning_rate': 5.817641431750234e-06}
+[Rank 0] Trainer log: {'loss': 0.2946, 'grad_norm': 7.998960971832275, 'learning_rate': 5.817641431750234e-06}[Rank 2] Trainer log: {'loss': 0.2946, 'grad_norm': 7.998960971832275, 'learning_rate': 5.817641431750234e-06}
+
+{'loss': 0.2946, 'grad_norm': 7.998960971832275, 'learning_rate': 5.817641431750234e-06, 'epoch': 0.66}
+tensor(-0.0009, device='cuda:3', grad_fn=<AddBackward0>) tensor(-0.0009, device='cuda:3', grad_fn=<MulBackward0>)
+tensor(-0.0007, device='cuda:1', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:1', grad_fn=<MulBackward0>)
+tensor(-0.0007, device='cuda:2', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:2', grad_fn=<MulBackward0>)
+tensor(0.0528, device='cuda:0', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:0', grad_fn=<MulBackward0>)
+{'train/tv_loss': 0.00027445454616099597, 'train/lm_loss': 2.1790270693600178e-05, 'train/info_loss': 1.6867830709088594e-05, 'train/ref_loss': 0.250593900680542, 'train/uncertainty_loss': 0.0052795026451349265, 'train/video_loss': 0.2580859065055847, 'train/total_loss': 0.2581076920032501}
+tensor(-0.0013, device='cuda:1', grad_fn=<AddBackward0>) tensor(-0.0013, device='cuda:1', grad_fn=<MulBackward0>)
+tensor(-0.0012, device='cuda:0', grad_fn=<AddBackward0>) tensor(-0.0012, device='cuda:0', grad_fn=<MulBackward0>)
+{'train/tv_loss': None, 'train/lm_loss': 0.21145300865173342, 'train/info_loss': 0.3242379426956177, 'train/ref_loss': None, 'train/uncertainty_loss': -0.00012023800518363715, 'train/video_loss': 0.3241176903247833, 'train/total_loss': 0.5355706810951233}
+tensor(-0.0009, device='cuda:2', grad_fn=<AddBackward0>) tensor(-0.0009, device='cuda:2', grad_fn=<MulBackward0>)
+tensor(0.0132, device='cuda:3', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:3', grad_fn=<MulBackward0>)
+[Rank 2] Trainer log: {'loss': 0.3252, 'grad_norm': 10.534229278564453, 'learning_rate': 5.807954069812862e-06}[Rank 1] Trainer log: {'loss': 0.3252, 'grad_norm': 10.534229278564453, 'learning_rate': 5.807954069812862e-06}
+[Rank 3] Trainer log: {'loss': 0.3252, 'grad_norm': 10.534229278564453, 'learning_rate': 5.807954069812862e-06}
+
+[Rank 0] Trainer log: {'loss': 0.3252, 'grad_norm': 10.534229278564453, 'learning_rate': 5.807954069812862e-06}
+{'loss': 0.3252, 'grad_norm': 10.534229278564453, 'learning_rate': 5.807954069812862e-06, 'epoch': 0.66}
+tensor(-0.0012, device='cuda:2', grad_fn=<AddBackward0>) tensor(-0.0012, device='cuda:2', grad_fn=<MulBackward0>)
+tensor(-0.0013, device='cuda:3', grad_fn=<AddBackward0>) tensor(-0.0013, device='cuda:3', grad_fn=<MulBackward0>)
+tensor(-0.0007, device='cuda:1', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:1', grad_fn=<MulBackward0>)
+tensor(0.0294, device='cuda:0', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:0', grad_fn=<MulBackward0>)
+{'train/tv_loss': 0.00017149079358205201, 'train/lm_loss': 4.7129800077527764e-05, 'train/info_loss': 2.455650974297896e-05, 'train/ref_loss': 0.2356904149055481, 'train/uncertainty_loss': 0.0029447738081216815, 'train/video_loss': 0.2400316596031189, 'train/total_loss': 0.24007879197597504}
+tensor(-0.0011, device='cuda:2', grad_fn=<AddBackward0>) tensor(-0.0011, device='cuda:2', grad_fn=<MulBackward0>)
+tensor(-0.0010, device='cuda:3', grad_fn=<AddBackward0>) tensor(-0.0010, device='cuda:3', grad_fn=<MulBackward0>)
+tensor(-0.0009, device='cuda:0', grad_fn=<AddBackward0>) tensor(-0.0009, device='cuda:0', grad_fn=<MulBackward0>)
+{'train/tv_loss': None, 'train/lm_loss': 0.10311661958694458, 'train/info_loss': 0.17362730205059052, 'train/ref_loss': None, 'train/uncertainty_loss': -9.315699571743608e-05, 'train/video_loss': 0.17353413999080658, 'train/total_loss': 0.2766507565975189}
+tensor(0.2431, device='cuda:1', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:1', grad_fn=<MulBackward0>)
+[Rank 0] Trainer log: {'loss': 0.3809, 'grad_norm': 4.766957759857178, 'learning_rate': 5.798271478279253e-06}[Rank 1] Trainer log: {'loss': 0.3809, 'grad_norm': 4.766957759857178, 'learning_rate': 5.798271478279253e-06}[Rank 3] Trainer log: {'loss': 0.3809, 'grad_norm': 4.766957759857178, 'learning_rate': 5.798271478279253e-06}
+
+[Rank 2] Trainer log: {'loss': 0.3809, 'grad_norm': 4.766957759857178, 'learning_rate': 5.798271478279253e-06}
+
+{'loss': 0.3809, 'grad_norm': 4.766957759857178, 'learning_rate': 5.798271478279253e-06, 'epoch': 0.66}
+tensor(-0.0014, device='cuda:2', grad_fn=<AddBackward0>) tensor(-0.0014, device='cuda:2', grad_fn=<MulBackward0>)
+tensor(-0.0011, device='cuda:3', grad_fn=<AddBackward0>) tensor(-0.0011, device='cuda:3', grad_fn=<MulBackward0>)
+tensor(0.1595, device='cuda:1', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:1', grad_fn=<MulBackward0>)
+tensor(-0.0007, device='cuda:0', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:0', grad_fn=<MulBackward0>)
+{'train/tv_loss': 0.00022061308845877647, 'train/lm_loss': 3.185018722433597e-05, 'train/info_loss': 1.9430735846981406e-05, 'train/ref_loss': 0.18868407607078552, 'train/uncertainty_loss': -6.92706322297454e-05, 'train/video_loss': 0.1903991401195526, 'train/total_loss': 0.19043098390102386}
+tensor(-0.0011, device='cuda:0', grad_fn=<AddBackward0>) tensor(-0.0011, device='cuda:0', grad_fn=<MulBackward0>)
+{'train/tv_loss': None, 'train/lm_loss': 0.29790046215057375, 'train/info_loss': 0.1837206333875656, 'train/ref_loss': None, 'train/uncertainty_loss': -0.00011250831885263325, 'train/video_loss': 0.18360812962055206, 'train/total_loss': 0.48150861263275146}
+tensor(-0.0009, device='cuda:2', grad_fn=<AddBackward0>) tensor(-0.0009, device='cuda:2', grad_fn=<MulBackward0>)
+tensor(-0.0009, device='cuda:3', grad_fn=<AddBackward0>) tensor(-0.0009, device='cuda:3', grad_fn=<MulBackward0>)
+tensor(-0.0009, device='cuda:1', grad_fn=<AddBackward0>) tensor(-0.0009, device='cuda:1', grad_fn=<MulBackward0>)
+[Rank 1] Trainer log: {'loss': 0.3361, 'grad_norm': 4.297579288482666, 'learning_rate': 5.788593668167854e-06}[Rank 0] Trainer log: {'loss': 0.3361, 'grad_norm': 4.297579288482666, 'learning_rate': 5.788593668167854e-06}
+[Rank 3] Trainer log: {'loss': 0.3361, 'grad_norm': 4.297579288482666, 'learning_rate': 5.788593668167854e-06}
+
+[Rank 2] Trainer log: {'loss': 0.3361, 'grad_norm': 4.297579288482666, 'learning_rate': 5.788593668167854e-06}
+{'loss': 0.3361, 'grad_norm': 4.297579288482666, 'learning_rate': 5.788593668167854e-06, 'epoch': 0.66}
+tensor(-0.0014, device='cuda:2', grad_fn=<AddBackward0>) tensor(-0.0014, device='cuda:2', grad_fn=<MulBackward0>)
+tensor(-0.0011, device='cuda:1', grad_fn=<AddBackward0>) tensor(-0.0011, device='cuda:1', grad_fn=<MulBackward0>)
+tensor(0.1702, device='cuda:0', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:0', grad_fn=<MulBackward0>)
+{'train/tv_loss': 0.00018524311017245056, 'train/lm_loss': 3.623634111136198e-05, 'train/info_loss': 2.0980394765501842e-05, 'train/ref_loss': 0.32768750190734863, 'train/uncertainty_loss': 0.017021270096302034, 'train/video_loss': 0.346211701631546, 'train/total_loss': 0.34624794125556946}
+tensor(-0.0007, device='cuda:3', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:3', grad_fn=<MulBackward0>)
+tensor(-0.0012, device='cuda:1', grad_fn=<AddBackward0>) tensor(-0.0012, device='cuda:1', grad_fn=<MulBackward0>)
+tensor(-0.0009, device='cuda:2', grad_fn=<AddBackward0>) tensor(-0.0009, device='cuda:2', grad_fn=<MulBackward0>)
+tensor(0.1074, device='cuda:3', grad_fn=<AddBackward0>) tensor(-0.0008, device='cuda:3', grad_fn=<MulBackward0>)
+tensor(0.0083, device='cuda:0', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:0', grad_fn=<MulBackward0>)
+{'train/tv_loss': 0.00019137355266138912, 'train/lm_loss': 3.5974127240478995e-05, 'train/info_loss': 2.235124156868551e-05, 'train/ref_loss': 0.22056730091571808, 'train/uncertainty_loss': 0.0008338701911270619, 'train/video_loss': 0.22295451164245605, 'train/total_loss': 0.222990483045578}
+[Rank 0] Trainer log: {'loss': 0.3789, 'grad_norm': 8.171895980834961, 'learning_rate': 5.7789206504916815e-06}[Rank 2] Trainer log: {'loss': 0.3789, 'grad_norm': 8.171895980834961, 'learning_rate': 5.7789206504916815e-06}[Rank 3] Trainer log: {'loss': 0.3789, 'grad_norm': 8.171895980834961, 'learning_rate': 5.7789206504916815e-06}
+
+
+{'loss': 0.3789, 'grad_norm': 8.171895980834961, 'learning_rate': 5.7789206504916815e-06, 'epoch': 0.66}[Rank 1] Trainer log: {'loss': 0.3789, 'grad_norm': 8.171895980834961, 'learning_rate': 5.7789206504916815e-06}
+
+tensor(-0.0013, device='cuda:3', grad_fn=<AddBackward0>) tensor(-0.0013, device='cuda:3', grad_fn=<MulBackward0>)
+tensor(-0.0013, device='cuda:1', grad_fn=<AddBackward0>) tensor(-0.0013, device='cuda:1', grad_fn=<MulBackward0>)
+tensor(-0.0007, device='cuda:2', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:2', grad_fn=<MulBackward0>)
+tensor(-0.0007, device='cuda:0', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:0', grad_fn=<MulBackward0>)
+{'train/tv_loss': 0.00026274872943758963, 'train/lm_loss': 3.1874023261480036e-05, 'train/info_loss': 2.0682384274550714e-05, 'train/ref_loss': 0.1149064302444458, 'train/uncertainty_loss': -7.323419558815659e-05, 'train/video_loss': 0.11695586889982224, 'train/total_loss': 0.11698774248361588}
+tensor(-0.0013, device='cuda:2', grad_fn=<AddBackward0>) tensor(-0.0013, device='cuda:2', grad_fn=<MulBackward0>)
+tensor(-0.0010, device='cuda:3', grad_fn=<AddBackward0>) tensor(-0.0010, device='cuda:3', grad_fn=<MulBackward0>)
+tensor(-0.0010, device='cuda:1', grad_fn=<AddBackward0>) tensor(-0.0010, device='cuda:1', grad_fn=<MulBackward0>)
+tensor(0.0592, device='cuda:0', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:0', grad_fn=<MulBackward0>)
+{'train/tv_loss': 0.00016578567447140814, 'train/lm_loss': 6.0000957455486065e-05, 'train/info_loss': 2.574854443082586e-05, 'train/ref_loss': 0.25501570105552673, 'train/uncertainty_loss': 0.0059223812073469165, 'train/video_loss': 0.2622901201248169, 'train/total_loss': 0.2623501121997833}
+[Rank 3] Trainer log: {'loss': 0.3454, 'grad_norm': 3.6991794109344482, 'learning_rate': 5.769252436258295e-06}
+[Rank 1] Trainer log: {'loss': 0.3454, 'grad_norm': 3.6991794109344482, 'learning_rate': 5.769252436258295e-06}[Rank 0] Trainer log: {'loss': 0.3454, 'grad_norm': 3.6991794109344482, 'learning_rate': 5.769252436258295e-06}
+
+[Rank 2] Trainer log: {'loss': 0.3454, 'grad_norm': 3.6991794109344482, 'learning_rate': 5.769252436258295e-06}
+{'loss': 0.3454, 'grad_norm': 3.6991794109344482, 'learning_rate': 5.769252436258295e-06, 'epoch': 0.66}
+tensor(-0.0011, device='cuda:1', grad_fn=<AddBackward0>) tensor(-0.0011, device='cuda:1', grad_fn=<MulBackward0>)
+tensor(-0.0015, device='cuda:2', grad_fn=<AddBackward0>) tensor(-0.0015, device='cuda:2', grad_fn=<MulBackward0>)
+tensor(-0.0007, device='cuda:0', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:0', grad_fn=<MulBackward0>)
+{'train/tv_loss': 0.0003217862220481038, 'train/lm_loss': 4.653389332816005e-05, 'train/info_loss': 2.1338008082238957e-05, 'train/ref_loss': 0.17965355515480042, 'train/uncertainty_loss': -7.330098887905479e-05, 'train/video_loss': 0.1821758896112442, 'train/total_loss': 0.1822224259376526}
+tensor(-0.0014, device='cuda:3', grad_fn=<AddBackward0>) tensor(-0.0014, device='cuda:3', grad_fn=<MulBackward0>)
+tensor(-0.0014, device='cuda:2', grad_fn=<AddBackward0>) tensor(-0.0014, device='cuda:2', grad_fn=<MulBackward0>)
+tensor(-0.0009, device='cuda:1', grad_fn=<AddBackward0>) tensor(-0.0009, device='cuda:1', grad_fn=<MulBackward0>)
+tensor(-0.0007, device='cuda:3', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:3', grad_fn=<MulBackward0>)
+tensor(-0.0009, device='cuda:0', grad_fn=<AddBackward0>) tensor(-0.0009, device='cuda:0', grad_fn=<MulBackward0>)
+{'train/tv_loss': None, 'train/lm_loss': 0.21704933643341065, 'train/info_loss': 0.08072395622730255, 'train/ref_loss': None, 'train/uncertainty_loss': -9.479672298766673e-05, 'train/video_loss': 0.08062916249036789, 'train/total_loss': 0.29767850041389465}
+[Rank 1] Trainer log: {'loss': 0.3814, 'grad_norm': 3.887155294418335, 'learning_rate': 5.759589036469793e-06}[Rank 0] Trainer log: {'loss': 0.3814, 'grad_norm': 3.887155294418335, 'learning_rate': 5.759589036469793e-06}[Rank 3] Trainer log: {'loss': 0.3814, 'grad_norm': 3.887155294418335, 'learning_rate': 5.759589036469793e-06}
+
+
+[Rank 2] Trainer log: {'loss': 0.3814, 'grad_norm': 3.887155294418335, 'learning_rate': 5.759589036469793e-06}
+{'loss': 0.3814, 'grad_norm': 3.887155294418335, 'learning_rate': 5.759589036469793e-06, 'epoch': 0.66}
+tensor(-0.0014, device='cuda:3', grad_fn=<AddBackward0>) tensor(-0.0014, device='cuda:3', grad_fn=<MulBackward0>)
+tensor(-0.0012, device='cuda:2', grad_fn=<AddBackward0>) tensor(-0.0012, device='cuda:2', grad_fn=<MulBackward0>)
+tensor(-0.0009, device='cuda:0', grad_fn=<AddBackward0>) tensor(-0.0009, device='cuda:0', grad_fn=<MulBackward0>)
+{'train/tv_loss': None, 'train/lm_loss': 0.15884593725204468, 'train/info_loss': 0.09679526090621948, 'train/ref_loss': None, 'train/uncertainty_loss': -9.324780548922718e-05, 'train/video_loss': 0.09670200943946838, 'train/total_loss': 0.2555479407310486}
+tensor(-0.0009, device='cuda:1', grad_fn=<AddBackward0>) tensor(-0.0009, device='cuda:1', grad_fn=<MulBackward0>)
+tensor(-0.0013, device='cuda:3', grad_fn=<AddBackward0>) tensor(-0.0013, device='cuda:3', grad_fn=<MulBackward0>)
+tensor(-0.0010, device='cuda:0', grad_fn=<AddBackward0>) tensor(-0.0010, device='cuda:0', grad_fn=<MulBackward0>)
+{'train/tv_loss': None, 'train/lm_loss': 0.13753668069839478, 'train/info_loss': 0.23872123658657074, 'train/ref_loss': None, 'train/uncertainty_loss': -0.0001031742780469358, 'train/video_loss': 0.23861806094646454, 'train/total_loss': 0.37615475058555603}
+tensor(0.1312, device='cuda:2', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:2', grad_fn=<MulBackward0>)
+tensor(-0.0007, device='cuda:1', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:1', grad_fn=<MulBackward0>)
+[Rank 0] Trainer log: {'loss': 0.3451, 'grad_norm': 9.478434562683105, 'learning_rate': 5.749930462122784e-06}[Rank 3] Trainer log: {'loss': 0.3451, 'grad_norm': 9.478434562683105, 'learning_rate': 5.749930462122784e-06}[Rank 2] Trainer log: {'loss': 0.3451, 'grad_norm': 9.478434562683105, 'learning_rate': 5.749930462122784e-06}
+
+[Rank 1] Trainer log: {'loss': 0.3451, 'grad_norm': 9.478434562683105, 'learning_rate': 5.749930462122784e-06}
+
+{'loss': 0.3451, 'grad_norm': 9.478434562683105, 'learning_rate': 5.749930462122784e-06, 'epoch': 0.66}
+tensor(-0.0015, device='cuda:2', grad_fn=<AddBackward0>) tensor(-0.0015, device='cuda:2', grad_fn=<MulBackward0>)
+tensor(-0.0013, device='cuda:0', grad_fn=<AddBackward0>) tensor(-0.0013, device='cuda:0', grad_fn=<MulBackward0>)
+{'train/tv_loss': None, 'train/lm_loss': 0.3149502038955689, 'train/info_loss': 0.1556193083524704, 'train/ref_loss': None, 'train/uncertainty_loss': -0.00012698310893028976, 'train/video_loss': 0.15549232065677643, 'train/total_loss': 0.470442533493042}
+tensor(-0.0010, device='cuda:3', grad_fn=<AddBackward0>) tensor(-0.0010, device='cuda:3', grad_fn=<MulBackward0>)
+tensor(-0.0008, device='cuda:1', grad_fn=<AddBackward0>) tensor(-0.0008, device='cuda:1', grad_fn=<MulBackward0>)
+tensor(-0.0009, device='cuda:2', grad_fn=<AddBackward0>) tensor(-0.0009, device='cuda:2', grad_fn=<MulBackward0>)
+tensor(-0.0009, device='cuda:0', grad_fn=<AddBackward0>) tensor(-0.0009, device='cuda:0', grad_fn=<MulBackward0>)
+{'train/tv_loss': None, 'train/lm_loss': 0.0765468716621399, 'train/info_loss': 0.2542925477027893, 'train/ref_loss': None, 'train/uncertainty_loss': -8.723060018382967e-05, 'train/video_loss': 0.2542053163051605, 'train/total_loss': 0.3307521939277649}
+tensor(-0.0007, device='cuda:1', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:1', grad_fn=<MulBackward0>)
+tensor(0.0120, device='cuda:3', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:3', grad_fn=<MulBackward0>)
+[Rank 0] Trainer log: {'loss': 0.2922, 'grad_norm': 1.9175676107406616, 'learning_rate': 5.740276724208397e-06}[Rank 1] Trainer log: {'loss': 0.2922, 'grad_norm': 1.9175676107406616, 'learning_rate': 5.740276724208397e-06}
+[Rank 2] Trainer log: {'loss': 0.2922, 'grad_norm': 1.9175676107406616, 'learning_rate': 5.740276724208397e-06}
+
+[Rank 3] Trainer log: {'loss': 0.2922, 'grad_norm': 1.9175676107406616, 'learning_rate': 5.740276724208397e-06}
+{'loss': 0.2922, 'grad_norm': 1.9175676107406616, 'learning_rate': 5.740276724208397e-06, 'epoch': 0.66}
+tensor(-0.0012, device='cuda:1', grad_fn=<AddBackward0>) tensor(-0.0012, device='cuda:1', grad_fn=<MulBackward0>)
+tensor(0.1122, device='cuda:2', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:2', grad_fn=<MulBackward0>)
+tensor(0.1959, device='cuda:3', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:3', grad_fn=<MulBackward0>)
+tensor(-0.0007, device='cuda:0', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:0', grad_fn=<MulBackward0>)
+{'train/tv_loss': 0.00016258273972198368, 'train/lm_loss': 6.1025843024253845e-05, 'train/info_loss': 2.413929905742407e-05, 'train/ref_loss': 0.16883832216262817, 'train/uncertainty_loss': -7.159045781008899e-05, 'train/video_loss': 0.17009153962135315, 'train/total_loss': 0.17015255987644196}
+tensor(-0.0014, device='cuda:2', grad_fn=<AddBackward0>) tensor(-0.0014, device='cuda:2', grad_fn=<MulBackward0>)
+tensor(-0.0010, device='cuda:3', grad_fn=<AddBackward0>) tensor(-0.0010, device='cuda:3', grad_fn=<MulBackward0>)
+tensor(0.3660, device='cuda:0', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:0', grad_fn=<MulBackward0>)
+{'train/tv_loss': 0.0002153045265004039, 'train/lm_loss': 2.496086817700416e-05, 'train/info_loss': 1.8238688426208682e-05, 'train/ref_loss': 0.4590832591056824, 'train/uncertainty_loss': 0.03660423159599304, 'train/video_loss': 0.49742814898490906, 'train/total_loss': 0.49745312333106995}
+tensor(0.2555, device='cuda:1', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:1', grad_fn=<MulBackward0>)
+[Rank 3] Trainer log: {'loss': 0.3685, 'grad_norm': 5.5717644691467285, 'learning_rate': 5.7306278337122525e-06}[Rank 2] Trainer log: {'loss': 0.3685, 'grad_norm': 5.5717644691467285, 'learning_rate': 5.7306278337122525e-06}
+[Rank 1] Trainer log: {'loss': 0.3685, 'grad_norm': 5.5717644691467285, 'learning_rate': 5.7306278337122525e-06}
+
+[Rank 0] Trainer log: {'loss': 0.3685, 'grad_norm': 5.5717644691467285, 'learning_rate': 5.7306278337122525e-06}
+{'loss': 0.3685, 'grad_norm': 5.5717644691467285, 'learning_rate': 5.7306278337122525e-06, 'epoch': 0.66}
+tensor(-0.0014, device='cuda:2', grad_fn=<AddBackward0>) tensor(-0.0014, device='cuda:2', grad_fn=<MulBackward0>)
+tensor(-0.0013, device='cuda:3', grad_fn=<AddBackward0>) tensor(-0.0013, device='cuda:3', grad_fn=<MulBackward0>)
+tensor(-0.0009, device='cuda:1', grad_fn=<AddBackward0>) tensor(-0.0009, device='cuda:1', grad_fn=<MulBackward0>)
+tensor(1.1109, device='cuda:0', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:0', grad_fn=<MulBackward0>)
+{'train/tv_loss': 0.0002126151230186224, 'train/lm_loss': 2.4722478701733053e-05, 'train/info_loss': 1.740425250318367e-05, 'train/ref_loss': 0.9555550813674927, 'train/uncertainty_loss': 0.11109327077865601, 'train/video_loss': 1.0683666467666626, 'train/total_loss': 1.0683913230895996}
+tensor(-0.0013, device='cuda:3', grad_fn=<AddBackward0>) tensor(-0.0013, device='cuda:3', grad_fn=<MulBackward0>)
+tensor(-0.0011, device='cuda:1', grad_fn=<AddBackward0>) tensor(-0.0011, device='cuda:1', grad_fn=<MulBackward0>)
+tensor(-0.0012, device='cuda:2', grad_fn=<AddBackward0>) tensor(-0.0012, device='cuda:2', grad_fn=<MulBackward0>)
+tensor(0.0930, device='cuda:0', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:0', grad_fn=<MulBackward0>)
+{'train/tv_loss': 0.00013671087799593806, 'train/lm_loss': 3.657006600406021e-05, 'train/info_loss': 1.9132725356030278e-05, 'train/ref_loss': 0.2791999578475952, 'train/uncertainty_loss': 0.009300475567579269, 'train/video_loss': 0.2896132469177246, 'train/total_loss': 0.2896498143672943}
+[Rank 1] Trainer log: {'loss': 0.4932, 'grad_norm': 7.025022029876709, 'learning_rate': 5.720983801614455e-06}[Rank 0] Trainer log: {'loss': 0.4932, 'grad_norm': 7.025022029876709, 'learning_rate': 5.720983801614455e-06}
+[Rank 3] Trainer log: {'loss': 0.4932, 'grad_norm': 7.025022029876709, 'learning_rate': 5.720983801614455e-06}
+
+[Rank 2] Trainer log: {'loss': 0.4932, 'grad_norm': 7.025022029876709, 'learning_rate': 5.720983801614455e-06}
+{'loss': 0.4932, 'grad_norm': 7.025022029876709, 'learning_rate': 5.720983801614455e-06, 'epoch': 0.66}
+tensor(-0.0012, device='cuda:1', grad_fn=<AddBackward0>) tensor(-0.0012, device='cuda:1', grad_fn=<MulBackward0>)
+tensor(-0.0011, device='cuda:3', grad_fn=<AddBackward0>) tensor(-0.0011, device='cuda:3', grad_fn=<MulBackward0>)
+tensor(-0.0012, device='cuda:0', grad_fn=<AddBackward0>) tensor(-0.0012, device='cuda:0', grad_fn=<MulBackward0>)
+{'train/tv_loss': None, 'train/lm_loss': 0.2787301301956177, 'train/info_loss': 0.2266109138727188, 'train/ref_loss': None, 'train/uncertainty_loss': -0.00011889106826856732, 'train/video_loss': 0.2264920175075531, 'train/total_loss': 0.5052221417427063}
+tensor(-0.0007, device='cuda:2', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:2', grad_fn=<MulBackward0>)
+tensor(-0.0013, device='cuda:2', grad_fn=<AddBackward0>) tensor(-0.0013, device='cuda:2', grad_fn=<MulBackward0>)
+tensor(-0.0007, device='cuda:1', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:1', grad_fn=<MulBackward0>)
+tensor(-0.0007, device='cuda:3', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:3', grad_fn=<MulBackward0>)
+tensor(-0.0011, device='cuda:0', grad_fn=<AddBackward0>) tensor(-0.0011, device='cuda:0', grad_fn=<MulBackward0>)
+{'train/tv_loss': None, 'train/lm_loss': 0.13043994903564454, 'train/info_loss': 0.17577716708183289, 'train/ref_loss': None, 'train/uncertainty_loss': -0.0001127044321037829, 'train/video_loss': 0.17566446959972382, 'train/total_loss': 0.3061044216156006}
+[Rank 1] Trainer log: {'loss': 0.3514, 'grad_norm': 5.906411170959473, 'learning_rate': 5.7113446388895855e-06}[Rank 3] Trainer log: {'loss': 0.3514, 'grad_norm': 5.906411170959473, 'learning_rate': 5.7113446388895855e-06}[Rank 0] Trainer log: {'loss': 0.3514, 'grad_norm': 5.906411170959473, 'learning_rate': 5.7113446388895855e-06}
+
+
+[Rank 2] Trainer log: {'loss': 0.3514, 'grad_norm': 5.906411170959473, 'learning_rate': 5.7113446388895855e-06}
+{'loss': 0.3514, 'grad_norm': 5.906411170959473, 'learning_rate': 5.7113446388895855e-06, 'epoch': 0.66}
+tensor(-0.0014, device='cuda:2', grad_fn=<AddBackward0>) tensor(-0.0014, device='cuda:2', grad_fn=<MulBackward0>)
+tensor(-0.0010, device='cuda:1', grad_fn=<AddBackward0>) tensor(-0.0010, device='cuda:1', grad_fn=<MulBackward0>)
+tensor(-0.0008, device='cuda:3', grad_fn=<AddBackward0>) tensor(-0.0008, device='cuda:3', grad_fn=<MulBackward0>)
+tensor(-0.0007, device='cuda:0', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:0', grad_fn=<MulBackward0>)
+{'train/tv_loss': 0.00023483284749090672, 'train/lm_loss': 3.645087999757379e-05, 'train/info_loss': 2.235124156868551e-05, 'train/ref_loss': 0.06819528341293335, 'train/uncertainty_loss': -6.94944174028933e-05, 'train/video_loss': 0.07002680748701096, 'train/total_loss': 0.07006325572729111}
+tensor(-0.0012, device='cuda:1', grad_fn=<AddBackward0>) tensor(-0.0012, device='cuda:1', grad_fn=<MulBackward0>)
+tensor(-0.0012, device='cuda:3', grad_fn=<AddBackward0>) tensor(-0.0012, device='cuda:3', grad_fn=<MulBackward0>)
+tensor(-0.0010, device='cuda:2', grad_fn=<AddBackward0>) tensor(-0.0010, device='cuda:2', grad_fn=<MulBackward0>)
+tensor(0.5629, device='cuda:0', grad_fn=<AddBackward0>) tensor(-0.0008, device='cuda:0', grad_fn=<MulBackward0>)
+{'train/tv_loss': 0.000887374859303236, 'train/lm_loss': 4.150436725467444e-05, 'train/info_loss': 2.270885306643322e-05, 'train/ref_loss': 0.5856791138648987, 'train/uncertainty_loss': 0.05629110932350159, 'train/video_loss': 0.649091899394989, 'train/total_loss': 0.6491333842277527}
+[Rank 1] Trainer log: {'loss': 0.4015, 'grad_norm': 11.06789493560791, 'learning_rate': 5.701710356506665e-06}
+[Rank 3] Trainer log: {'loss': 0.4015, 'grad_norm': 11.06789493560791, 'learning_rate': 5.701710356506665e-06}
+[Rank 0] Trainer log: {'loss': 0.4015, 'grad_norm': 11.06789493560791, 'learning_rate': 5.701710356506665e-06}[Rank 2] Trainer log: {'loss': 0.4015, 'grad_norm': 11.06789493560791, 'learning_rate': 5.701710356506665e-06}
+
+{'loss': 0.4015, 'grad_norm': 11.06789493560791, 'learning_rate': 5.701710356506665e-06, 'epoch': 0.66}
+tensor(-0.0010, device='cuda:3', grad_fn=<AddBackward0>) tensor(-0.0010, device='cuda:3', grad_fn=<MulBackward0>)
+tensor(-0.0007, device='cuda:2', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:2', grad_fn=<MulBackward0>)
+tensor(-0.0007, device='cuda:1', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:1', grad_fn=<MulBackward0>)
+tensor(0.0157, device='cuda:0', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:0', grad_fn=<MulBackward0>)
+{'train/tv_loss': 0.00012852794025093318, 'train/lm_loss': 5.418520304374397e-05, 'train/info_loss': 2.3424076061928645e-05, 'train/ref_loss': 0.22761771082878113, 'train/uncertainty_loss': 0.001567848213016987, 'train/video_loss': 0.23023721575737, 'train/total_loss': 0.23029139637947083}
+tensor(-0.0009, device='cuda:2', grad_fn=<AddBackward0>) tensor(-0.0009, device='cuda:2', grad_fn=<MulBackward0>)
+tensor(-0.0010, device='cuda:3', grad_fn=<AddBackward0>) tensor(-0.0010, device='cuda:3', grad_fn=<MulBackward0>)
+tensor(-0.0008, device='cuda:1', grad_fn=<AddBackward0>) tensor(-0.0008, device='cuda:1', grad_fn=<MulBackward0>)
+tensor(-0.0007, device='cuda:0', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:0', grad_fn=<MulBackward0>)
+{'train/tv_loss': 0.0001663352712057531, 'train/lm_loss': 2.46032839640975e-05, 'train/info_loss': 1.710624019324314e-05, 'train/ref_loss': 0.1792670488357544, 'train/uncertainty_loss': -7.328792125917972e-05, 'train/video_loss': 0.18054156005382538, 'train/total_loss': 0.18056616187095642}
+[Rank 1] Trainer log: {'loss': 0.2317, 'grad_norm': 2.1956772804260254, 'learning_rate': 5.6920809654291945e-06}[Rank 3] Trainer log: {'loss': 0.2317, 'grad_norm': 2.1956772804260254, 'learning_rate': 5.6920809654291945e-06}[Rank 0] Trainer log: {'loss': 0.2317, 'grad_norm': 2.1956772804260254, 'learning_rate': 5.6920809654291945e-06}
+
+[Rank 2] Trainer log: {'loss': 0.2317, 'grad_norm': 2.1956772804260254, 'learning_rate': 5.6920809654291945e-06}
+
+{'loss': 0.2317, 'grad_norm': 2.1956772804260254, 'learning_rate': 5.6920809654291945e-06, 'epoch': 0.66}
+tensor(-0.0009, device='cuda:3', grad_fn=<AddBackward0>) tensor(-0.0009, device='cuda:3', grad_fn=<MulBackward0>)
+tensor(-0.0007, device='cuda:1', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:1', grad_fn=<MulBackward0>)
+tensor(-0.0007, device='cuda:2', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:2', grad_fn=<MulBackward0>)
+tensor(-0.0007, device='cuda:0', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:0', grad_fn=<MulBackward0>)
+{'train/tv_loss': 0.00016776639968156816, 'train/lm_loss': 2.7893029619008305e-05, 'train/info_loss': 2.0682384274550714e-05, 'train/ref_loss': 0.2100030779838562, 'train/uncertainty_loss': -7.060928619466722e-05, 'train/video_loss': 0.21129527688026428, 'train/total_loss': 0.21132317185401917}
+tensor(-0.0013, device='cuda:2', grad_fn=<AddBackward0>) tensor(-0.0013, device='cuda:2', grad_fn=<MulBackward0>)
+tensor(-0.0009, device='cuda:1', grad_fn=<AddBackward0>) tensor(-0.0009, device='cuda:1', grad_fn=<MulBackward0>)
+tensor(-0.0007, device='cuda:0', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:0', grad_fn=<MulBackward0>)
+{'train/tv_loss': 0.00012384995352476835, 'train/lm_loss': 4.760652373079211e-05, 'train/info_loss': 2.3781687559676357e-05, 'train/ref_loss': 0.09827212989330292, 'train/uncertainty_loss': -7.054724264889956e-05, 'train/video_loss': 0.09921616315841675, 'train/total_loss': 0.09926377236843109}
+tensor(0.1509, device='cuda:3', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:3', grad_fn=<MulBackward0>)
+[Rank 2] Trainer log: {'loss': 0.2525, 'grad_norm': 3.496741771697998, 'learning_rate': 5.6824564766150724e-06}[Rank 3] Trainer log: {'loss': 0.2525, 'grad_norm': 3.496741771697998, 'learning_rate': 5.6824564766150724e-06}[Rank 0] Trainer log: {'loss': 0.2525, 'grad_norm': 3.496741771697998, 'learning_rate': 5.6824564766150724e-06}
+
+[Rank 1] Trainer log: {'loss': 0.2525, 'grad_norm': 3.496741771697998, 'learning_rate': 5.6824564766150724e-06}
+
+{'loss': 0.2525, 'grad_norm': 3.496741771697998, 'learning_rate': 5.6824564766150724e-06, 'epoch': 0.66}
+tensor(-0.0014, device='cuda:2', grad_fn=<AddBackward0>) tensor(-0.0014, device='cuda:2', grad_fn=<MulBackward0>)
+tensor(-0.0012, device='cuda:0', grad_fn=<AddBackward0>) tensor(-0.0012, device='cuda:0', grad_fn=<MulBackward0>)
+{'train/tv_loss': None, 'train/lm_loss': 0.20042383670806885, 'train/info_loss': 0.1449424922466278, 'train/ref_loss': None, 'train/uncertainty_loss': -0.00011801900109276175, 'train/video_loss': 0.14482447504997253, 'train/total_loss': 0.3452483117580414}
+tensor(-0.0010, device='cuda:1', grad_fn=<AddBackward0>) tensor(-0.0010, device='cuda:1', grad_fn=<MulBackward0>)
+tensor(-0.0007, device='cuda:3', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:3', grad_fn=<MulBackward0>)
+tensor(-0.0013, device='cuda:0', grad_fn=<AddBackward0>) tensor(-0.0013, device='cuda:0', grad_fn=<MulBackward0>)
+{'train/tv_loss': None, 'train/lm_loss': 0.45617589950561527, 'train/info_loss': 0.24490997195243835, 'train/ref_loss': None, 'train/uncertainty_loss': -0.00012810240732505918, 'train/video_loss': 0.24478186666965485, 'train/total_loss': 0.7009577751159668}
+tensor(-0.0009, device='cuda:1', grad_fn=<AddBackward0>) tensor(-0.0009, device='cuda:1', grad_fn=<MulBackward0>)
+tensor(-0.0007, device='cuda:2', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:2', grad_fn=<MulBackward0>)
+tensor(-0.0007, device='cuda:3', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:3', grad_fn=<MulBackward0>)
+[Rank 2] Trainer log: {'loss': 0.3158, 'grad_norm': 4.859067440032959, 'learning_rate': 5.6728369010166426e-06}[Rank 1] Trainer log: {'loss': 0.3158, 'grad_norm': 4.859067440032959, 'learning_rate': 5.6728369010166426e-06}[Rank 0] Trainer log: {'loss': 0.3158, 'grad_norm': 4.859067440032959, 'learning_rate': 5.6728369010166426e-06}
+
+[Rank 3] Trainer log: {'loss': 0.3158, 'grad_norm': 4.859067440032959, 'learning_rate': 5.6728369010166426e-06}
+
+{'loss': 0.3158, 'grad_norm': 4.859067440032959, 'learning_rate': 5.6728369010166426e-06, 'epoch': 0.66}
+tensor(-0.0013, device='cuda:3', grad_fn=<AddBackward0>) tensor(-0.0013, device='cuda:3', grad_fn=<MulBackward0>)
+tensor(-0.0011, device='cuda:2', grad_fn=<AddBackward0>) tensor(-0.0011, device='cuda:2', grad_fn=<MulBackward0>)
+tensor(-0.0011, device='cuda:1', grad_fn=<AddBackward0>) tensor(-0.0011, device='cuda:1', grad_fn=<MulBackward0>)
+tensor(-0.0010, device='cuda:0', grad_fn=<AddBackward0>) tensor(-0.0010, device='cuda:0', grad_fn=<MulBackward0>)
+{'train/tv_loss': None, 'train/lm_loss': 0.11436709165573121, 'train/info_loss': 0.26202645897865295, 'train/ref_loss': None, 'train/uncertainty_loss': -9.564846986904741e-05, 'train/video_loss': 0.26193082332611084, 'train/total_loss': 0.3762979209423065}
+tensor(-0.0015, device='cuda:2', grad_fn=<AddBackward0>) tensor(-0.0015, device='cuda:2', grad_fn=<MulBackward0>)
+tensor(-0.0012, device='cuda:1', grad_fn=<AddBackward0>) tensor(-0.0012, device='cuda:1', grad_fn=<MulBackward0>)
+tensor(-0.0011, device='cuda:3', grad_fn=<AddBackward0>) tensor(-0.0011, device='cuda:3', grad_fn=<MulBackward0>)
+tensor(-0.0009, device='cuda:0', grad_fn=<AddBackward0>) tensor(-0.0009, device='cuda:0', grad_fn=<MulBackward0>)
+{'train/tv_loss': None, 'train/lm_loss': 0.15787471532821656, 'train/info_loss': 0.17485037446022034, 'train/ref_loss': None, 'train/uncertainty_loss': -9.450375218875707e-05, 'train/video_loss': 0.17475587129592896, 'train/total_loss': 0.33263057470321655}
+[Rank 1] Trainer log: {'loss': 0.4708, 'grad_norm': 2.322233200073242, 'learning_rate': 5.663222249580649e-06}[Rank 3] Trainer log: {'loss': 0.4708, 'grad_norm': 2.322233200073242, 'learning_rate': 5.663222249580649e-06}[Rank 0] Trainer log: {'loss': 0.4708, 'grad_norm': 2.322233200073242, 'learning_rate': 5.663222249580649e-06}
+
+[Rank 2] Trainer log: {'loss': 0.4708, 'grad_norm': 2.322233200073242, 'learning_rate': 5.663222249580649e-06}
+
+{'loss': 0.4708, 'grad_norm': 2.322233200073242, 'learning_rate': 5.663222249580649e-06, 'epoch': 0.66}
+tensor(-0.0011, device='cuda:1', grad_fn=<AddBackward0>) tensor(-0.0011, device='cuda:1', grad_fn=<MulBackward0>)
+tensor(-0.0011, device='cuda:3', grad_fn=<AddBackward0>) tensor(-0.0011, device='cuda:3', grad_fn=<MulBackward0>)
+tensor(0.2767, device='cuda:2', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:2', grad_fn=<MulBackward0>)
+tensor(0.0772, device='cuda:0', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:0', grad_fn=<MulBackward0>)
+{'train/tv_loss': 0.00016861287876963617, 'train/lm_loss': 4.701061698142439e-05, 'train/info_loss': 2.235124156868551e-05, 'train/ref_loss': 0.2696513533592224, 'train/uncertainty_loss': 0.007722488045692444, 'train/video_loss': 0.2787451148033142, 'train/total_loss': 0.2787921130657196}
+tensor(-0.0012, device='cuda:0', grad_fn=<AddBackward0>) tensor(-0.0012, device='cuda:0', grad_fn=<MulBackward0>)
+{'train/tv_loss': None, 'train/lm_loss': 0.20862238407135011, 'train/info_loss': 0.1744736284017563, 'train/ref_loss': None, 'train/uncertainty_loss': -0.0001176536548882723, 'train/video_loss': 0.17435596883296967, 'train/total_loss': 0.3829783499240875}
+tensor(-0.0010, device='cuda:1', grad_fn=<AddBackward0>) tensor(-0.0010, device='cuda:1', grad_fn=<MulBackward0>)
+tensor(-0.0007, device='cuda:2', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:2', grad_fn=<MulBackward0>)
+tensor(0.2500, device='cuda:3', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:3', grad_fn=<MulBackward0>)
+[Rank 3] Trainer log: {'loss': 0.3702, 'grad_norm': 10.16737174987793, 'learning_rate': 5.653612533248233e-06}[Rank 1] Trainer log: {'loss': 0.3702, 'grad_norm': 10.16737174987793, 'learning_rate': 5.653612533248233e-06}
+[Rank 0] Trainer log: {'loss': 0.3702, 'grad_norm': 10.16737174987793, 'learning_rate': 5.653612533248233e-06}
+
+[Rank 2] Trainer log: {'loss': 0.3702, 'grad_norm': 10.16737174987793, 'learning_rate': 5.653612533248233e-06}
+{'loss': 0.3702, 'grad_norm': 10.16737174987793, 'learning_rate': 5.653612533248233e-06, 'epoch': 0.66}
+tensor(-0.0014, device='cuda:3', grad_fn=<AddBackward0>) tensor(-0.0014, device='cuda:3', grad_fn=<MulBackward0>)
+tensor(-0.0010, device='cuda:1', grad_fn=<AddBackward0>) tensor(-0.0010, device='cuda:1', grad_fn=<MulBackward0>)
+tensor(-0.0007, device='cuda:2', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:2', grad_fn=<MulBackward0>)
+tensor(0.0352, device='cuda:0', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:0', grad_fn=<MulBackward0>)
+{'train/tv_loss': 0.00026960894465446474, 'train/lm_loss': 2.4698639754205944e-05, 'train/info_loss': 1.7702266632113606e-05, 'train/ref_loss': 0.24252426624298096, 'train/uncertainty_loss': 0.0035187847912311557, 'train/video_loss': 0.2482176274061203, 'train/total_loss': 0.2482423186302185}
+tensor(-0.0012, device='cuda:1', grad_fn=<AddBackward0>) tensor(-0.0012, device='cuda:1', grad_fn=<MulBackward0>)
+tensor(-0.0012, device='cuda:2', grad_fn=<AddBackward0>) tensor(-0.0012, device='cuda:2', grad_fn=<MulBackward0>)
+tensor(-0.0007, device='cuda:0', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:0', grad_fn=<MulBackward0>)
+{'train/tv_loss': 0.00018143276683986188, 'train/lm_loss': 3.5974127240478995e-05, 'train/info_loss': 1.9728748156921938e-05, 'train/ref_loss': 0.08057326078414917, 'train/uncertainty_loss': -7.370269158855081e-05, 'train/video_loss': 0.08197075128555298, 'train/total_loss': 0.08200672268867493}
+tensor(0.3041, device='cuda:3', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:3', grad_fn=<MulBackward0>)
+[Rank 2] Trainer log: {'loss': 0.3183, 'grad_norm': 8.234452247619629, 'learning_rate': 5.644007762954926e-06}[Rank 1] Trainer log: {'loss': 0.3183, 'grad_norm': 8.234452247619629, 'learning_rate': 5.644007762954926e-06}
+
+[Rank 3] Trainer log: {'loss': 0.3183, 'grad_norm': 8.234452247619629, 'learning_rate': 5.644007762954926e-06}[Rank 0] Trainer log: {'loss': 0.3183, 'grad_norm': 8.234452247619629, 'learning_rate': 5.644007762954926e-06}
+
+{'loss': 0.3183, 'grad_norm': 8.234452247619629, 'learning_rate': 5.644007762954926e-06, 'epoch': 0.66}
+tensor(-0.0012, device='cuda:3', grad_fn=<AddBackward0>) tensor(-0.0012, device='cuda:3', grad_fn=<MulBackward0>)
+tensor(-0.0010, device='cuda:1', grad_fn=<AddBackward0>) tensor(-0.0010, device='cuda:1', grad_fn=<MulBackward0>)
+tensor(-0.0007, device='cuda:2', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:2', grad_fn=<MulBackward0>)
+tensor(-0.0009, device='cuda:0', grad_fn=<AddBackward0>) tensor(-0.0009, device='cuda:0', grad_fn=<MulBackward0>)
+{'train/tv_loss': None, 'train/lm_loss': 0.06672886610031128, 'train/info_loss': 0.18479126691818237, 'train/ref_loss': None, 'train/uncertainty_loss': -9.260554797947408e-05, 'train/video_loss': 0.1846986562013626, 'train/total_loss': 0.2514275312423706}
+tensor(-0.0011, device='cuda:3', grad_fn=<AddBackward0>) tensor(-0.0011, device='cuda:3', grad_fn=<MulBackward0>)
+tensor(-0.0014, device='cuda:1', grad_fn=<AddBackward0>) tensor(-0.0014, device='cuda:1', grad_fn=<MulBackward0>)
+tensor(-0.0007, device='cuda:2', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:2', grad_fn=<MulBackward0>)
+tensor(-0.0007, device='cuda:0', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:0', grad_fn=<MulBackward0>)
+{'train/tv_loss': 0.00025838704314082864, 'train/lm_loss': 2.7893029619008305e-05, 'train/info_loss': 1.7702266632113606e-05, 'train/ref_loss': 0.09182281792163849, 'train/uncertainty_loss': -6.915096309967339e-05, 'train/video_loss': 0.09383846819400787, 'train/total_loss': 0.09386636316776276}
+[Rank 1] Trainer log: {'loss': 0.2844, 'grad_norm': 3.304973602294922, 'learning_rate': 5.634407949630617e-06}[Rank 0] Trainer log: {'loss': 0.2844, 'grad_norm': 3.304973602294922, 'learning_rate': 5.634407949630617e-06}[Rank 3] Trainer log: {'loss': 0.2844, 'grad_norm': 3.304973602294922, 'learning_rate': 5.634407949630617e-06}
+
+[Rank 2] Trainer log: {'loss': 0.2844, 'grad_norm': 3.304973602294922, 'learning_rate': 5.634407949630617e-06}
+
+{'loss': 0.2844, 'grad_norm': 3.304973602294922, 'learning_rate': 5.634407949630617e-06, 'epoch': 0.66}
+tensor(-0.0009, device='cuda:1', grad_fn=<AddBackward0>) tensor(-0.0009, device='cuda:1', grad_fn=<MulBackward0>)
+tensor(1.0766, device='cuda:2', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:2', grad_fn=<MulBackward0>)
+tensor(-0.0007, device='cuda:3', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:3', grad_fn=<MulBackward0>)
+tensor(0.7360, device='cuda:0', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:0', grad_fn=<MulBackward0>)
+{'train/tv_loss': 0.00015928944339975716, 'train/lm_loss': 4.684376472141594e-05, 'train/info_loss': 2.1993630070937797e-05, 'train/ref_loss': 0.7353293299674988, 'train/uncertainty_loss': 0.07359982132911683, 'train/video_loss': 0.8102254271507263, 'train/total_loss': 0.8102722764015198}
+tensor(-0.0012, device='cuda:2', grad_fn=<AddBackward0>) tensor(-0.0012, device='cuda:2', grad_fn=<MulBackward0>)
+tensor(-0.0013, device='cuda:1', grad_fn=<AddBackward0>) tensor(-0.0013, device='cuda:1', grad_fn=<MulBackward0>)
+tensor(-0.0011, device='cuda:0', grad_fn=<AddBackward0>) tensor(-0.0011, device='cuda:0', grad_fn=<MulBackward0>)
+{'train/tv_loss': None, 'train/lm_loss': 0.2558208465576172, 'train/info_loss': 0.18679678440093994, 'train/ref_loss': None, 'train/uncertainty_loss': -0.00010758317075669766, 'train/video_loss': 0.18668919801712036, 'train/total_loss': 0.44251003861427307}
+tensor(-0.0011, device='cuda:3', grad_fn=<AddBackward0>) tensor(-0.0011, device='cuda:3', grad_fn=<MulBackward0>)
+[Rank 3] Trainer log: {'loss': 0.4651, 'grad_norm': 17.812118530273438, 'learning_rate': 5.624813104199567e-06}
+[Rank 2] Trainer log: {'loss': 0.4651, 'grad_norm': 17.812118530273438, 'learning_rate': 5.624813104199567e-06}
+[Rank 0] Trainer log: {'loss': 0.4651, 'grad_norm': 17.812118530273438, 'learning_rate': 5.624813104199567e-06}[Rank 1] Trainer log: {'loss': 0.4651, 'grad_norm': 17.812118530273438, 'learning_rate': 5.624813104199567e-06}
+
+{'loss': 0.4651, 'grad_norm': 17.812118530273438, 'learning_rate': 5.624813104199567e-06, 'epoch': 0.66}
+tensor(-0.0010, device='cuda:2', grad_fn=<AddBackward0>) tensor(-0.0010, device='cuda:2', grad_fn=<MulBackward0>)
+tensor(-0.0007, device='cuda:1', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:1', grad_fn=<MulBackward0>)
+tensor(-0.0007, device='cuda:3', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:3', grad_fn=<MulBackward0>)
+tensor(0.4664, device='cuda:0', grad_fn=<AddBackward0>) tensor(-0.0007, device='cuda:0', grad_fn=<MulBackward0>)
+{'train/tv_loss': 0.00020973044447600842, 'train/lm_loss': 4.715363611467183e-05, 'train/info_loss': 2.235124156868551e-05, 'train/ref_loss': 0.5165361166000366, 'train/uncertainty_loss': 0.0466405063867569, 'train/video_loss': 0.5648768544197083, 'train/total_loss': 0.5649240016937256}