Training in progress, step 27

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:029953f60badc5f481c9b69192a9687fcc02dc7d8397259a1fcacb9eb79f2bdc
+oid sha256:f4b105f8ba9adeb788677a2b0fc9f3480c4afb4a2fc1b943d0cce350b32bf647
 size 1204780872

train.log

@@ -1560,3 +1560,42 @@ Time to load cpu_adam op: 2.2944772243499756 seconds
 {'train/tv_loss': 0.018152934312820435, 'train/lm_loss': 6.412452697753906, 'train/info_loss': 3.024618625640869, 'train/ref_loss': 0.8245805501937866, 'train/uncertainty_loss': 0.17770199775695802, 'train/video_loss': 4.099513053894043}
 {'train/tv_loss': 0.023061570525169373, 'train/lm_loss': 6.410730361938477, 'train/info_loss': 3.009031295776367, 'train/ref_loss': 1.7581236362457275, 'train/uncertainty_loss': 0.165232253074646, 'train/video_loss': 5.024633407592773}
 {'train/tv_loss': None, 'train/lm_loss': 0.5883082747459412, 'train/info_loss': 0.5237130522727966, 'train/ref_loss': None, 'train/uncertainty_loss': 0.14144222736358644, 'train/video_loss': 0.665155291557312}
+{'train/tv_loss': 0.02307373732328415, 'train/lm_loss': 6.408398628234863, 'train/info_loss': 3.0012378692626953, 'train/ref_loss': 1.7808878421783447, 'train/uncertainty_loss': 0.1509645700454712, 'train/video_loss': 5.02538537979126}
+{'train/tv_loss': None, 'train/lm_loss': 0.9507150650024414, 'train/info_loss': 0.7013111114501953, 'train/ref_loss': None, 'train/uncertainty_loss': 0.1412076473236084, 'train/video_loss': 0.8425187468528748}
+{'train/tv_loss': None, 'train/lm_loss': 0.8176541924476624, 'train/info_loss': 0.5695577263832092, 'train/ref_loss': None, 'train/uncertainty_loss': 0.1411933183670044, 'train/video_loss': 0.7107510566711426}
+{'train/tv_loss': 0.018126337230205538, 'train/lm_loss': 6.410552024841309, 'train/info_loss': 3.009031295776367, 'train/ref_loss': 2.3561911582946777, 'train/uncertainty_loss': 0.1734020233154297, 'train/video_loss': 5.6111297607421875}
+[Rank 2] Trainer log: {'loss': 4.0766, 'grad_norm': 11.943275451660156, 'learning_rate': 3.125e-06}
+[Rank 0] Trainer log: {'loss': 4.0766, 'grad_norm': 11.943275451660156, 'learning_rate': 3.125e-06}
+[Rank 3] Trainer log: {'loss': 4.0766, 'grad_norm': 11.943275451660156, 'learning_rate': 3.125e-06}[Rank 1] Trainer log: {'loss': 4.0766, 'grad_norm': 11.943275451660156, 'learning_rate': 3.125e-06}
+
+{'loss': 4.0766, 'grad_norm': 11.943275451660156, 'learning_rate': 3.125e-06, 'epoch': 0.01}
+{'train/tv_loss': None, 'train/lm_loss': 0.9312324523925781, 'train/info_loss': 0.7997239828109741, 'train/ref_loss': None, 'train/uncertainty_loss': 0.14140816926956176, 'train/video_loss': 0.9411321878433228}
+{'train/tv_loss': None, 'train/lm_loss': 0.46152347326278687, 'train/info_loss': 0.7567092776298523, 'train/ref_loss': None, 'train/uncertainty_loss': 0.1412665605545044, 'train/video_loss': 0.8979758620262146}
+{'train/tv_loss': None, 'train/lm_loss': 0.582965075969696, 'train/info_loss': 0.5308855175971985, 'train/ref_loss': None, 'train/uncertainty_loss': 0.14128931760787963, 'train/video_loss': 0.6721748113632202}
+{'train/tv_loss': None, 'train/lm_loss': 0.9812306761741638, 'train/info_loss': 0.5562796592712402, 'train/ref_loss': None, 'train/uncertainty_loss': 0.14140853881835938, 'train/video_loss': 0.6976882219314575}
+{'train/tv_loss': None, 'train/lm_loss': 0.6936975717544556, 'train/info_loss': 0.2903560400009155, 'train/ref_loss': None, 'train/uncertainty_loss': 0.1412665605545044, 'train/video_loss': 0.43162262439727783}
+{'train/tv_loss': 0.023120765388011933, 'train/lm_loss': 6.408127307891846, 'train/info_loss': 2.9973411560058594, 'train/ref_loss': 1.279544711112976, 'train/uncertainty_loss': 0.14335076808929445, 'train/video_loss': 4.512719631195068}
+{'train/tv_loss': None, 'train/lm_loss': 0.7783567905426025, 'train/info_loss': 0.9829906821250916, 'train/ref_loss': None, 'train/uncertainty_loss': 0.14142510890960694, 'train/video_loss': 1.1244157552719116}
+{'train/tv_loss': None, 'train/lm_loss': 1.119819164276123, 'train/info_loss': 0.36488640308380127, 'train/ref_loss': None, 'train/uncertainty_loss': 0.14145026206970215, 'train/video_loss': 0.5063366889953613}
+[Rank 1] Trainer log: {'loss': 2.79, 'grad_norm': 5.420574188232422, 'learning_rate': 3.2500000000000002e-06}[Rank 3] Trainer log: {'loss': 2.79, 'grad_norm': 5.420574188232422, 'learning_rate': 3.2500000000000002e-06}[Rank 0] Trainer log: {'loss': 2.79, 'grad_norm': 5.420574188232422, 'learning_rate': 3.2500000000000002e-06}
+
+
+[Rank 2] Trainer log: {'loss': 2.79, 'grad_norm': 5.420574188232422, 'learning_rate': 3.2500000000000002e-06}
+{'loss': 2.79, 'grad_norm': 5.420574188232422, 'learning_rate': 3.2500000000000002e-06, 'epoch': 0.01}
+{'train/tv_loss': None, 'train/lm_loss': 0.8753480911254883, 'train/info_loss': 0.6647682785987854, 'train/ref_loss': None, 'train/uncertainty_loss': 0.14118621349334717, 'train/video_loss': 0.8059545159339905}
+{'train/tv_loss': 0.02476734071969986, 'train/lm_loss': 6.386311054229736, 'train/info_loss': 2.985651969909668, 'train/ref_loss': 1.7279162406921387, 'train/uncertainty_loss': 0.15406183004379273, 'train/video_loss': 4.966699123382568}
+{'train/tv_loss': None, 'train/lm_loss': 0.6388610601425171, 'train/info_loss': 1.011538028717041, 'train/ref_loss': None, 'train/uncertainty_loss': 0.1412294387817383, 'train/video_loss': 1.1527674198150635}
+{'train/tv_loss': None, 'train/lm_loss': 0.6446877121925354, 'train/info_loss': 0.8607529997825623, 'train/ref_loss': None, 'train/uncertainty_loss': 0.14118621349334717, 'train/video_loss': 1.0019391775131226}
+{'train/tv_loss': 0.020772358775138857, 'train/lm_loss': 6.390253067016602, 'train/info_loss': 2.9973411560058594, 'train/ref_loss': 2.2043981552124023, 'train/uncertainty_loss': 0.18207603693008423, 'train/video_loss': 5.466904640197754}
+{'train/tv_loss': None, 'train/lm_loss': 1.3000074625015259, 'train/info_loss': 0.5173362493515015, 'train/ref_loss': None, 'train/uncertainty_loss': 0.1413517951965332, 'train/video_loss': 0.6586880683898926}
+{'train/tv_loss': 0.026385802030563357, 'train/lm_loss': 6.386507987976074, 'train/info_loss': 2.989548444747925, 'train/ref_loss': 1.9725675582885742, 'train/uncertainty_loss': 0.16719863414764405, 'train/video_loss': 5.23485803604126}
+{'train/tv_loss': 0.022161963582038882, 'train/lm_loss': 6.3933000564575195, 'train/info_loss': 3.012928009033203, 'train/ref_loss': 0.9150100946426392, 'train/uncertainty_loss': 0.1290119767189026, 'train/video_loss': 4.1455979347229}
+[Rank 3] Trainer log: {'loss': 4.8028, 'grad_norm': 12.116530418395996, 'learning_rate': 3.3750000000000003e-06}[Rank 0] Trainer log: {'loss': 4.8028, 'grad_norm': 12.116530418395996, 'learning_rate': 3.3750000000000003e-06}[Rank 2] Trainer log: {'loss': 4.8028, 'grad_norm': 12.116530418395996, 'learning_rate': 3.3750000000000003e-06}
+
+
+[Rank 1] Trainer log: {'loss': 4.8028, 'grad_norm': 12.116530418395996, 'learning_rate': 3.3750000000000003e-06}
+{'loss': 4.8028, 'grad_norm': 12.116530418395996, 'learning_rate': 3.3750000000000003e-06, 'epoch': 0.01}
+{'train/tv_loss': None, 'train/lm_loss': 1.281248688697815, 'train/info_loss': 0.3776136636734009, 'train/ref_loss': None, 'train/uncertainty_loss': 0.1411444664001465, 'train/video_loss': 0.5187581181526184}
+{'train/tv_loss': None, 'train/lm_loss': 0.9832038879394531, 'train/info_loss': 0.6202327013015747, 'train/ref_loss': None, 'train/uncertainty_loss': 0.1411651611328125, 'train/video_loss': 0.7613978385925293}
+{'train/tv_loss': 0.027196800708770754, 'train/lm_loss': 6.3878889083862305, 'train/info_loss': 2.9973411560058594, 'train/ref_loss': 0.9530278444290161, 'train/uncertainty_loss': 0.15620580911636353, 'train/video_loss': 4.215362071990967}
+{'train/tv_loss': None, 'train/lm_loss': 0.5092962980270386, 'train/info_loss': 0.6516232490539551, 'train/ref_loss': None, 'train/uncertainty_loss': 0.14117213487625122, 'train/video_loss': 0.7927954196929932}