diff --git "a/335101.err" "b/335101.err"
new file mode 100644--- /dev/null
+++ "b/335101.err"
@@ -0,0 +1,7873 @@
+/var/spool/slurmd/job335101/slurm_script: line 12: activate: No such file or directory
+W0215 19:21:03.132000 133066830354240 torch/distributed/run.py:779] 
+W0215 19:21:03.132000 133066830354240 torch/distributed/run.py:779] *****************************************
+W0215 19:21:03.132000 133066830354240 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0215 19:21:03.132000 133066830354240 torch/distributed/run.py:779] *****************************************
+W0215 19:21:03.133000 127054241253184 torch/distributed/run.py:779] 
+W0215 19:21:03.133000 127054241253184 torch/distributed/run.py:779] *****************************************
+W0215 19:21:03.133000 127054241253184 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0215 19:21:03.133000 127054241253184 torch/distributed/run.py:779] *****************************************
+W0215 19:21:03.133000 129572943243072 torch/distributed/run.py:779] 
+W0215 19:21:03.133000 129572943243072 torch/distributed/run.py:779] *****************************************
+W0215 19:21:03.133000 129572943243072 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0215 19:21:03.133000 129572943243072 torch/distributed/run.py:779] *****************************************
+W0215 19:21:03.133000 140662069983040 torch/distributed/run.py:779] 
+W0215 19:21:03.133000 140662069983040 torch/distributed/run.py:779] *****************************************
+W0215 19:21:03.133000 140662069983040 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0215 19:21:03.133000 140662069983040 torch/distributed/run.py:779] *****************************************
+W0215 19:21:03.133000 126469684954944 torch/distributed/run.py:779] 
+W0215 19:21:03.133000 126469684954944 torch/distributed/run.py:779] *****************************************
+W0215 19:21:03.133000 126469684954944 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0215 19:21:03.133000 126469684954944 torch/distributed/run.py:779] *****************************************
+W0215 19:21:03.133000 128273080436544 torch/distributed/run.py:779] 
+W0215 19:21:03.133000 128273080436544 torch/distributed/run.py:779] *****************************************
+W0215 19:21:03.133000 128273080436544 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0215 19:21:03.133000 128273080436544 torch/distributed/run.py:779] *****************************************
+W0215 19:21:03.133000 138557481989952 torch/distributed/run.py:779] 
+W0215 19:21:03.133000 138557481989952 torch/distributed/run.py:779] *****************************************
+W0215 19:21:03.133000 138557481989952 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0215 19:21:03.133000 138557481989952 torch/distributed/run.py:779] *****************************************
+W0215 19:21:03.133000 130231383332672 torch/distributed/run.py:779] 
+W0215 19:21:03.133000 130231383332672 torch/distributed/run.py:779] *****************************************
+W0215 19:21:03.133000 130231383332672 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0215 19:21:03.133000 130231383332672 torch/distributed/run.py:779] *****************************************
+W0215 19:21:03.133000 123572905850688 torch/distributed/run.py:779] 
+W0215 19:21:03.133000 123572905850688 torch/distributed/run.py:779] *****************************************
+W0215 19:21:03.133000 123572905850688 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0215 19:21:03.133000 123572905850688 torch/distributed/run.py:779] *****************************************
+W0215 19:21:03.133000 126876645988160 torch/distributed/run.py:779] 
+W0215 19:21:03.133000 126876645988160 torch/distributed/run.py:779] *****************************************
+W0215 19:21:03.133000 126876645988160 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0215 19:21:03.133000 126876645988160 torch/distributed/run.py:779] *****************************************
+W0215 19:21:03.133000 138510042982208 torch/distributed/run.py:779] 
+W0215 19:21:03.133000 138510042982208 torch/distributed/run.py:779] *****************************************
+W0215 19:21:03.133000 138510042982208 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0215 19:21:03.133000 138510042982208 torch/distributed/run.py:779] *****************************************
+W0215 19:21:03.133000 135494447343424 torch/distributed/run.py:779] 
+W0215 19:21:03.133000 135494447343424 torch/distributed/run.py:779] *****************************************
+W0215 19:21:03.133000 135494447343424 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0215 19:21:03.133000 135494447343424 torch/distributed/run.py:779] *****************************************
+W0215 19:21:03.133000 128980037068608 torch/distributed/run.py:779] 
+W0215 19:21:03.133000 128980037068608 torch/distributed/run.py:779] *****************************************
+W0215 19:21:03.133000 128980037068608 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0215 19:21:03.133000 128980037068608 torch/distributed/run.py:779] *****************************************
+W0215 19:21:03.133000 134808981854016 torch/distributed/run.py:779] 
+W0215 19:21:03.133000 134808981854016 torch/distributed/run.py:779] *****************************************
+W0215 19:21:03.133000 134808981854016 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0215 19:21:03.133000 134808981854016 torch/distributed/run.py:779] *****************************************
+W0215 19:21:03.133000 131641266206528 torch/distributed/run.py:779] 
+W0215 19:21:03.133000 131641266206528 torch/distributed/run.py:779] *****************************************
+W0215 19:21:03.133000 131641266206528 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0215 19:21:03.133000 131641266206528 torch/distributed/run.py:779] *****************************************
+W0215 19:21:03.133000 131517812614976 torch/distributed/run.py:779] 
+W0215 19:21:03.133000 131517812614976 torch/distributed/run.py:779] *****************************************
+W0215 19:21:03.133000 131517812614976 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0215 19:21:03.133000 131517812614976 torch/distributed/run.py:779] *****************************************
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+Traceback (most recent call last):
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+Traceback (most recent call last):
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from llava.train.train import train
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from llava.train.train import train
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+    from llava.train.train import train
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+    from llava.train.train import train    
+from llava.train.train import train
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+    from llava.train.train import train
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+        from llava.train.train import trainfrom llava.train.train import train
+
+Traceback (most recent call last):
+Traceback (most recent call last):
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+Traceback (most recent call last):
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from llava.train.train import train
+    from llava.train.train import train  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from llava.train.train import train
+Traceback (most recent call last):
+    from llava.train.train import train
+    from llava.train.train import train
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+                from llava.train.train import trainfrom llava.train.train import trainfrom llava.train.train import trainfrom llava.train.train import train
+
+
+
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from llava.train.train import train
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+        from llava.train.train import trainfrom llava.train.train import train
+
+
+    from llava.train.train import train  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from llava.train.train import train
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+from llava.train.train import train
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    Traceback (most recent call last):
+from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from llava.train.train import train
+    from llava.train.train import train
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from llava.train.train import train
+    from llava.train.train import train
+    from llava.train.train import train
+    from llava.train.train import train  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from llava.train.train import train
+    from .model import LlavaLlamaForCausalLM
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+from llava.train.train import train
+    from .model import LlavaLlamaForCausalLM
+    from llava.train.train import train
+Traceback (most recent call last):
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from llava.train.train import train
+    from llava.train.train import train    
+from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from .model import LlavaLlamaForCausalLM
+Traceback (most recent call last):
+    from llava.train.train import train
+Traceback (most recent call last):
+Traceback (most recent call last):
+    from llava.train.train import train    
+from llava.train.train import train
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from llava.train.train import train
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .model import LlavaLlamaForCausalLM    
+from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+from llava.train.train import train
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from llava.train.train import train
+        from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM
+    
+from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+        from .model import LlavaLlamaForCausalLMfrom llava.train.train import train
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+        from .model import LlavaLlamaForCausalLM
+from .model import LlavaLlamaForCausalLM
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+        from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM
+
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from llava.train.train import train  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+Traceback (most recent call last):
+        from .model import LlavaLlamaForCausalLM    from .model import LlavaLlamaForCausalLM
+from .model import LlavaLlamaForCausalLM
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from llava.train.train import train
+Traceback (most recent call last):
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from .model import LlavaLlamaForCausalLM
+    from llava.train.train import train
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from llava.train.train import train
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+Traceback (most recent call last):
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from llava.train.train import train
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from .model import LlavaLlamaForCausalLMTraceback (most recent call last):
+
+Traceback (most recent call last):
+    from .model import LlavaLlamaForCausalLM
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .model import LlavaLlamaForCausalLM
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from .model import LlavaLlamaForCausalLM    
+from .model import LlavaLlamaForCausalLM
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from llava.train.train import train
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .model import LlavaLlamaForCausalLM
+    from .model import LlavaLlamaForCausalLM
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .model import LlavaLlamaForCausalLM
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+            from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+
+from .model import LlavaLlamaForCausalLM
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+            from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from llava.train.train import train
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+        from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigTraceback (most recent call last):
+from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+from .model import LlavaLlamaForCausalLM
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+        from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM
+
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from llava.train.train import train
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+        from llava.train.train import trainfrom llava.train.train import train
+
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+Traceback (most recent call last):
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from .model import LlavaLlamaForCausalLM
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig    
+from llava.train.train import train
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .model import LlavaLlamaForCausalLMfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+
+Traceback (most recent call last):
+    Traceback (most recent call last):
+from llava.train.train import train
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig        
+from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+
+Traceback (most recent call last):
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+            from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM
+
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from .model import LlavaLlamaForCausalLM
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from .model import LlavaLlamaForCausalLM
+    from llava.train.train import train  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from llava.train.train import train    
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from llava.train.train import trainTraceback (most recent call last):
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from .model import LlavaLlamaForCausalLM
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    Traceback (most recent call last):
+from .model import LlavaLlamaForCausalLM
+    from .model import LlavaLlamaForCausalLM
+    Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM    
+from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from llava.train.train import train  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from .model import LlavaLlamaForCausalLM
+from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+        from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM    
+
+from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+        from llava.train.train import trainfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM        
+
+from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+
+    from llava.train.train import train
+    Traceback (most recent call last):
+from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+        from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+
+        from .model import LlavaLlamaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+    
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from llava.train.train import train  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+
+    from llava.train.train import train  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+                from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+
+
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+            from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+
+
+    from .model import LlavaLlamaForCausalLM
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+from llava.train.train import train  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+
+from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+            from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM
+
+
+    from .model import LlavaLlamaForCausalLM
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+from .model import LlavaLlamaForCausalLM  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+        from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+        from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+        from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+        from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM    
+from .model import LlavaLlamaForCausalLM
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM    
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .model import LlavaLlamaForCausalLM
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+    from llava.train.train import train
+from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+            from .model import LlavaLlamaForCausalLMfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+from llava.train.train import train
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+from llava.train.train import train
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from llava.train.train import train    
+from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+        from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+        from llava.train.train import trainfrom llava.train.train import train
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+        from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM    
+
+from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+        from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+        from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom .model import LlavaLlamaForCausalLM
+
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+        from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM
+
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .model import LlavaLlamaForCausalLM  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+        from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+    from .model import LlavaLlamaForCausalLM
+        from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_ditfrom .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+Traceback (most recent call last):
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+        from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+        from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+        from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM
+
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+    from .model import LlavaLlamaForCausalLM
+        from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .model import LlavaLlamaForCausalLM  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+          File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from llava.train.train import train
+    from .model import LlavaLlamaForCausalLM  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+        from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+
+    from .model import LlavaLlamaForCausalLM
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+        from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+        from .model import LlavaLlamaForCausalLMfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from llava.train.train import train
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+        from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_ditfrom .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+        from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+        from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+        from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_ditfrom .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+        from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+        from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM
+    
+from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+from llava.train.train import train
+    from .model import LlavaLlamaForCausalLM
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from .model import LlavaLlamaForCausalLM
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig    
+from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM    
+from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+        from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+Traceback (most recent call last):
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+        from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+        from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig    
+from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from llava.train.train import train
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+        from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from llava.train.train import train
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from .model import LlavaLlamaForCausalLM
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+        from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_ditfrom .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+        from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM    
+from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+Traceback (most recent call last):
+                    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_ditfrom .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_ditfrom .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_ditfrom .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+
+
+
+
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from .model import LlavaLlamaForCausalLM
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+            from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+        from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_ditfrom .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+        from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+    from llava.train.train import train
+ImportErrorImportError: : cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+
+        from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_ditfrom .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+        from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+        from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+Traceback (most recent call last):
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from llava.train.train import train
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+Traceback (most recent call last):
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+Traceback (most recent call last):
+        from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from llava.train.train import train
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+        from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+            from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+
+    from llava.train.train import train
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit    
+from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+ImportErrorImportError: : cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)ImportErrorcannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+
+: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+    from .model import LlavaLlamaForCausalLM
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    ImportErrorfrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina: 
+cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+        from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn    
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+          File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+Traceback (most recent call last):
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn    
+from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+        from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+ImportErrorImportErrorImportError: : : cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+
+
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+    from llava.train.train import train
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+ImportError        : from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_luminacannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn    
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+        from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+
+ImportError: ImportErrorcannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py): 
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)    
+from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+    ImportErrorImportErrorfrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina: 
+cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py): 
+    cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+        from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+ImportErrorImportError: : cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+ImportError: ImportErrorcannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py): 
+cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    ImportErrorfrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)    
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+
+ImportError    : from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminacannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    ImportErrorfrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+
+        from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError
+: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)ImportError
+: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+ImportErrorImportErrorImportError: : : cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+
+
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn    
+from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+ImportError:     cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)ImportError
+from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina: 
+cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    ImportErrorfrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina: 
+cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)    
+from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    ImportErrorfrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    
+from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminaImportError
+: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError    : from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminacannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)ImportError
+
+: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminaImportError
+: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError    : ImportErrorcannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py): from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminaImportError
+: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)    
+from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+    ImportErrorfrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina: 
+cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)ImportError
+    : from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminacannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminaImportError
+: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)ImportError
+: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError    : from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminacannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+
+ImportError:     cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+    ImportErrorfrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina    : 
+from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminacannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+ImportError
+: ImportErrorcannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py): cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportErrorImportError: : cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+ImportError    : from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminacannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+
+ImportError:     cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+
+ImportError:     cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina    
+from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: ImportError    cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py):     
+cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+
+
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)    
+from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminaImportError
+: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from llava.train.train import train    
+from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+        from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+Traceback (most recent call last):
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from llava.train.train import train
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig    
+from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+Traceback (most recent call last):
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM    
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom .model import LlavaLlamaForCausalLM
+
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+        from llava.train.train import trainfrom llava.train.train import train
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from .model import LlavaLlamaForCausalLM
+          File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM
+
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from .model import LlavaLlamaForCausalLM
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+        from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+        from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_ditfrom .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+        from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminaImportError
+:     cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminaImportError
+: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina    
+from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportErrorImportError: : ImportErrorcannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+: 
+cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)ImportError
+: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+W0215 19:21:39.751000 140662069983040 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2642870 closing signal SIGTERM
+W0215 19:21:39.751000 140662069983040 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2642871 closing signal SIGTERM
+W0215 19:21:39.751000 140662069983040 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2642872 closing signal SIGTERM
+W0215 19:21:39.751000 140662069983040 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2642874 closing signal SIGTERM
+W0215 19:21:39.752000 140662069983040 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2642876 closing signal SIGTERM
+W0215 19:21:39.752000 140662069983040 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2642877 closing signal SIGTERM
+W0215 19:21:39.752000 128273080436544 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2895376 closing signal SIGTERM
+W0215 19:21:39.753000 128273080436544 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2895377 closing signal SIGTERM
+W0215 19:21:39.753000 128273080436544 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2895379 closing signal SIGTERM
+W0215 19:21:39.753000 128273080436544 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2895381 closing signal SIGTERM
+W0215 19:21:39.753000 128273080436544 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2895383 closing signal SIGTERM
+W0215 19:21:39.853000 123572905850688 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3238188 closing signal SIGTERM
+W0215 19:21:39.853000 123572905850688 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3238191 closing signal SIGTERM
+W0215 19:21:39.854000 123572905850688 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3238192 closing signal SIGTERM
+W0215 19:21:39.855000 123572905850688 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3238194 closing signal SIGTERM
+W0215 19:21:39.855000 123572905850688 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3238196 closing signal SIGTERM
+W0215 19:21:39.855000 123572905850688 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3238197 closing signal SIGTERM
+W0215 19:21:39.951000 135494447343424 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2905502 closing signal SIGTERM
+W0215 19:21:39.951000 135494447343424 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2905503 closing signal SIGTERM
+W0215 19:21:39.952000 135494447343424 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2905504 closing signal SIGTERM
+W0215 19:21:39.952000 135494447343424 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2905505 closing signal SIGTERM
+W0215 19:21:39.952000 135494447343424 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2905506 closing signal SIGTERM
+W0215 19:21:39.952000 135494447343424 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2905508 closing signal SIGTERM
+W0215 19:21:39.955000 129572943243072 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2640062 closing signal SIGTERM
+W0215 19:21:39.956000 129572943243072 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2640064 closing signal SIGTERM
+W0215 19:21:39.956000 129572943243072 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2640065 closing signal SIGTERM
+W0215 19:21:39.956000 129572943243072 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2640066 closing signal SIGTERM
+W0215 19:21:39.956000 129572943243072 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2640067 closing signal SIGTERM
+W0215 19:21:39.957000 129572943243072 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2640068 closing signal SIGTERM
+W0215 19:21:39.958000 129572943243072 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2640069 closing signal SIGTERM
+W0215 19:21:40.152000 138557481989952 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2640103 closing signal SIGTERM
+W0215 19:21:40.152000 138557481989952 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2640104 closing signal SIGTERM
+W0215 19:21:40.152000 138557481989952 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2640106 closing signal SIGTERM
+W0215 19:21:40.152000 138557481989952 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2640107 closing signal SIGTERM
+W0215 19:21:40.152000 138557481989952 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2640108 closing signal SIGTERM
+W0215 19:21:40.152000 138557481989952 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2640109 closing signal SIGTERM
+W0215 19:21:40.154000 126876645988160 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3174001 closing signal SIGTERM
+W0215 19:21:40.154000 126876645988160 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3174002 closing signal SIGTERM
+W0215 19:21:40.154000 126876645988160 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3174003 closing signal SIGTERM
+W0215 19:21:40.154000 126876645988160 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3174004 closing signal SIGTERM
+W0215 19:21:40.154000 126876645988160 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3174006 closing signal SIGTERM
+W0215 19:21:40.154000 126876645988160 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3174007 closing signal SIGTERM
+W0215 19:21:40.154000 126876645988160 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3174008 closing signal SIGTERM
+W0215 19:21:40.253000 138510042982208 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2904321 closing signal SIGTERM
+W0215 19:21:40.253000 126469684954944 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2637522 closing signal SIGTERM
+W0215 19:21:40.253000 138510042982208 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2904325 closing signal SIGTERM
+W0215 19:21:40.253000 126469684954944 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2637523 closing signal SIGTERM
+W0215 19:21:40.253000 138510042982208 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2904326 closing signal SIGTERM
+W0215 19:21:40.253000 138510042982208 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2904327 closing signal SIGTERM
+W0215 19:21:40.253000 126469684954944 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2637524 closing signal SIGTERM
+W0215 19:21:40.253000 126469684954944 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2637525 closing signal SIGTERM
+W0215 19:21:40.254000 126469684954944 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2637528 closing signal SIGTERM
+W0215 19:21:40.254000 126469684954944 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2637529 closing signal SIGTERM
+W0215 19:21:40.350000 127054241253184 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2645750 closing signal SIGTERM
+W0215 19:21:40.350000 127054241253184 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2645751 closing signal SIGTERM
+W0215 19:21:40.351000 127054241253184 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2645752 closing signal SIGTERM
+W0215 19:21:40.351000 127054241253184 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2645753 closing signal SIGTERM
+W0215 19:21:40.351000 127054241253184 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2645754 closing signal SIGTERM
+W0215 19:21:40.351000 127054241253184 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2645755 closing signal SIGTERM
+W0215 19:21:40.351000 127054241253184 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2645756 closing signal SIGTERM
+W0215 19:21:40.451000 133066830354240 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2636371 closing signal SIGTERM
+W0215 19:21:40.452000 133066830354240 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2636372 closing signal SIGTERM
+W0215 19:21:40.452000 133066830354240 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2636373 closing signal SIGTERM
+W0215 19:21:40.452000 133066830354240 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2636374 closing signal SIGTERM
+W0215 19:21:40.452000 133066830354240 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2636375 closing signal SIGTERM
+W0215 19:21:40.452000 134808981854016 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2907025 closing signal SIGTERM
+W0215 19:21:40.452000 133066830354240 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2636376 closing signal SIGTERM
+W0215 19:21:40.452000 134808981854016 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2907026 closing signal SIGTERM
+W0215 19:21:40.452000 133066830354240 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2636378 closing signal SIGTERM
+W0215 19:21:40.452000 134808981854016 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2907027 closing signal SIGTERM
+W0215 19:21:40.452000 134808981854016 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2907028 closing signal SIGTERM
+W0215 19:21:40.452000 134808981854016 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2907030 closing signal SIGTERM
+W0215 19:21:40.454000 128980037068608 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3291113 closing signal SIGTERM
+W0215 19:21:40.455000 128980037068608 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3291114 closing signal SIGTERM
+W0215 19:21:40.456000 128980037068608 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3291115 closing signal SIGTERM
+W0215 19:21:40.456000 128980037068608 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3291116 closing signal SIGTERM
+W0215 19:21:40.456000 128980037068608 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3291119 closing signal SIGTERM
+W0215 19:21:40.655000 131517812614976 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3241630 closing signal SIGTERM
+W0215 19:21:40.655000 131517812614976 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3241632 closing signal SIGTERM
+W0215 19:21:40.655000 131517812614976 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3241633 closing signal SIGTERM
+W0215 19:21:40.656000 131517812614976 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3241635 closing signal SIGTERM
+W0215 19:21:40.656000 131517812614976 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3241636 closing signal SIGTERM
+W0215 19:21:40.656000 131517812614976 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3241637 closing signal SIGTERM
+W0215 19:21:40.659000 130231383332672 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3290568 closing signal SIGTERM
+W0215 19:21:40.660000 130231383332672 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3290569 closing signal SIGTERM
+W0215 19:21:40.661000 130231383332672 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3290570 closing signal SIGTERM
+W0215 19:21:40.661000 130231383332672 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3290571 closing signal SIGTERM
+W0215 19:21:40.661000 130231383332672 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3290572 closing signal SIGTERM
+W0215 19:21:40.662000 130231383332672 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3290574 closing signal SIGTERM
+W0215 19:21:40.858000 131641266206528 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2899408 closing signal SIGTERM
+W0215 19:21:40.859000 131641266206528 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2899409 closing signal SIGTERM
+W0215 19:21:40.859000 131641266206528 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2899410 closing signal SIGTERM
+W0215 19:21:40.859000 131641266206528 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2899411 closing signal SIGTERM
+W0215 19:21:40.860000 131641266206528 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2899412 closing signal SIGTERM
+W0215 19:21:40.860000 131641266206528 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2899413 closing signal SIGTERM
+W0215 19:21:40.860000 131641266206528 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2899414 closing signal SIGTERM
+E0215 19:21:41.660000 135494447343424 torch/distributed/elastic/multiprocessing/api.py:833] failed (exitcode: 1) local_rank: 0 (pid: 2905501) of binary: /data/home/zhaojiang/miniconda3/envs/test/bin/python3.11
+Traceback (most recent call last):
+  File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in <module>
+    sys.exit(main())
+             ^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper
+    return f(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main
+    run(args)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run
+    elastic_launch(
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__
+    return launch_agent(self._config, self._entrypoint, list(args))
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 264, in launch_agent
+    raise ChildFailedError(
+torch.distributed.elastic.multiprocessing.errors.ChildFailedError: 
+============================================================
+llava/train/train_mem.py FAILED
+------------------------------------------------------------
+Failures:
+[1]:
+  time      : 2025-02-15_19:21:39
+  host      : h100-st-p548xlarge-188.ar-ai-use2.hpcaas
+  rank      : 94 (local_rank: 6)
+  exitcode  : 1 (pid: 2905507)
+  error_file: <N/A>
+  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
+------------------------------------------------------------
+Root Cause (first observed failure):
+[0]:
+  time      : 2025-02-15_19:21:39
+  host      : h100-st-p548xlarge-188.ar-ai-use2.hpcaas
+  rank      : 88 (local_rank: 0)
+  exitcode  : 1 (pid: 2905501)
+  error_file: <N/A>
+  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
+============================================================
+E0215 19:21:41.713000 138557481989952 torch/distributed/elastic/multiprocessing/api.py:833] failed (exitcode: 1) local_rank: 2 (pid: 2640105) of binary: /data/home/zhaojiang/miniconda3/envs/test/bin/python3.11
+Traceback (most recent call last):
+  File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in <module>
+    sys.exit(main())
+             ^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper
+E0215 19:21:41.715000 126876645988160 torch/distributed/elastic/multiprocessing/api.py:833] failed (exitcode: 1) local_rank: 4 (pid: 3174005) of binary: /data/home/zhaojiang/miniconda3/envs/test/bin/python3.11
+    return f(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main
+Traceback (most recent call last):
+  File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in <module>
+    sys.exit(main())
+             ^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper
+    run(args)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run
+    return f(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main
+    elastic_launch(
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__
+    return launch_agent(self._config, self._entrypoint, list(args))
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 264, in launch_agent
+    run(args)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run
+    raise ChildFailedError(
+torch.distributed.elastic.multiprocessing.errors.ChildFailedError: 
+============================================================
+llava/train/train_mem.py FAILED
+------------------------------------------------------------
+Failures:
+[1]:
+  time      : 2025-02-15_19:21:40
+  host      : h100-st-p548xlarge-74.ar-ai-use2.hpcaas
+  rank      : 39 (local_rank: 7)
+  exitcode  : 1 (pid: 2640110)
+  error_file: <N/A>
+  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
+------------------------------------------------------------
+Root Cause (first observed failure):
+[0]:
+  time      : 2025-02-15_19:21:40
+  host      : h100-st-p548xlarge-74.ar-ai-use2.hpcaas
+  rank      : 34 (local_rank: 2)
+  exitcode  : 1 (pid: 2640105)
+  error_file: <N/A>
+  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
+============================================================
+    elastic_launch(
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__
+    return launch_agent(self._config, self._entrypoint, list(args))
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 264, in launch_agent
+    raise ChildFailedError(
+torch.distributed.elastic.multiprocessing.errors.ChildFailedError: 
+============================================================
+llava/train/train_mem.py FAILED
+------------------------------------------------------------
+Failures:
+  <NO_OTHER_FAILURES>
+------------------------------------------------------------
+Root Cause (first observed failure):
+[0]:
+  time      : 2025-02-15_19:21:40
+  host      : h100-st-p548xlarge-145.ar-ai-use2.hpcaas
+  rank      : 84 (local_rank: 4)
+  exitcode  : 1 (pid: 3174005)
+  error_file: <N/A>
+  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
+============================================================
+E0215 19:21:41.876000 133066830354240 torch/distributed/elastic/multiprocessing/api.py:833] failed (exitcode: 1) local_rank: 6 (pid: 2636377) of binary: /data/home/zhaojiang/miniconda3/envs/test/bin/python3.11
+Traceback (most recent call last):
+  File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in <module>
+    sys.exit(main())
+             ^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper
+    return f(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main
+    run(args)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run
+    elastic_launch(
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__
+    return launch_agent(self._config, self._entrypoint, list(args))
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 264, in launch_agent
+    raise ChildFailedError(
+torch.distributed.elastic.multiprocessing.errors.ChildFailedError: 
+============================================================
+llava/train/train_mem.py FAILED
+------------------------------------------------------------
+Failures:
+  <NO_OTHER_FAILURES>
+------------------------------------------------------------
+Root Cause (first observed failure):
+[0]:
+  time      : 2025-02-15_19:21:40
+  host      : h100-st-p548xlarge-75.ar-ai-use2.hpcaas
+  rank      : 46 (local_rank: 6)
+  exitcode  : 1 (pid: 2636377)
+  error_file: <N/A>
+  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
+============================================================
+E0215 19:21:41.921000 128273080436544 torch/distributed/elastic/multiprocessing/api.py:833] failed (exitcode: 1) local_rank: 2 (pid: 2895378) of binary: /data/home/zhaojiang/miniconda3/envs/test/bin/python3.11
+Traceback (most recent call last):
+  File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in <module>
+    sys.exit(main())
+             ^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper
+    return f(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main
+    run(args)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run
+    elastic_launch(
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__
+    return launch_agent(self._config, self._entrypoint, list(args))
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 264, in launch_agent
+    raise ChildFailedError(
+torch.distributed.elastic.multiprocessing.errors.ChildFailedError: 
+============================================================
+llava/train/train_mem.py FAILED
+------------------------------------------------------------
+Failures:
+[1]:
+  time      : 2025-02-15_19:21:39
+  host      : h100-st-p548xlarge-113.ar-ai-use2.hpcaas
+  rank      : 60 (local_rank: 4)
+  exitcode  : 1 (pid: 2895380)
+  error_file: <N/A>
+  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
+[2]:
+  time      : 2025-02-15_19:21:39
+  host      : h100-st-p548xlarge-113.ar-ai-use2.hpcaas
+  rank      : 62 (local_rank: 6)
+  exitcode  : 1 (pid: 2895382)
+  error_file: <N/A>
+  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
+------------------------------------------------------------
+Root Cause (first observed failure):
+[0]:
+  time      : 2025-02-15_19:21:39
+  host      : h100-st-p548xlarge-113.ar-ai-use2.hpcaas
+  rank      : 58 (local_rank: 2)
+  exitcode  : 1 (pid: 2895378)
+  error_file: <N/A>
+  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
+============================================================
+srun: error: h100-st-p548xlarge-188: task 11: Exited with exit code 1
+srun: Terminating StepId=335101.0
+slurmstepd: error: *** STEP 335101.0 ON h100-st-p548xlarge-44 CANCELLED AT 2025-02-15T19:21:41 ***
+W0215 19:21:42.000000 127054241253184 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers
+W0215 19:21:42.000000 127054241253184 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2645752 closing signal SIGTERM
+W0215 19:21:42.000000 127054241253184 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2645755 closing signal SIGTERM
+W0215 19:21:42.000000 131517812614976 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers
+W0215 19:21:42.000000 131641266206528 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers
+W0215 19:21:42.001000 131517812614976 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3241633 closing signal SIGTERM
+W0215 19:21:42.000000 140662069983040 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers
+W0215 19:21:42.001000 131641266206528 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2899408 closing signal SIGTERM
+W0215 19:21:42.001000 140662069983040 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2642874 closing signal SIGTERM
+W0215 19:21:42.000000 130231383332672 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers
+W0215 19:21:42.001000 131641266206528 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2899409 closing signal SIGTERM
+W0215 19:21:42.000000 123572905850688 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers
+W0215 19:21:42.000000 126469684954944 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers
+W0215 19:21:42.001000 130231383332672 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3290569 closing signal SIGTERM
+W0215 19:21:42.000000 128980037068608 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers
+W0215 19:21:42.001000 123572905850688 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3238188 closing signal SIGTERM
+W0215 19:21:42.001000 129572943243072 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers
+W0215 19:21:42.001000 126469684954944 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2637528 closing signal SIGTERM
+W0215 19:21:42.001000 129572943243072 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2640062 closing signal SIGTERM
+W0215 19:21:42.001000 128980037068608 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3291113 closing signal SIGTERM
+W0215 19:21:42.001000 134808981854016 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers
+W0215 19:21:42.001000 129572943243072 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2640064 closing signal SIGTERM
+W0215 19:21:42.001000 134808981854016 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2907026 closing signal SIGTERM
+W0215 19:21:42.001000 138510042982208 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers
+W0215 19:21:42.001000 130231383332672 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3290574 closing signal SIGTERM
+W0215 19:21:42.001000 138510042982208 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2904325 closing signal SIGTERM
+W0215 19:21:42.001000 129572943243072 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2640069 closing signal SIGTERM
+srun: error: h100-st-p548xlarge-145: task 10: Exited with exit code 1
+srun: error: h100-st-p548xlarge-74: task 4: Exited with exit code 1
+Traceback (most recent call last):
+  File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in <module>
+Traceback (most recent call last):
+  File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in <module>
+    sys.exit(main())
+             ^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper
+    sys.exit(main())
+             ^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper
+    return f(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main
+    return f(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main
+    run(args)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run
+    run(args)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run
+    elastic_launch(
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__
+    elastic_launch(
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__
+    return launch_agent(self._config, self._entrypoint, list(args))
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent
+    result = agent.run()
+             ^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper
+    return launch_agent(self._config, self._entrypoint, list(args))
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent
+    result = agent.run()
+             ^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper
+    result = f(*args, **kwargs)
+             ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run
+    result = f(*args, **kwargs)
+             ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run
+    result = self._invoke_run(role)
+             ^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run
+    result = self._invoke_run(role)
+             ^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run
+    run_result = self._monitor_workers(self._worker_group)
+                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper
+    run_result = self._monitor_workers(self._worker_group)
+    result = f(*args, **kwargs)
+                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper
+             ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers
+    result = f(*args, **kwargs)
+             ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers
+    result = self._pcontext.wait(0)
+             ^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait
+    result = self._pcontext.wait(0)
+             ^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait
+    return self._poll()
+           ^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll
+    return self._poll()
+           ^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll
+    self.close()  # terminate all running procs
+    ^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close
+    self._close(death_sig=death_sig, timeout=timeout)
+    self.close()  # terminate all running procs
+    ^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close
+    self._close(death_sig=death_sig, timeout=timeout)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close
+    handler.proc.wait(time_to_wait)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close
+    handler.proc.wait(time_to_wait)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait
+    return self._wait(timeout=timeout)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait
+    return self._wait(timeout=timeout)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait
+    time.sleep(delay)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler
+    raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
+torch.distributed.elastic.multiprocessing.api.SignalException: Process 2904238 got signal: 15
+    time.sleep(delay)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler
+    raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
+torch.distributed.elastic.multiprocessing.api.SignalException: Process 2642154 got signal: 15
+Traceback (most recent call last):
+  File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in <module>
+    sys.exit(main())
+             ^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper
+    return f(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main
+    run(args)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run
+    elastic_launch(
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__
+    return launch_agent(self._config, self._entrypoint, list(args))
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent
+    result = agent.run()
+             ^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper
+    result = f(*args, **kwargs)
+             ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run
+    result = self._invoke_run(role)
+             ^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run
+    run_result = self._monitor_workers(self._worker_group)
+                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper
+    result = f(*args, **kwargs)
+             ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers
+    result = self._pcontext.wait(0)
+             ^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait
+    return self._poll()
+           ^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll
+    self.close()  # terminate all running procs
+    ^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close
+    self._close(death_sig=death_sig, timeout=timeout)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close
+    handler.proc.wait(time_to_wait)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait
+    return self._wait(timeout=timeout)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait
+    time.sleep(delay)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler
+    raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
+torch.distributed.elastic.multiprocessing.api.SignalException: Process 2645034 got signal: 15
+srun: error: h100-st-p548xlarge-113: task 7: Terminated
+srun: error: h100-st-p548xlarge-75: task 5: Terminated
+Traceback (most recent call last):
+  File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in <module>
+    sys.exit(main())
+             ^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper
+    return f(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main
+    run(args)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run
+    elastic_launch(
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__
+    return launch_agent(self._config, self._entrypoint, list(args))
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent
+    result = agent.run()
+             ^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper
+    result = f(*args, **kwargs)
+             ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run
+    result = self._invoke_run(role)
+             ^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run
+    run_result = self._monitor_workers(self._worker_group)
+                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper
+    result = f(*args, **kwargs)
+             ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers
+    result = self._pcontext.wait(0)
+             ^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait
+    return self._poll()
+           ^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll
+    self.close()  # terminate all running procs
+    ^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close
+    self._close(death_sig=death_sig, timeout=timeout)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close
+    handler.proc.wait(time_to_wait)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait
+    return self._wait(timeout=timeout)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait
+    time.sleep(delay)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler
+    raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
+torch.distributed.elastic.multiprocessing.api.SignalException: Process 3291031 got signal: 15
+Traceback (most recent call last):
+  File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in <module>
+    sys.exit(main())
+Traceback (most recent call last):
+  File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in <module>
+             ^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper
+    sys.exit(main())
+             ^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper
+    return f(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main
+    return f(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main
+    run(args)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run
+    run(args)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run
+    elastic_launch(
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__
+    elastic_launch(
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__
+    return launch_agent(self._config, self._entrypoint, list(args))
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent
+    result = agent.run()
+             ^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper
+    return launch_agent(self._config, self._entrypoint, list(args))
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent
+    result = agent.run()
+             ^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper
+    result = f(*args, **kwargs)
+             ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run
+    result = f(*args, **kwargs)
+             ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run
+    result = self._invoke_run(role)
+             ^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run
+    run_result = self._monitor_workers(self._worker_group)
+                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper
+    result = self._invoke_run(role)
+    result = f(*args, **kwargs)
+             ^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run
+             ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers
+    run_result = self._monitor_workers(self._worker_group)
+                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper
+    result = f(*args, **kwargs)
+             ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers
+    result = self._pcontext.wait(0)
+             ^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait
+    result = self._pcontext.wait(0)
+             ^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait
+    return self._poll()
+           ^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll
+    self.close()  # terminate all running procs
+    ^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close
+    return self._poll()
+    self._close(death_sig=death_sig, timeout=timeout)
+           ^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close
+    self.close()  # terminate all running procs
+    handler.proc.wait(time_to_wait)
+    ^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait
+    self._close(death_sig=death_sig, timeout=timeout)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close
+    handler.proc.wait(time_to_wait)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait
+    return self._wait(timeout=timeout)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait
+    time.sleep(delay)
+    return self._wait(timeout=timeout)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait
+    raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
+torch.distributed.elastic.multiprocessing.api.SignalException: Process 2636799 got signal: 15
+    time.sleep(delay)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler
+    raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
+torch.distributed.elastic.multiprocessing.api.SignalException: Process 3238106 got signal: 15
+srun: error: h100-st-p548xlarge-62: task 3: Exited with exit code 1
+srun: error: h100-st-p548xlarge-189: task 12: Exited with exit code 1
+srun: error: h100-st-p548xlarge-44: task 0: Exited with exit code 1
+srun: error: h100-st-p548xlarge-199: task 13: Exited with exit code 1
+Traceback (most recent call last):
+  File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in <module>
+    sys.exit(main())
+             ^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper
+    return f(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main
+    run(args)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run
+    elastic_launch(
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__
+    return launch_agent(self._config, self._entrypoint, list(args))
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent
+    result = agent.run()
+             ^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper
+    result = f(*args, **kwargs)
+             ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run
+    result = self._invoke_run(role)
+             ^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run
+    run_result = self._monitor_workers(self._worker_group)
+                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper
+    result = f(*args, **kwargs)
+             ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers
+    result = self._pcontext.wait(0)
+             ^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait
+srun: error: h100-st-p548xlarge-143: task 8: Exited with exit code 1
+    return self._poll()
+           ^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll
+    self.close()  # terminate all running procs
+    ^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close
+    self._close(death_sig=death_sig, timeout=timeout)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close
+    handler.proc.wait(time_to_wait)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait
+    return self._wait(timeout=timeout)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait
+    time.sleep(delay)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler
+    raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
+torch.distributed.elastic.multiprocessing.api.SignalException: Process 3241553 got signal: 15
+srun: error: h100-st-p548xlarge-61: task 2: Exited with exit code 1
+Traceback (most recent call last):
+  File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in <module>
+    sys.exit(main())
+             ^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper
+    return f(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main
+    run(args)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run
+    elastic_launch(
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__
+    return launch_agent(self._config, self._entrypoint, list(args))
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent
+    result = agent.run()
+             ^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper
+    result = f(*args, **kwargs)
+             ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run
+    result = self._invoke_run(role)
+             ^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run
+    run_result = self._monitor_workers(self._worker_group)
+                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper
+    result = f(*args, **kwargs)
+             ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers
+    result = self._pcontext.wait(0)
+             ^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait
+    return self._poll()
+           ^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll
+    self.close()  # terminate all running procs
+    ^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close
+    self._close(death_sig=death_sig, timeout=timeout)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close
+    handler.proc.wait(time_to_wait)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait
+    return self._wait(timeout=timeout)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait
+    time.sleep(delay)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler
+    raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
+torch.distributed.elastic.multiprocessing.api.SignalException: Process 2906901 got signal: 15
+Traceback (most recent call last):
+  File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in <module>
+    sys.exit(main())
+             ^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper
+    return f(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main
+    run(args)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run
+    elastic_launch(
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__
+    return launch_agent(self._config, self._entrypoint, list(args))
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent
+    result = agent.run()
+             ^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper
+    result = f(*args, **kwargs)
+             ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run
+    result = self._invoke_run(role)
+             ^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run
+    run_result = self._monitor_workers(self._worker_group)
+                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper
+    result = f(*args, **kwargs)
+             ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers
+    result = self._pcontext.wait(0)
+             ^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait
+    return self._poll()
+           ^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll
+    self.close()  # terminate all running procs
+    ^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close
+    self._close(death_sig=death_sig, timeout=timeout)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close
+    handler.proc.wait(time_to_wait)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait
+    return self._wait(timeout=timeout)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait
+    time.sleep(delay)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler
+    raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
+torch.distributed.elastic.multiprocessing.api.SignalException: Process 2639198 got signal: 15
+Traceback (most recent call last):
+  File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in <module>
+    sys.exit(main())
+             ^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper
+    return f(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main
+    run(args)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run
+    elastic_launch(
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__
+    return launch_agent(self._config, self._entrypoint, list(args))
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent
+    result = agent.run()
+             ^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper
+    result = f(*args, **kwargs)
+             ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run
+    result = self._invoke_run(role)
+             ^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run
+    run_result = self._monitor_workers(self._worker_group)
+                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper
+    result = f(*args, **kwargs)
+             ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers
+    result = self._pcontext.wait(0)
+             ^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait
+    return self._poll()
+           ^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll
+    self.close()  # terminate all running procs
+    ^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close
+    self._close(death_sig=death_sig, timeout=timeout)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close
+    handler.proc.wait(time_to_wait)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait
+    return self._wait(timeout=timeout)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait
+    time.sleep(delay)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler
+    raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
+torch.distributed.elastic.multiprocessing.api.SignalException: Process 2899328 got signal: 15
+Traceback (most recent call last):
+  File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in <module>
+    sys.exit(main())
+             ^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper
+    return f(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main
+    run(args)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run
+    elastic_launch(
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__
+    return launch_agent(self._config, self._entrypoint, list(args))
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent
+    result = agent.run()
+             ^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper
+    result = f(*args, **kwargs)
+             ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run
+    result = self._invoke_run(role)
+             ^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run
+    run_result = self._monitor_workers(self._worker_group)
+                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper
+    result = f(*args, **kwargs)
+             ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers
+    result = self._pcontext.wait(0)
+             ^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait
+    return self._poll()
+           ^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll
+    self.close()  # terminate all running procs
+    ^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close
+    self._close(death_sig=death_sig, timeout=timeout)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close
+    handler.proc.wait(time_to_wait)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait
+    return self._wait(timeout=timeout)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait
+    time.sleep(delay)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler
+    raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
+torch.distributed.elastic.multiprocessing.api.SignalException: Process 3289853 got signal: 15
+srun: error: h100-st-p548xlarge-144: task 9: Exited with exit code 1
+srun: error: h100-st-p548xlarge-45: task 1: Exited with exit code 1
+srun: error: h100-st-p548xlarge-201: task 15: Exited with exit code 1
+srun: error: h100-st-p548xlarge-200: task 14: Exited with exit code 1
+srun: error: h100-st-p548xlarge-112: task 6: Exited with exit code 1
+srun: Force Terminated StepId=335101.0
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/upload_pretrain.py", line 16, in <module>
+    upload_folder(
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/huggingface_hub/utils/_validators.py", line 114, in _inner_fn
+    return fn(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/huggingface_hub/hf_api.py", line 1398, in _inner
+    return fn(self, *args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/huggingface_hub/hf_api.py", line 4824, in upload_folder
+    add_operations = _prepare_upload_folder_additions(
+                     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/huggingface_hub/hf_api.py", line 9227, in _prepare_upload_folder_additions
+    raise ValueError(f"Provided path: '{folder_path}' is not a directory")
+ValueError: Provided path: '/fsx-project/xichenpan/PISCES/latent_query_ar_20241220' is not a directory
+
Reformatting /fsx_0/user/zhaojiang/data/ShareGPT4V/blip_laion_cc_sbu_558k.json:   0%|          | 0/558128 [00:00<?, ?it/s]
Reformatting /fsx_0/user/zhaojiang/data/ShareGPT4V/blip_laion_cc_sbu_558k.json:  14%|█▍        | 79907/558128 [00:00<00:00, 798488.22it/s]
Reformatting /fsx_0/user/zhaojiang/data/ShareGPT4V/blip_laion_cc_sbu_558k.json:  29%|██▉       | 163403/558128 [00:00<00:00, 819907.49it/s]
Reformatting /fsx_0/user/zhaojiang/data/ShareGPT4V/blip_laion_cc_sbu_558k.json:  44%|████▍     | 246535/558128 [00:00<00:00, 825093.85it/s]
Reformatting /fsx_0/user/zhaojiang/data/ShareGPT4V/blip_laion_cc_sbu_558k.json:  59%|█████▉    | 330090/558128 [00:00<00:00, 829199.75it/s]
Reformatting /fsx_0/user/zhaojiang/data/ShareGPT4V/blip_laion_cc_sbu_558k.json:  74%|███████▍  | 413011/558128 [00:00<00:00, 826867.22it/s]
Reformatting /fsx_0/user/zhaojiang/data/ShareGPT4V/blip_laion_cc_sbu_558k.json:  89%|████████▉ | 496757/558128 [00:00<00:00, 830441.95it/s]
Reformatting /fsx_0/user/zhaojiang/data/ShareGPT4V/blip_laion_cc_sbu_558k.json: 100%|██████████| 558128/558128 [00:00<00:00, 825085.04it/s]
+W0215 19:27:10.909000 126842061190976 torch/distributed/run.py:779] 
+W0215 19:27:10.909000 126842061190976 torch/distributed/run.py:779] *****************************************
+W0215 19:27:10.909000 126842061190976 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0215 19:27:10.909000 126842061190976 torch/distributed/run.py:779] *****************************************
+W0215 19:27:10.937000 139528338814784 torch/distributed/run.py:779] 
+W0215 19:27:10.937000 139528338814784 torch/distributed/run.py:779] *****************************************
+W0215 19:27:10.937000 139528338814784 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0215 19:27:10.937000 139528338814784 torch/distributed/run.py:779] *****************************************
+W0215 19:27:10.939000 127715067463488 torch/distributed/run.py:779] 
+W0215 19:27:10.939000 127715067463488 torch/distributed/run.py:779] *****************************************
+W0215 19:27:10.939000 127715067463488 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0215 19:27:10.939000 127715067463488 torch/distributed/run.py:779] *****************************************
+W0215 19:27:10.940000 129994478602048 torch/distributed/run.py:779] 
+W0215 19:27:10.940000 129994478602048 torch/distributed/run.py:779] *****************************************
+W0215 19:27:10.940000 129994478602048 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0215 19:27:10.940000 129994478602048 torch/distributed/run.py:779] *****************************************
+W0215 19:27:10.959000 136256322365248 torch/distributed/run.py:779] 
+W0215 19:27:10.959000 136256322365248 torch/distributed/run.py:779] *****************************************
+W0215 19:27:10.959000 136256322365248 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0215 19:27:10.959000 136256322365248 torch/distributed/run.py:779] *****************************************
+W0215 19:27:10.961000 126996985308992 torch/distributed/run.py:779] 
+W0215 19:27:10.961000 126996985308992 torch/distributed/run.py:779] *****************************************
+W0215 19:27:10.961000 126996985308992 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0215 19:27:10.961000 126996985308992 torch/distributed/run.py:779] *****************************************
+W0215 19:27:10.966000 134743156152128 torch/distributed/run.py:779] 
+W0215 19:27:10.966000 134743156152128 torch/distributed/run.py:779] *****************************************
+W0215 19:27:10.966000 134743156152128 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0215 19:27:10.966000 134743156152128 torch/distributed/run.py:779] *****************************************
+W0215 19:27:11.004000 138343381481280 torch/distributed/run.py:779] 
+W0215 19:27:11.004000 138343381481280 torch/distributed/run.py:779] *****************************************
+W0215 19:27:11.004000 138343381481280 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0215 19:27:11.004000 138343381481280 torch/distributed/run.py:779] *****************************************
+W0215 19:27:11.025000 124287080048448 torch/distributed/run.py:779] 
+W0215 19:27:11.025000 124287080048448 torch/distributed/run.py:779] *****************************************
+W0215 19:27:11.025000 124287080048448 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0215 19:27:11.025000 124287080048448 torch/distributed/run.py:779] *****************************************
+W0215 19:27:11.075000 123909406807872 torch/distributed/run.py:779] 
+W0215 19:27:11.075000 123909406807872 torch/distributed/run.py:779] *****************************************
+W0215 19:27:11.075000 123909406807872 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0215 19:27:11.075000 123909406807872 torch/distributed/run.py:779] *****************************************
+W0215 19:27:11.128000 140227751167808 torch/distributed/run.py:779] 
+W0215 19:27:11.128000 140227751167808 torch/distributed/run.py:779] *****************************************
+W0215 19:27:11.128000 140227751167808 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0215 19:27:11.128000 140227751167808 torch/distributed/run.py:779] *****************************************
+W0215 19:27:11.165000 126092689319744 torch/distributed/run.py:779] 
+W0215 19:27:11.165000 126092689319744 torch/distributed/run.py:779] *****************************************
+W0215 19:27:11.165000 126092689319744 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0215 19:27:11.165000 126092689319744 torch/distributed/run.py:779] *****************************************
+W0215 19:27:11.173000 127678089664320 torch/distributed/run.py:779] 
+W0215 19:27:11.173000 127678089664320 torch/distributed/run.py:779] *****************************************
+W0215 19:27:11.173000 127678089664320 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0215 19:27:11.173000 127678089664320 torch/distributed/run.py:779] *****************************************
+W0215 19:27:11.237000 130310563845952 torch/distributed/run.py:779] 
+W0215 19:27:11.237000 130310563845952 torch/distributed/run.py:779] *****************************************
+W0215 19:27:11.237000 130310563845952 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0215 19:27:11.237000 130310563845952 torch/distributed/run.py:779] *****************************************
+W0215 19:27:11.241000 135383820547904 torch/distributed/run.py:779] 
+W0215 19:27:11.241000 135383820547904 torch/distributed/run.py:779] *****************************************
+W0215 19:27:11.241000 135383820547904 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0215 19:27:11.241000 135383820547904 torch/distributed/run.py:779] *****************************************
+W0215 19:27:11.378000 137783691786048 torch/distributed/run.py:779] 
+W0215 19:27:11.378000 137783691786048 torch/distributed/run.py:779] *****************************************
+W0215 19:27:11.378000 137783691786048 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0215 19:27:11.378000 137783691786048 torch/distributed/run.py:779] *****************************************
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from llava.train.train import train    
+from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from .model import LlavaLlamaForCausalLM
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+        Traceback (most recent call last):
+from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from llava.train.train import train
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+        from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+        from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+        from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+
+    ImportErrorImportErrorfrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina: : 
+cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from llava.train.train import train
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from .model import LlavaLlamaForCausalLM
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+        from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit    
+from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit    
+from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+        from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    
+from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+        from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+
+ImportErrorImportError: : cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from llava.train.train import train
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from llava.train.train import train
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from .model import LlavaLlamaForCausalLM
+    from llava.train.train import train  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+Traceback (most recent call last):
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .model import LlavaLlamaForCausalLM
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from llava.train.train import train      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+
+from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from llava.train.train import train
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+        from .model import LlavaLlamaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+        from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+        from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+Traceback (most recent call last):
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+        from llava.train.train import trainfrom llava.train.train import train
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+        from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig    
+from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM    
+from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+Traceback (most recent call last):
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from llava.train.train import train
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn    
+from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina    
+from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+Traceback (most recent call last):
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+ImportErrorImportError: : cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from llava.train.train import train
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+Traceback (most recent call last):
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+    from llava.train.train import train
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit    
+from llava.train.train import train
+        from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+        from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+
+    from .model import LlavaLlamaForCausalLM  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+        from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+        from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_ditfrom .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+        from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+ImportError: ImportErrorcannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py): 
+cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+        from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+        from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+
+    ImportErrorImportErrorfrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina: : 
+cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from llava.train.train import train
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .model import LlavaLlamaForCausalLM
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+    from .model import LlavaLlamaForCausalLM  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+        from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+Traceback (most recent call last):
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+        from llava.train.train import trainfrom llava.train.train import train
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+        from llava.train.train import train
+from llava.train.train import train
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+        from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+        from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+        from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+        from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+        from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+        from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+        from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+
+ImportErrorImportError: : cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from llava.train.train import train
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+from .model import LlavaLlamaForCausalLM
+Traceback (most recent call last):
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+    from llava.train.train import train  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+        from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .model import LlavaLlamaForCausalLM
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+        from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+        from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+        from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+        from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+        from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_ditfrom .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+        from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_ditfrom .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina    
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: ImportErrorcannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+        from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+
+        from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+
+ImportErrorImportError: : cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+
+ImportErrorImportError: : cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_fwd")
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from llava.train.train import train
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+Traceback (most recent call last):
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from llava.train.train import train
+        from llava.train.train import trainfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+
+    from llava.train.train import train
+        from llava.train.train import train  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+        from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM
+
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+from .model import LlavaLlamaForCausalLM  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+
+    from .model import LlavaLlamaForCausalLM  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from llava.train.train import train
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit    
+from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError    : from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminacannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from llava.train.train import train
+Traceback (most recent call last):
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from llava.train.train import train
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from .model import LlavaLlamaForCausalLM
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from .model import LlavaLlamaForCausalLM
+        from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .model import LlavaLlamaForCausalLM
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+          File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+        from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+        from llava.train.train import trainfrom llava.train.train import train
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+        from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+        from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+        from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+        from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_ditfrom .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+        from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+        from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+
+ImportErrorImportError: : cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+Traceback (most recent call last):
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from llava.train.train import train
+    from llava.train.train import train
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from .model import LlavaLlamaForCausalLM    
+from .model import LlavaLlamaForCausalLM
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+            from .model import LlavaLlamaForCausalLMfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+
+
+from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+            from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+        from llava.train.train import trainfrom llava.train.train import train
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+        from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .model import LlavaLlamaForCausalLM
+          File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+        from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+    ImportErrorfrom .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit    
+from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit    
+from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn    
+from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+        from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+        from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+
+        from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminaImportError
+: ImportErrorcannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py): 
+cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+ImportErrorImportError: : cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+Traceback (most recent call last):
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from llava.train.train import train
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+            from .model import LlavaLlamaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom .model import LlavaLlamaForCausalLM
+
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .model import LlavaLlamaForCausalLM
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig    
+from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from llava.train.train import train
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina    
+from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError:     cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.
+  @torch.library.impl_abstract("xformers_flash::flash_bwd")
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+Traceback (most recent call last):
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+    from llava.train.train import train    
+from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+        from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+        from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM
+
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+        from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+        from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+
+        from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+        from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_ditfrom .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+
+        from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_ditfrom .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+        from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+
+        from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+        from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+        from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+
+ImportErrorImportError: : cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+
+ImportErrorImportError: : cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from llava.train.train import train
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+        from llava.train.train import trainfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+    ImportErrorfrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina: 
+cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from llava.train.train import train
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from llava.train.train import train
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from .model import LlavaLlamaForCausalLM
+    from .model import LlavaLlamaForCausalLM    
+from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig        
+from .model import LlavaLlamaForCausalLMfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigTraceback (most recent call last):
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+        from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+
+from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    Traceback (most recent call last):
+from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+        from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminaImportError
+:     cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+Traceback (most recent call last):
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)    
+from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+Traceback (most recent call last):
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+Traceback (most recent call last):
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from llava.train.train import train
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from llava.train.train import train
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .model import LlavaLlamaForCausalLM
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+ImportError    : from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM    
+from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+        from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_ditfrom .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+
+      File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+        from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+            from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+
+
+    ImportErrorImportErrorImportError: from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina: : cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+
+
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+W0215 19:27:31.363000 126996985308992 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3180759 closing signal SIGTERM
+W0215 19:27:31.364000 126996985308992 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3180760 closing signal SIGTERM
+W0215 19:27:31.364000 126996985308992 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3180761 closing signal SIGTERM
+W0215 19:27:31.365000 126996985308992 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3180762 closing signal SIGTERM
+W0215 19:27:31.365000 126996985308992 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3180763 closing signal SIGTERM
+W0215 19:27:31.365000 126996985308992 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3180764 closing signal SIGTERM
+W0215 19:27:31.365000 126996985308992 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3180765 closing signal SIGTERM
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+W0215 19:27:32.561000 138343381481280 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3247541 closing signal SIGTERM
+W0215 19:27:32.562000 138343381481280 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3247542 closing signal SIGTERM
+W0215 19:27:32.562000 138343381481280 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3247543 closing signal SIGTERM
+W0215 19:27:32.562000 138343381481280 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3247544 closing signal SIGTERM
+W0215 19:27:32.563000 138343381481280 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3247545 closing signal SIGTERM
+W0215 19:27:32.563000 138343381481280 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3247546 closing signal SIGTERM
+W0215 19:27:32.563000 138343381481280 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3247547 closing signal SIGTERM
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+W0215 19:27:32.860000 134743156152128 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2646061 closing signal SIGTERM
+W0215 19:27:32.860000 134743156152128 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2646062 closing signal SIGTERM
+W0215 19:27:32.861000 134743156152128 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2646063 closing signal SIGTERM
+W0215 19:27:32.861000 134743156152128 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2646064 closing signal SIGTERM
+W0215 19:27:32.861000 134743156152128 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2646065 closing signal SIGTERM
+W0215 19:27:32.866000 134743156152128 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2646066 closing signal SIGTERM
+W0215 19:27:32.868000 135383820547904 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2912960 closing signal SIGTERM
+W0215 19:27:32.869000 135383820547904 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2912961 closing signal SIGTERM
+W0215 19:27:32.869000 135383820547904 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2912962 closing signal SIGTERM
+W0215 19:27:32.870000 135383820547904 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2912963 closing signal SIGTERM
+W0215 19:27:32.870000 135383820547904 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2912965 closing signal SIGTERM
+W0215 19:27:32.871000 135383820547904 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2912966 closing signal SIGTERM
+W0215 19:27:32.871000 135383820547904 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2912967 closing signal SIGTERM
+W0215 19:27:32.959000 139528338814784 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2645985 closing signal SIGTERM
+W0215 19:27:32.960000 139528338814784 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2645986 closing signal SIGTERM
+W0215 19:27:32.960000 139528338814784 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2645988 closing signal SIGTERM
+W0215 19:27:32.960000 139528338814784 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2645989 closing signal SIGTERM
+W0215 19:27:32.960000 139528338814784 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2645990 closing signal SIGTERM
+W0215 19:27:32.960000 139528338814784 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2645991 closing signal SIGTERM
+W0215 19:27:32.961000 126842061190976 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2910245 closing signal SIGTERM
+W0215 19:27:32.961000 139528338814784 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2645992 closing signal SIGTERM
+W0215 19:27:32.961000 126842061190976 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2910246 closing signal SIGTERM
+W0215 19:27:32.961000 126842061190976 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2910247 closing signal SIGTERM
+W0215 19:27:32.962000 126842061190976 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2910248 closing signal SIGTERM
+W0215 19:27:32.962000 126842061190976 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2910249 closing signal SIGTERM
+W0215 19:27:32.962000 126842061190976 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2910250 closing signal SIGTERM
+W0215 19:27:32.962000 126842061190976 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2910251 closing signal SIGTERM
+W0215 19:27:33.070000 124287080048448 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3297884 closing signal SIGTERM
+W0215 19:27:33.070000 124287080048448 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3297885 closing signal SIGTERM
+W0215 19:27:33.071000 124287080048448 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3297886 closing signal SIGTERM
+W0215 19:27:33.072000 124287080048448 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3297887 closing signal SIGTERM
+W0215 19:27:33.073000 124287080048448 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3297888 closing signal SIGTERM
+W0215 19:27:33.073000 124287080048448 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3297889 closing signal SIGTERM
+W0215 19:27:33.073000 124287080048448 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3297890 closing signal SIGTERM
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+W0215 19:27:33.169000 136256322365248 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3244112 closing signal SIGTERM
+W0215 19:27:33.170000 136256322365248 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3244113 closing signal SIGTERM
+W0215 19:27:33.171000 136256322365248 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3244114 closing signal SIGTERM
+W0215 19:27:33.171000 136256322365248 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3244115 closing signal SIGTERM
+W0215 19:27:33.172000 136256322365248 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3244116 closing signal SIGTERM
+W0215 19:27:33.174000 136256322365248 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3244118 closing signal SIGTERM
+W0215 19:27:33.174000 136256322365248 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3244119 closing signal SIGTERM
+Traceback (most recent call last):
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in <module>
+    from llava.train.train import train
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in <module>
+    from .model import LlavaLlamaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in <module>
+    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in <module>
+    from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in <module>
+    from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in <module>
+    from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn
+  File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in <module>
+    from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina
+ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)
+W0215 19:27:33.370000 123909406807872 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2906700 closing signal SIGTERM
+W0215 19:27:33.371000 123909406807872 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2906701 closing signal SIGTERM
+W0215 19:27:33.371000 123909406807872 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2906702 closing signal SIGTERM
+W0215 19:27:33.372000 123909406807872 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2906704 closing signal SIGTERM
+W0215 19:27:33.373000 123909406807872 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2906705 closing signal SIGTERM
+W0215 19:27:33.373000 123909406807872 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2906706 closing signal SIGTERM
+W0215 19:27:33.373000 123909406807872 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2906707 closing signal SIGTERM
+W0215 19:27:33.480000 127715067463488 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2643399 closing signal SIGTERM
+W0215 19:27:33.481000 127715067463488 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2643400 closing signal SIGTERM
+W0215 19:27:33.481000 127715067463488 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2643402 closing signal SIGTERM
+W0215 19:27:33.481000 127715067463488 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2643403 closing signal SIGTERM
+W0215 19:27:33.483000 127715067463488 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2643405 closing signal SIGTERM
+W0215 19:27:33.483000 127715067463488 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2643406 closing signal SIGTERM
+W0215 19:27:33.488000 129994478602048 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2648773 closing signal SIGTERM
+W0215 19:27:33.490000 129994478602048 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2648774 closing signal SIGTERM
+W0215 19:27:33.490000 129994478602048 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2648775 closing signal SIGTERM
+W0215 19:27:33.490000 129994478602048 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2648776 closing signal SIGTERM
+W0215 19:27:33.491000 129994478602048 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2648777 closing signal SIGTERM
+W0215 19:27:33.491000 129994478602048 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2648780 closing signal SIGTERM
+W0215 19:27:33.668000 126092689319744 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2903515 closing signal SIGTERM
+W0215 19:27:33.669000 126092689319744 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2903516 closing signal SIGTERM
+W0215 19:27:33.670000 126092689319744 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2903517 closing signal SIGTERM
+W0215 19:27:33.671000 126092689319744 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2903519 closing signal SIGTERM
+W0215 19:27:33.671000 126092689319744 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2903520 closing signal SIGTERM
+W0215 19:27:33.672000 126092689319744 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2903521 closing signal SIGTERM
+W0215 19:27:33.673000 126092689319744 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2903522 closing signal SIGTERM
+W0215 19:27:33.666000 140227751167808 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2651683 closing signal SIGTERM
+W0215 19:27:33.678000 140227751167808 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2651684 closing signal SIGTERM
+W0215 19:27:33.678000 140227751167808 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2651685 closing signal SIGTERM
+W0215 19:27:33.679000 140227751167808 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2651686 closing signal SIGTERM
+W0215 19:27:33.679000 140227751167808 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2651687 closing signal SIGTERM
+W0215 19:27:33.679000 140227751167808 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2651688 closing signal SIGTERM
+W0215 19:27:33.680000 140227751167808 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2651689 closing signal SIGTERM
+W0215 19:27:33.873000 137783691786048 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2644328 closing signal SIGTERM
+W0215 19:27:33.875000 137783691786048 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2644329 closing signal SIGTERM
+W0215 19:27:33.875000 137783691786048 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2644330 closing signal SIGTERM
+W0215 19:27:33.876000 137783691786048 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2644331 closing signal SIGTERM
+W0215 19:27:33.876000 137783691786048 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2644332 closing signal SIGTERM
+W0215 19:27:33.877000 137783691786048 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2644333 closing signal SIGTERM
+W0215 19:27:33.877000 137783691786048 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2644334 closing signal SIGTERM
+W0215 19:27:34.085000 130310563845952 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3297018 closing signal SIGTERM
+W0215 19:27:34.088000 130310563845952 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3297020 closing signal SIGTERM
+W0215 19:27:34.088000 130310563845952 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3297021 closing signal SIGTERM
+W0215 19:27:34.089000 130310563845952 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3297022 closing signal SIGTERM
+W0215 19:27:34.090000 130310563845952 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3297023 closing signal SIGTERM
+W0215 19:27:34.091000 130310563845952 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3297024 closing signal SIGTERM
+W0215 19:27:34.092000 130310563845952 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3297025 closing signal SIGTERM
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
+  @autocast_custom_fwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
+  @autocast_custom_bwd
+E0215 19:27:34.448000 126996985308992 torch/distributed/elastic/multiprocessing/api.py:833] failed (exitcode: 1) local_rank: 7 (pid: 3180766) of binary: /data/home/zhaojiang/miniconda3/envs/test/bin/python3.11
+Traceback (most recent call last):
+  File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in <module>
+    sys.exit(main())
+             ^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper
+    return f(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main
+    run(args)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run
+    elastic_launch(
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__
+    return launch_agent(self._config, self._entrypoint, list(args))
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 264, in launch_agent
+    raise ChildFailedError(
+torch.distributed.elastic.multiprocessing.errors.ChildFailedError: 
+============================================================
+llava/train/train_mem.py FAILED
+------------------------------------------------------------
+Failures:
+  <NO_OTHER_FAILURES>
+------------------------------------------------------------
+Root Cause (first observed failure):
+[0]:
+  time      : 2025-02-15_19:27:31
+  host      : h100-st-p548xlarge-145.ar-ai-use2.hpcaas
+  rank      : 87 (local_rank: 7)
+  exitcode  : 1 (pid: 3180766)
+  error_file: <N/A>
+  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
+============================================================
+E0215 19:27:34.538000 126842061190976 torch/distributed/elastic/multiprocessing/api.py:833] failed (exitcode: 1) local_rank: 0 (pid: 2910244) of binary: /data/home/zhaojiang/miniconda3/envs/test/bin/python3.11
+Traceback (most recent call last):
+  File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in <module>
+    sys.exit(main())
+             ^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper
+    return f(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main
+    run(args)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run
+    elastic_launch(
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__
+    return launch_agent(self._config, self._entrypoint, list(args))
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 264, in launch_agent
+    raise ChildFailedError(
+torch.distributed.elastic.multiprocessing.errors.ChildFailedError: 
+============================================================
+llava/train/train_mem.py FAILED
+------------------------------------------------------------
+Failures:
+  <NO_OTHER_FAILURES>
+------------------------------------------------------------
+Root Cause (first observed failure):
+[0]:
+  time      : 2025-02-15_19:27:32
+  host      : h100-st-p548xlarge-189.ar-ai-use2.hpcaas
+  rank      : 96 (local_rank: 0)
+  exitcode  : 1 (pid: 2910244)
+  error_file: <N/A>
+  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
+============================================================
+W0215 19:27:34.675000 127678089664320 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2912793 closing signal SIGTERM
+W0215 19:27:34.676000 127678089664320 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2912794 closing signal SIGTERM
+W0215 19:27:34.676000 127678089664320 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2912795 closing signal SIGTERM
+W0215 19:27:34.677000 127678089664320 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2912796 closing signal SIGTERM
+W0215 19:27:34.677000 127678089664320 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2912797 closing signal SIGTERM
+W0215 19:27:34.677000 127678089664320 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2912798 closing signal SIGTERM
+W0215 19:27:34.677000 127678089664320 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2912800 closing signal SIGTERM
+srun: error: h100-st-p548xlarge-145: task 10: Exited with exit code 1
+srun: Terminating StepId=335101.1
+slurmstepd: error: *** STEP 335101.1 ON h100-st-p548xlarge-44 CANCELLED AT 2025-02-15T19:27:34 ***
+W0215 19:27:34.775000 140227751167808 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers
+W0215 19:27:34.776000 140227751167808 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2651683 closing signal SIGTERM
+W0215 19:27:34.775000 124287080048448 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers
+W0215 19:27:34.776000 129994478602048 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers
+W0215 19:27:34.776000 129994478602048 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2648773 closing signal SIGTERM
+W0215 19:27:34.776000 138343381481280 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers
+W0215 19:27:34.776000 124287080048448 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3297884 closing signal SIGTERM
+W0215 19:27:34.776000 138343381481280 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3247541 closing signal SIGTERM
+W0215 19:27:34.776000 124287080048448 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3297886 closing signal SIGTERM
+W0215 19:27:34.776000 137783691786048 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers
+W0215 19:27:34.776000 129994478602048 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2648776 closing signal SIGTERM
+W0215 19:27:34.776000 137783691786048 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2644329 closing signal SIGTERM
+W0215 19:27:34.776000 130310563845952 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers
+W0215 19:27:34.776000 137783691786048 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2644330 closing signal SIGTERM
+W0215 19:27:34.776000 130310563845952 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3297022 closing signal SIGTERM
+W0215 19:27:34.776000 123909406807872 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers
+W0215 19:27:34.776000 135383820547904 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers
+W0215 19:27:34.776000 127715067463488 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers
+W0215 19:27:34.776000 123909406807872 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2906700 closing signal SIGTERM
+W0215 19:27:34.776000 127715067463488 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2643400 closing signal SIGTERM
+W0215 19:27:34.776000 135383820547904 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2912967 closing signal SIGTERM
+W0215 19:27:34.776000 139528338814784 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers
+W0215 19:27:34.776000 126092689319744 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers
+W0215 19:27:34.776000 139528338814784 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2645988 closing signal SIGTERM
+W0215 19:27:34.776000 126092689319744 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2903515 closing signal SIGTERM
+W0215 19:27:34.776000 134743156152128 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers
+W0215 19:27:34.777000 127715067463488 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2643406 closing signal SIGTERM
+W0215 19:27:34.777000 134743156152128 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2646061 closing signal SIGTERM
+W0215 19:27:34.777000 139528338814784 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2645991 closing signal SIGTERM
+W0215 19:27:34.777000 126092689319744 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2903519 closing signal SIGTERM
+W0215 19:27:34.776000 136256322365248 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers
+W0215 19:27:34.777000 134743156152128 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2646062 closing signal SIGTERM
+W0215 19:27:34.777000 136256322365248 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3244115 closing signal SIGTERM
+W0215 19:27:34.777000 134743156152128 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2646063 closing signal SIGTERM
+W0215 19:27:34.777000 139528338814784 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2645992 closing signal SIGTERM
+W0215 19:27:34.777000 136256322365248 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3244116 closing signal SIGTERM
+W0215 19:27:34.777000 134743156152128 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2646065 closing signal SIGTERM
+W0215 19:27:34.778000 127678089664320 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers
+W0215 19:27:34.779000 127678089664320 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2912793 closing signal SIGTERM
+W0215 19:27:34.779000 127678089664320 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2912794 closing signal SIGTERM
+W0215 19:27:34.780000 127678089664320 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2912795 closing signal SIGTERM
+W0215 19:27:34.780000 127678089664320 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2912796 closing signal SIGTERM
+W0215 19:27:34.780000 127678089664320 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2912797 closing signal SIGTERM
+W0215 19:27:34.780000 127678089664320 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2912798 closing signal SIGTERM
+W0215 19:27:34.780000 127678089664320 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2912800 closing signal SIGTERM
+srun: error: h100-st-p548xlarge-189: task 12: Exited with exit code 1
+Traceback (most recent call last):
+  File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in <module>
+    sys.exit(main())
+             ^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper
+    return f(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main
+    run(args)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run
+    elastic_launch(
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__
+    return launch_agent(self._config, self._entrypoint, list(args))
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent
+    result = agent.run()
+             ^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper
+    result = f(*args, **kwargs)
+             ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run
+    result = self._invoke_run(role)
+             ^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run
+    run_result = self._monitor_workers(self._worker_group)
+                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper
+    result = f(*args, **kwargs)
+             ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers
+    result = self._pcontext.wait(0)
+             ^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait
+    return self._poll()
+           ^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll
+    self.close()  # terminate all running procs
+    ^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close
+    self._close(death_sig=death_sig, timeout=timeout)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close
+    handler.proc.wait(time_to_wait)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait
+    return self._wait(timeout=timeout)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait
+    time.sleep(delay)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler
+    raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
+torch.distributed.elastic.multiprocessing.api.SignalException: Process 3247468 got signal: 15
+Traceback (most recent call last):
+  File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in <module>
+    sys.exit(main())
+             ^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper
+Traceback (most recent call last):
+  File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in <module>
+    sys.exit(main())
+             ^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper
+    return f(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main
+    return f(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main
+    run(args)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run
+    run(args)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run
+    elastic_launch(
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__
+    return launch_agent(self._config, self._entrypoint, list(args))
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent
+    elastic_launch(
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__
+    result = agent.run()
+             ^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper
+    return launch_agent(self._config, self._entrypoint, list(args))
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent
+    result = f(*args, **kwargs)
+             ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run
+    result = agent.run()
+             ^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper
+    result = self._invoke_run(role)
+    result = f(*args, **kwargs)
+             ^^^^^^^^^^^^^^^^^^
+             ^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run
+    run_result = self._monitor_workers(self._worker_group)
+    result = self._invoke_run(role)
+                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+             ^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run
+    result = f(*args, **kwargs)
+             ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers
+    run_result = self._monitor_workers(self._worker_group)
+                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper
+    result = self._pcontext.wait(0)
+             ^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait
+    result = f(*args, **kwargs)
+             ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers
+    return self._poll()
+           ^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll
+    result = self._pcontext.wait(0)
+             ^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait
+    return self._poll()
+           ^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll
+    self.close()  # terminate all running procs
+    ^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close
+    self._close(death_sig=death_sig, timeout=timeout)
+    self.close()  # terminate all running procs
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close
+    ^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close
+    self._close(death_sig=death_sig, timeout=timeout)
+    handler.proc.wait(time_to_wait)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait
+    handler.proc.wait(time_to_wait)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait
+    return self._wait(timeout=timeout)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait
+    return self._wait(timeout=timeout)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait
+    time.sleep(delay)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler
+    raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
+    time.sleep(delay)
+torch.distributed.elastic.multiprocessing.api.SignalException: Process 2912890 got signal: 15
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler
+    raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
+torch.distributed.elastic.multiprocessing.api.SignalException: Process 2643332 got signal: 15
+Traceback (most recent call last):
+  File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in <module>
+    sys.exit(main())
+             ^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper
+    return f(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main
+    run(args)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run
+    elastic_launch(
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__
+    return launch_agent(self._config, self._entrypoint, list(args))
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent
+    result = agent.run()
+             ^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper
+    result = f(*args, **kwargs)
+             ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run
+    result = self._invoke_run(role)
+             ^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run
+    run_result = self._monitor_workers(self._worker_group)
+                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper
+    result = f(*args, **kwargs)
+             ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers
+    result = self._pcontext.wait(0)
+             ^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait
+    return self._poll()
+           ^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll
+    self.close()  # terminate all running procs
+    ^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close
+    self._close(death_sig=death_sig, timeout=timeout)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close
+    handler.proc.wait(time_to_wait)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait
+    return self._wait(timeout=timeout)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait
+    time.sleep(delay)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler
+    raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
+torch.distributed.elastic.multiprocessing.api.SignalException: Process 2645996 got signal: 15
+Traceback (most recent call last):
+  File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in <module>
+    sys.exit(main())
+             ^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper
+    return f(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main
+    run(args)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run
+    elastic_launch(
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__
+    return launch_agent(self._config, self._entrypoint, list(args))
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent
+    result = agent.run()
+             ^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper
+    result = f(*args, **kwargs)
+             ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run
+    result = self._invoke_run(role)
+             ^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run
+    run_result = self._monitor_workers(self._worker_group)
+                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper
+    result = f(*args, **kwargs)
+             ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers
+    result = self._pcontext.wait(0)
+             ^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait
+    return self._poll()
+           ^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll
+    self.close()  # terminate all running procs
+    ^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close
+    self._close(death_sig=death_sig, timeout=timeout)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close
+    handler.proc.wait(time_to_wait)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait
+    return self._wait(timeout=timeout)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait
+    time.sleep(delay)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler
+    raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
+torch.distributed.elastic.multiprocessing.api.SignalException: Process 3297813 got signal: 15
+Traceback (most recent call last):
+  File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in <module>
+    sys.exit(main())
+             ^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper
+    return f(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main
+    run(args)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run
+    elastic_launch(
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__
+    return launch_agent(self._config, self._entrypoint, list(args))
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent
+    result = agent.run()
+             ^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper
+    result = f(*args, **kwargs)
+             ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run
+    result = self._invoke_run(role)
+             ^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run
+    run_result = self._monitor_workers(self._worker_group)
+                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper
+    result = f(*args, **kwargs)
+             ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers
+    result = self._pcontext.wait(0)
+             ^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait
+    return self._poll()
+           ^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll
+    self.close()  # terminate all running procs
+    ^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close
+    self._close(death_sig=death_sig, timeout=timeout)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close
+    handler.proc.wait(time_to_wait)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait
+    return self._wait(timeout=timeout)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait
+    time.sleep(delay)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler
+    raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
+torch.distributed.elastic.multiprocessing.api.SignalException: Process 2648705 got signal: 15
+srun: error: h100-st-p548xlarge-144: task 9: Exited with exit code 1
+Traceback (most recent call last):
+  File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in <module>
+    sys.exit(main())
+             ^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper
+    return f(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main
+    run(args)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run
+    elastic_launch(
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__
+    return launch_agent(self._config, self._entrypoint, list(args))
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent
+    result = agent.run()
+             ^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper
+    result = f(*args, **kwargs)
+             ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run
+    result = self._invoke_run(role)
+             ^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run
+    run_result = self._monitor_workers(self._worker_group)
+                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper
+    result = f(*args, **kwargs)
+             ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers
+    result = self._pcontext.wait(0)
+             ^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait
+    return self._poll()
+           ^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll
+    self.close()  # terminate all running procs
+    ^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close
+    self._close(death_sig=death_sig, timeout=timeout)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close
+    handler.proc.wait(time_to_wait)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait
+    return self._wait(timeout=timeout)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait
+    time.sleep(delay)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler
+    raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
+torch.distributed.elastic.multiprocessing.api.SignalException: Process 3244040 got signal: 15
+srun: error: h100-st-p548xlarge-201: task 15: Exited with exit code 1
+srun: error: h100-st-p548xlarge-61: task 2: Exited with exit code 1
+srun: error: h100-st-p548xlarge-74: task 4: Exited with exit code 1
+srun: error: h100-st-p548xlarge-112: task 6: Exited with exit code 1
+srun: error: h100-st-p548xlarge-62: task 3: Exited with exit code 1
+Traceback (most recent call last):
+  File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in <module>
+    sys.exit(main())
+             ^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper
+    return f(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main
+    run(args)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run
+    elastic_launch(
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__
+    return launch_agent(self._config, self._entrypoint, list(args))
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent
+    result = agent.run()
+             ^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper
+    result = f(*args, **kwargs)
+             ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run
+    result = self._invoke_run(role)
+             ^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run
+    run_result = self._monitor_workers(self._worker_group)
+                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper
+    result = f(*args, **kwargs)
+             ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers
+    result = self._pcontext.wait(0)
+             ^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait
+    return self._poll()
+           ^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll
+    self.close()  # terminate all running procs
+    ^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close
+    self._close(death_sig=death_sig, timeout=timeout)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close
+    handler.proc.wait(time_to_wait)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait
+    return self._wait(timeout=timeout)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait
+    time.sleep(delay)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler
+    raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
+torch.distributed.elastic.multiprocessing.api.SignalException: Process 2645919 got signal: 15
+srun: error: h100-st-p548xlarge-143: task 8: Exited with exit code 1
+Traceback (most recent call last):
+  File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in <module>
+    sys.exit(main())
+             ^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper
+    return f(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main
+    run(args)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run
+    elastic_launch(
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__
+    return launch_agent(self._config, self._entrypoint, list(args))
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent
+    result = agent.run()
+             ^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper
+    result = f(*args, **kwargs)
+             ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run
+    result = self._invoke_run(role)
+             ^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run
+    run_result = self._monitor_workers(self._worker_group)
+                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper
+    result = f(*args, **kwargs)
+             ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers
+    result = self._pcontext.wait(0)
+             ^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait
+    return self._poll()
+           ^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll
+    self.close()  # terminate all running procs
+    ^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close
+    self._close(death_sig=death_sig, timeout=timeout)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close
+    handler.proc.wait(time_to_wait)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait
+    return self._wait(timeout=timeout)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait
+    time.sleep(delay)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler
+    raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
+torch.distributed.elastic.multiprocessing.api.SignalException: Process 2903348 got signal: 15
+Traceback (most recent call last):
+  File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in <module>
+    sys.exit(main())
+             ^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper
+    return f(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main
+    run(args)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run
+    elastic_launch(
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__
+    return launch_agent(self._config, self._entrypoint, list(args))
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent
+    result = agent.run()
+             ^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper
+    result = f(*args, **kwargs)
+             ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run
+    result = self._invoke_run(role)
+             ^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run
+    run_result = self._monitor_workers(self._worker_group)
+                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper
+    result = f(*args, **kwargs)
+             ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers
+    result = self._pcontext.wait(0)
+             ^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait
+    return self._poll()
+           ^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll
+    self.close()  # terminate all running procs
+    ^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close
+    self._close(death_sig=death_sig, timeout=timeout)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close
+    handler.proc.wait(time_to_wait)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait
+    return self._wait(timeout=timeout)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait
+    time.sleep(delay)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler
+    raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
+torch.distributed.elastic.multiprocessing.api.SignalException: Process 2643622 got signal: 15
+Traceback (most recent call last):
+  File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in <module>
+    sys.exit(main())
+             ^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper
+    return f(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main
+    run(args)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run
+    elastic_launch(
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__
+    return launch_agent(self._config, self._entrypoint, list(args))
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent
+    result = agent.run()
+             ^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper
+    result = f(*args, **kwargs)
+             ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run
+    result = self._invoke_run(role)
+             ^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run
+    run_result = self._monitor_workers(self._worker_group)
+                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper
+    result = f(*args, **kwargs)
+             ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers
+    result = self._pcontext.wait(0)
+             ^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait
+    return self._poll()
+           ^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll
+    self.close()  # terminate all running procs
+    ^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close
+    self._close(death_sig=death_sig, timeout=timeout)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close
+    handler.proc.wait(time_to_wait)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait
+    return self._wait(timeout=timeout)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait
+    time.sleep(delay)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler
+    raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
+torch.distributed.elastic.multiprocessing.api.SignalException: Process 2651613 got signal: 15
+Traceback (most recent call last):
+  File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in <module>
+    sys.exit(main())
+             ^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper
+    return f(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main
+    run(args)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run
+    elastic_launch(
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__
+    return launch_agent(self._config, self._entrypoint, list(args))
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent
+    result = agent.run()
+             ^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper
+    result = f(*args, **kwargs)
+             ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run
+    result = self._invoke_run(role)
+             ^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run
+    run_result = self._monitor_workers(self._worker_group)
+                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper
+    result = f(*args, **kwargs)
+             ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers
+    result = self._pcontext.wait(0)
+             ^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait
+    return self._poll()
+           ^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll
+    self.close()  # terminate all running procs
+    ^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close
+    self._close(death_sig=death_sig, timeout=timeout)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close
+    handler.proc.wait(time_to_wait)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait
+    return self._wait(timeout=timeout)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait
+    time.sleep(delay)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler
+    raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
+torch.distributed.elastic.multiprocessing.api.SignalException: Process 3296950 got signal: 15
+srun: error: h100-st-p548xlarge-45: task 1: Exited with exit code 1
+Traceback (most recent call last):
+  File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in <module>
+    sys.exit(main())
+             ^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper
+    return f(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main
+    run(args)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run
+    elastic_launch(
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__
+    return launch_agent(self._config, self._entrypoint, list(args))
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent
+    result = agent.run()
+             ^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper
+    result = f(*args, **kwargs)
+             ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run
+    result = self._invoke_run(role)
+             ^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run
+    run_result = self._monitor_workers(self._worker_group)
+                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper
+    result = f(*args, **kwargs)
+             ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers
+    result = self._pcontext.wait(0)
+             ^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait
+    return self._poll()
+           ^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll
+    self.close()  # terminate all running procs
+    ^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close
+    self._close(death_sig=death_sig, timeout=timeout)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close
+    handler.proc.wait(time_to_wait)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait
+    return self._wait(timeout=timeout)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait
+    time.sleep(delay)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler
+    raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
+torch.distributed.elastic.multiprocessing.api.SignalException: Process 2906633 got signal: 15
+srun: error: h100-st-p548xlarge-113: task 7: Exited with exit code 1
+srun: error: h100-st-p548xlarge-75: task 5: Exited with exit code 1
+srun: error: h100-st-p548xlarge-44: task 0: Exited with exit code 1
+srun: error: h100-st-p548xlarge-199: task 13: Exited with exit code 1
+srun: error: h100-st-p548xlarge-200: task 14: Exited with exit code 1
+Traceback (most recent call last):
+  File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in <module>
+    sys.exit(main())
+             ^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper
+    return f(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main
+    run(args)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run
+    elastic_launch(
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__
+    return launch_agent(self._config, self._entrypoint, list(args))
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent
+    result = agent.run()
+             ^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper
+    result = f(*args, **kwargs)
+             ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run
+    result = self._invoke_run(role)
+             ^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run
+    run_result = self._monitor_workers(self._worker_group)
+                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper
+    result = f(*args, **kwargs)
+             ^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers
+    result = self._pcontext.wait(0)
+             ^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait
+    return self._poll()
+           ^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll
+    self.close()  # terminate all running procs
+    ^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close
+    self._close(death_sig=death_sig, timeout=timeout)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close
+    handler.proc.wait(time_to_wait)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait
+    return self._wait(timeout=timeout)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait
+    time.sleep(delay)
+  File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler
+    raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
+torch.distributed.elastic.multiprocessing.api.SignalException: Process 2912725 got signal: 15
+srun: error: h100-st-p548xlarge-188: task 11: Exited with exit code 1