diff --git "a/334413.err" "b/334413.err" new file mode 100644--- /dev/null +++ "b/334413.err" @@ -0,0 +1,8105 @@ + +CommandNotFoundError: Your shell has not been properly configured to use 'conda activate'. +To initialize your shell, run + + $ conda init + +Currently supported shells are: + - bash + - fish + - tcsh + - xonsh + - zsh + - powershell + +See 'conda init --help' for more information and options. + +IMPORTANT: You may need to close and restart your shell after running 'conda init'. + + +W0215 00:02:14.865000 129764909156160 torch/distributed/run.py:779] +W0215 00:02:14.865000 129764909156160 torch/distributed/run.py:779] ***************************************** +W0215 00:02:14.865000 129764909156160 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0215 00:02:14.865000 129764909156160 torch/distributed/run.py:779] ***************************************** +W0215 00:02:14.865000 128138573829952 torch/distributed/run.py:779] +W0215 00:02:14.865000 128138573829952 torch/distributed/run.py:779] ***************************************** +W0215 00:02:14.865000 128138573829952 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0215 00:02:14.865000 128138573829952 torch/distributed/run.py:779] ***************************************** +W0215 00:02:14.865000 123310759921472 torch/distributed/run.py:779] +W0215 00:02:14.865000 123310759921472 torch/distributed/run.py:779] ***************************************** +W0215 00:02:14.865000 123310759921472 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0215 00:02:14.865000 123310759921472 torch/distributed/run.py:779] ***************************************** +W0215 00:02:14.865000 129038857193280 torch/distributed/run.py:779] +W0215 00:02:14.865000 129038857193280 torch/distributed/run.py:779] ***************************************** +W0215 00:02:14.865000 129038857193280 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0215 00:02:14.865000 129038857193280 torch/distributed/run.py:779] ***************************************** +W0215 00:02:14.865000 124226095793984 torch/distributed/run.py:779] +W0215 00:02:14.865000 124226095793984 torch/distributed/run.py:779] ***************************************** +W0215 00:02:14.865000 124226095793984 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0215 00:02:14.865000 124226095793984 torch/distributed/run.py:779] ***************************************** +W0215 00:02:14.865000 129373585499968 torch/distributed/run.py:779] +W0215 00:02:14.865000 129373585499968 torch/distributed/run.py:779] ***************************************** +W0215 00:02:14.865000 129373585499968 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0215 00:02:14.865000 129373585499968 torch/distributed/run.py:779] ***************************************** +W0215 00:02:14.865000 129801103271744 torch/distributed/run.py:779] +W0215 00:02:14.865000 129801103271744 torch/distributed/run.py:779] ***************************************** +W0215 00:02:14.865000 129801103271744 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0215 00:02:14.865000 129801103271744 torch/distributed/run.py:779] ***************************************** +W0215 00:02:14.865000 132163522049856 torch/distributed/run.py:779] +W0215 00:02:14.865000 132163522049856 torch/distributed/run.py:779] ***************************************** +W0215 00:02:14.865000 132163522049856 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0215 00:02:14.865000 132163522049856 torch/distributed/run.py:779] ***************************************** +W0215 00:02:14.865000 138196363204416 torch/distributed/run.py:779] +W0215 00:02:14.865000 138196363204416 torch/distributed/run.py:779] ***************************************** +W0215 00:02:14.865000 138196363204416 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0215 00:02:14.865000 138196363204416 torch/distributed/run.py:779] ***************************************** +W0215 00:02:14.865000 137986558859072 torch/distributed/run.py:779] +W0215 00:02:14.865000 137986558859072 torch/distributed/run.py:779] ***************************************** +W0215 00:02:14.865000 137986558859072 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0215 00:02:14.865000 137986558859072 torch/distributed/run.py:779] ***************************************** +W0215 00:02:14.865000 129122235602752 torch/distributed/run.py:779] +W0215 00:02:14.865000 129122235602752 torch/distributed/run.py:779] ***************************************** +W0215 00:02:14.865000 129122235602752 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0215 00:02:14.865000 129122235602752 torch/distributed/run.py:779] ***************************************** +W0215 00:02:14.865000 127823105828672 torch/distributed/run.py:779] +W0215 00:02:14.865000 127823105828672 torch/distributed/run.py:779] ***************************************** +W0215 00:02:14.865000 127823105828672 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0215 00:02:14.865000 127823105828672 torch/distributed/run.py:779] ***************************************** +W0215 00:02:14.865000 135003158742848 torch/distributed/run.py:779] +W0215 00:02:14.865000 135003158742848 torch/distributed/run.py:779] ***************************************** +W0215 00:02:14.865000 135003158742848 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0215 00:02:14.865000 135003158742848 torch/distributed/run.py:779] ***************************************** +slurmstepd: error: Prolog hung on node h100-st-p548xlarge-384 +slurmstepd: error: Prolog hung on node h100-st-p548xlarge-390 +slurmstepd: error: Prolog hung on node h100-st-p548xlarge-278 +W0215 00:02:53.090000 125204035315520 torch/distributed/run.py:779] +W0215 00:02:53.090000 125204035315520 torch/distributed/run.py:779] ***************************************** +W0215 00:02:53.090000 125204035315520 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0215 00:02:53.090000 125204035315520 torch/distributed/run.py:779] ***************************************** +W0215 00:02:53.601000 123979779815232 torch/distributed/run.py:779] +W0215 00:02:53.601000 123979779815232 torch/distributed/run.py:779] ***************************************** +W0215 00:02:53.601000 123979779815232 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0215 00:02:53.601000 123979779815232 torch/distributed/run.py:779] ***************************************** +W0215 00:02:54.291000 127908677564224 torch/distributed/run.py:779] +W0215 00:02:54.291000 127908677564224 torch/distributed/run.py:779] ***************************************** +W0215 00:02:54.291000 127908677564224 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0215 00:02:54.291000 127908677564224 torch/distributed/run.py:779] ***************************************** +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") + @torch.library.impl_abstract("xformers_flash::flash_fwd") + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import trainfrom llava.train.train import train + +from llava.train.train import train +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train +from llava.train.train import train +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + from llava.train.train import train + from llava.train.train import train + from llava.train.train import train + from llava.train.train import train +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import trainfrom llava.train.train import train + + from llava.train.train import trainfrom llava.train.train import train + +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + from llava.train.train import train + from llava.train.train import train + Traceback (most recent call last): +from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + from llava.train.train import train +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + from .model import LlavaLlamaForCausalLM +from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM + + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import trainfrom llava.train.train import train + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +from llava.train.train import train +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + from llava.train.train import train File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +from llava.train.train import train +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + Traceback (most recent call last): +from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train +from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM + +from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + from .model import LlavaLlamaForCausalLM +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +Traceback (most recent call last): + from llava.train.train import train +from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +Traceback (most recent call last): + from llava.train.train import train + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig +from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + + from .model import LlavaLlamaForCausalLM + from .model import LlavaLlamaForCausalLM +from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +from llava.train.train import train + from .model import LlavaLlamaForCausalLM + from .model import LlavaLlamaForCausalLM +from .model import LlavaLlamaForCausalLM + from .model import LlavaLlamaForCausalLM + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM + + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from .model import LlavaLlamaForCausalLM +from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + from llava.train.train import train + from .model import LlavaLlamaForCausalLM +Traceback (most recent call last): + from .model import LlavaLlamaForCausalLM + from .model import LlavaLlamaForCausalLM + from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM + + from .model import LlavaLlamaForCausalLM + from .model import LlavaLlamaForCausalLM + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in +from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in +Traceback (most recent call last): + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .model import LlavaLlamaForCausalLM +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + + + from llava.train.train import train + from .model import LlavaLlamaForCausalLM + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from llava.train.train import train + from .model import LlavaLlamaForCausalLM + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM +from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM + from .model import LlavaLlamaForCausalLM + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from llava.train.train import train + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM + from .model import LlavaLlamaForCausalLM File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in +from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig +from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from llava.train.train import train +from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from llava.train.train import train + from llava.train.train import train +Traceback (most recent call last): + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in +from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +Traceback (most recent call last): + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig +from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .model import LlavaLlamaForCausalLM + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import trainfrom llava.train.train import train + + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +from llava.train.train import train + from .model import LlavaLlamaForCausalLM + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + + + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import trainfrom llava.train.train import train + + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from llava.train.train import train File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .model import LlavaLlamaForCausalLMfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM +from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig +from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + Traceback (most recent call last): +from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in +Traceback (most recent call last): + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .model import LlavaLlamaForCausalLM + from .model import LlavaLlamaForCausalLM +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in +from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + + + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM + + + from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .model import LlavaLlamaForCausalLM File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + + from .model import LlavaLlamaForCausalLM +from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from llava.train.train import train + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM +from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM +from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in +Traceback (most recent call last): + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from llava.train.train import train + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from llava.train.train import trainfrom .model import LlavaLlamaForCausalLM + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .model import LlavaLlamaForCausalLM + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + + from .model import LlavaLlamaForCausalLM + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from .model import LlavaLlamaForCausalLM + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from llava.train.train import train + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in +from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from llava.train.train import train + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from llava.train.train import train + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .model import LlavaLlamaForCausalLM + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig +from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + +from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .model import LlavaLlamaForCausalLM + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from llava.train.train import train File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .model import LlavaLlamaForCausalLM + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM +from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit +from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in +from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from llava.train.train import train + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .model import LlavaLlamaForCausalLM + from .model import LlavaLlamaForCausalLM + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_ditfrom .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + + + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM +from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_ditfrom .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in +from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.train.train import train + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit +from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn +from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + +from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in +from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in +from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn +from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + + + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: ImportErrorcannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py): +cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportErrorImportError: : cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + +from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + +ImportError : from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminacannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + +ImportError: ImportErrorcannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py): +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: ImportErrorcannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +: ImportErrorcannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py): + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + +from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + +ImportError File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in +: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn +ImportError: ImportErrorcannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py): +cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + ImportErrorfrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina: +cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + + cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)ImportError +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py): +cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)ImportError +ImportErrorImportError: : : cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + + + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn +ImportError : from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminacannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit +ImportErrorImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py): +cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from llava.train.train import train + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportErrorImportError: : ImportErrorcannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py): + +cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in +from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in +ImportError: File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in +cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)ImportError +: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from llava.train.train import train + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .model import LlavaLlamaForCausalLM +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)ImportError +: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +Traceback (most recent call last): + from .model import LlavaLlamaForCausalLM + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminaImportError +: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)ImportError +: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)ImportError +: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from llava.train.train import train + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +Traceback (most recent call last): +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .model import LlavaLlamaForCausalLM + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.train.train import train + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + ImportErrorfrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina: ImportError +cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py): +cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportErrorImportError: : cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.train.train import train + from llava.train.train import train + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from .model import LlavaLlamaForCausalLM +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: ImportErrorcannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina: + + cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: ImportErrorcannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py): +cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from llava.train.train import train + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from .model import LlavaLlamaForCausalLM + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from .model import LlavaLlamaForCausalLM + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +W0215 00:03:38.704000 129373585499968 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1995356 closing signal SIGTERM +W0215 00:03:38.705000 129373585499968 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1995357 closing signal SIGTERM +W0215 00:03:38.705000 129373585499968 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1995358 closing signal SIGTERM +W0215 00:03:38.705000 129373585499968 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1995359 closing signal SIGTERM +W0215 00:03:38.705000 129373585499968 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1995360 closing signal SIGTERM +W0215 00:03:38.705000 129373585499968 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1995361 closing signal SIGTERM +W0215 00:03:38.705000 129373585499968 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1995363 closing signal SIGTERM +W0215 00:03:38.806000 123979779815232 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1584269 closing signal SIGTERM +W0215 00:03:38.806000 123979779815232 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1584270 closing signal SIGTERM +W0215 00:03:38.807000 123979779815232 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1584271 closing signal SIGTERM +W0215 00:03:38.807000 123979779815232 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1584272 closing signal SIGTERM +W0215 00:03:38.807000 123979779815232 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1584273 closing signal SIGTERM +W0215 00:03:38.808000 123979779815232 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1584274 closing signal SIGTERM +W0215 00:03:38.809000 123979779815232 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1584276 closing signal SIGTERM +W0215 00:03:38.912000 129038857193280 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 988840 closing signal SIGTERM +W0215 00:03:38.913000 129038857193280 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 988841 closing signal SIGTERM +W0215 00:03:38.914000 129038857193280 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 988842 closing signal SIGTERM +W0215 00:03:38.914000 129038857193280 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 988844 closing signal SIGTERM +W0215 00:03:38.915000 129038857193280 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 988845 closing signal SIGTERM +W0215 00:03:38.915000 129038857193280 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 988846 closing signal SIGTERM +W0215 00:03:39.004000 137986558859072 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2158654 closing signal SIGTERM +W0215 00:03:39.005000 137986558859072 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2158655 closing signal SIGTERM +W0215 00:03:39.005000 137986558859072 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2158656 closing signal SIGTERM +W0215 00:03:39.005000 137986558859072 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2158657 closing signal SIGTERM +W0215 00:03:39.005000 137986558859072 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2158658 closing signal SIGTERM +W0215 00:03:39.005000 137986558859072 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2158660 closing signal SIGTERM +W0215 00:03:39.005000 137986558859072 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2158661 closing signal SIGTERM +W0215 00:03:39.010000 123310759921472 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 59703 closing signal SIGTERM +W0215 00:03:39.010000 123310759921472 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 59704 closing signal SIGTERM +W0215 00:03:39.010000 129764909156160 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 676284 closing signal SIGTERM +W0215 00:03:39.010000 123310759921472 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 59705 closing signal SIGTERM +W0215 00:03:39.011000 123310759921472 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 59706 closing signal SIGTERM +W0215 00:03:39.011000 123310759921472 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 59707 closing signal SIGTERM +W0215 00:03:39.010000 129764909156160 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 676285 closing signal SIGTERM +W0215 00:03:39.011000 123310759921472 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 59709 closing signal SIGTERM +W0215 00:03:39.011000 123310759921472 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 59710 closing signal SIGTERM +W0215 00:03:39.011000 129764909156160 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 676287 closing signal SIGTERM +W0215 00:03:39.011000 129764909156160 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 676288 closing signal SIGTERM +W0215 00:03:39.012000 129764909156160 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 676289 closing signal SIGTERM +W0215 00:03:39.013000 129764909156160 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 676290 closing signal SIGTERM +W0215 00:03:39.104000 124226095793984 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 58559 closing signal SIGTERM +W0215 00:03:39.105000 124226095793984 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 58560 closing signal SIGTERM +W0215 00:03:39.105000 124226095793984 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 58561 closing signal SIGTERM +W0215 00:03:39.105000 124226095793984 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 58562 closing signal SIGTERM +W0215 00:03:39.105000 124226095793984 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 58563 closing signal SIGTERM +W0215 00:03:39.105000 124226095793984 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 58564 closing signal SIGTERM +W0215 00:03:39.105000 124226095793984 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 58566 closing signal SIGTERM +W0215 00:03:39.106000 127908677564224 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3444097 closing signal SIGTERM +W0215 00:03:39.106000 127908677564224 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3444099 closing signal SIGTERM +W0215 00:03:39.106000 127908677564224 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3444100 closing signal SIGTERM +W0215 00:03:39.106000 127908677564224 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3444101 closing signal SIGTERM +W0215 00:03:39.106000 127908677564224 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3444104 closing signal SIGTERM +W0215 00:03:39.207000 132163522049856 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 59873 closing signal SIGTERM +W0215 00:03:39.208000 132163522049856 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 59875 closing signal SIGTERM +W0215 00:03:39.208000 132163522049856 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 59876 closing signal SIGTERM +W0215 00:03:39.208000 132163522049856 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 59877 closing signal SIGTERM +W0215 00:03:39.208000 132163522049856 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 59878 closing signal SIGTERM +W0215 00:03:39.208000 132163522049856 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 59879 closing signal SIGTERM +W0215 00:03:39.208000 132163522049856 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 59881 closing signal SIGTERM +W0215 00:03:39.307000 128138573829952 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 59442 closing signal SIGTERM +W0215 00:03:39.308000 128138573829952 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 59443 closing signal SIGTERM +W0215 00:03:39.308000 128138573829952 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 59445 closing signal SIGTERM +W0215 00:03:39.308000 128138573829952 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 59446 closing signal SIGTERM +W0215 00:03:39.308000 128138573829952 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 59447 closing signal SIGTERM +W0215 00:03:39.308000 128138573829952 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 59448 closing signal SIGTERM +W0215 00:03:39.308000 128138573829952 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 59449 closing signal SIGTERM +W0215 00:03:39.310000 135003158742848 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 59580 closing signal SIGTERM +W0215 00:03:39.311000 135003158742848 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 59581 closing signal SIGTERM +W0215 00:03:39.311000 135003158742848 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 59582 closing signal SIGTERM +W0215 00:03:39.311000 135003158742848 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 59583 closing signal SIGTERM +W0215 00:03:39.311000 135003158742848 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 59585 closing signal SIGTERM +W0215 00:03:39.312000 135003158742848 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 59587 closing signal SIGTERM +W0215 00:03:39.410000 127823105828672 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 59426 closing signal SIGTERM +W0215 00:03:39.410000 127823105828672 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 59427 closing signal SIGTERM +W0215 00:03:39.411000 127823105828672 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 59428 closing signal SIGTERM +W0215 00:03:39.411000 127823105828672 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 59429 closing signal SIGTERM +W0215 00:03:39.411000 127823105828672 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 59430 closing signal SIGTERM +W0215 00:03:39.411000 127823105828672 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 59431 closing signal SIGTERM +W0215 00:03:39.411000 127823105828672 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 59432 closing signal SIGTERM +W0215 00:03:39.411000 125204035315520 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2240631 closing signal SIGTERM +W0215 00:03:39.412000 125204035315520 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2240632 closing signal SIGTERM +W0215 00:03:39.412000 125204035315520 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2240633 closing signal SIGTERM +W0215 00:03:39.412000 125204035315520 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2240634 closing signal SIGTERM +W0215 00:03:39.412000 125204035315520 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2240635 closing signal SIGTERM +W0215 00:03:39.412000 125204035315520 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2240637 closing signal SIGTERM +W0215 00:03:39.511000 129801103271744 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 57797 closing signal SIGTERM +W0215 00:03:39.512000 129801103271744 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 57798 closing signal SIGTERM +W0215 00:03:39.513000 129801103271744 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 57799 closing signal SIGTERM +W0215 00:03:39.513000 129801103271744 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 57802 closing signal SIGTERM +W0215 00:03:39.513000 129801103271744 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 57803 closing signal SIGTERM +W0215 00:03:39.514000 129801103271744 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 57804 closing signal SIGTERM +W0215 00:03:39.715000 129122235602752 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 173832 closing signal SIGTERM +W0215 00:03:39.716000 129122235602752 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 173833 closing signal SIGTERM +W0215 00:03:39.716000 129122235602752 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 173834 closing signal SIGTERM +W0215 00:03:39.716000 129122235602752 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 173835 closing signal SIGTERM +W0215 00:03:39.716000 129122235602752 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 173836 closing signal SIGTERM +W0215 00:03:39.716000 129122235602752 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 173837 closing signal SIGTERM +E0215 00:03:40.891000 123310759921472 torch/distributed/elastic/multiprocessing/api.py:833] failed (exitcode: 1) local_rank: 5 (pid: 59708) of binary: /data/home/zhaojiang/miniconda3/envs/test/bin/python3.11 +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 264, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +llava/train/train_mem.py FAILED +------------------------------------------------------------ +Failures: + +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2025-02-15_00:03:39 + host : h100-st-p548xlarge-150.ar-ai-use2.hpcaas + rank : 13 (local_rank: 5) + exitcode : 1 (pid: 59708) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ +W0215 00:03:40.908000 138196363204416 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3608050 closing signal SIGTERM +W0215 00:03:40.908000 138196363204416 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3608051 closing signal SIGTERM +W0215 00:03:40.908000 138196363204416 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3608052 closing signal SIGTERM +W0215 00:03:40.908000 138196363204416 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3608053 closing signal SIGTERM +W0215 00:03:40.908000 138196363204416 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3608054 closing signal SIGTERM +W0215 00:03:40.908000 138196363204416 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3608057 closing signal SIGTERM +E0215 00:03:40.983000 129038857193280 torch/distributed/elastic/multiprocessing/api.py:833] failed (exitcode: 1) local_rank: 3 (pid: 988843) of binary: /data/home/zhaojiang/miniconda3/envs/test/bin/python3.11 +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 264, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +llava/train/train_mem.py FAILED +------------------------------------------------------------ +Failures: +[1]: + time : 2025-02-15_00:03:38 + host : h100-st-p548xlarge-160.ar-ai-use2.hpcaas + rank : 31 (local_rank: 7) + exitcode : 1 (pid: 988847) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2025-02-15_00:03:38 + host : h100-st-p548xlarge-160.ar-ai-use2.hpcaas + rank : 27 (local_rank: 3) + exitcode : 1 (pid: 988843) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ +E0215 00:03:41.073000 127908677564224 torch/distributed/elastic/multiprocessing/api.py:833] failed (exitcode: 1) local_rank: 1 (pid: 3444098) of binary: /data/home/zhaojiang/miniconda3/envs/test/bin/python3.11 +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ +E0215 00:03:41.080000 137986558859072 torch/distributed/elastic/multiprocessing/api.py:833] failed (exitcode: 1) local_rank: 5 (pid: 2158659) of binary: /data/home/zhaojiang/miniconda3/envs/test/bin/python3.11 + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 264, in launch_agent +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +llava/train/train_mem.py FAILED +------------------------------------------------------------ +Failures: +[1]: + time : 2025-02-15_00:03:39 + host : h100-st-p548xlarge-384.ar-ai-use2.hpcaas + rank : 117 (local_rank: 5) + exitcode : 1 (pid: 3444102) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[2]: + time : 2025-02-15_00:03:39 + host : h100-st-p548xlarge-384.ar-ai-use2.hpcaas + rank : 118 (local_rank: 6) + exitcode : 1 (pid: 3444103) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2025-02-15_00:03:39 + host : h100-st-p548xlarge-384.ar-ai-use2.hpcaas + rank : 113 (local_rank: 1) + exitcode : 1 (pid: 3444098) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 264, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +llava/train/train_mem.py FAILED +------------------------------------------------------------ +Failures: + +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2025-02-15_00:03:39 + host : h100-st-p548xlarge-346.ar-ai-use2.hpcaas + rank : 101 (local_rank: 5) + exitcode : 1 (pid: 2158659) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ +E0215 00:03:41.091000 123979779815232 torch/distributed/elastic/multiprocessing/api.py:833] failed (exitcode: 1) local_rank: 6 (pid: 1584275) of binary: /data/home/zhaojiang/miniconda3/envs/test/bin/python3.11 +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 264, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +llava/train/train_mem.py FAILED +------------------------------------------------------------ +Failures: + +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2025-02-15_00:03:38 + host : h100-st-p548xlarge-390.ar-ai-use2.hpcaas + rank : 126 (local_rank: 6) + exitcode : 1 (pid: 1584275) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ +E0215 00:03:41.122000 138196363204416 torch/distributed/elastic/multiprocessing/api.py:833] failed (exitcode: 1) local_rank: 5 (pid: 3608055) of binary: /data/home/zhaojiang/miniconda3/envs/test/bin/python3.11 +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 264, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +llava/train/train_mem.py FAILED +------------------------------------------------------------ +Failures: +[1]: + time : 2025-02-15_00:03:40 + host : h100-st-p548xlarge-238.ar-ai-use2.hpcaas + rank : 86 (local_rank: 6) + exitcode : 1 (pid: 3608056) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2025-02-15_00:03:40 + host : h100-st-p548xlarge-238.ar-ai-use2.hpcaas + rank : 85 (local_rank: 5) + exitcode : 1 (pid: 3608055) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ +E0215 00:03:41.141000 127823105828672 torch/distributed/elastic/multiprocessing/api.py:833] failed (exitcode: 1) local_rank: 0 (pid: 59425) of binary: /data/home/zhaojiang/miniconda3/envs/test/bin/python3.11 +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 264, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +llava/train/train_mem.py FAILED +------------------------------------------------------------ +Failures: + +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2025-02-15_00:03:39 + host : h100-st-p548xlarge-208.ar-ai-use2.hpcaas + rank : 40 (local_rank: 0) + exitcode : 1 (pid: 59425) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ +E0215 00:03:41.186000 124226095793984 torch/distributed/elastic/multiprocessing/api.py:833] failed (exitcode: 1) local_rank: 6 (pid: 58565) of binary: /data/home/zhaojiang/miniconda3/envs/test/bin/python3.11 +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 264, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +llava/train/train_mem.py FAILED +------------------------------------------------------------ +Failures: + +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2025-02-15_00:03:39 + host : h100-st-p548xlarge-210.ar-ai-use2.hpcaas + rank : 62 (local_rank: 6) + exitcode : 1 (pid: 58565) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ +srun: error: h100-st-p548xlarge-150: task 1: Exited with exit code 1 +srun: Terminating StepId=334413.0 +slurmstepd: error: *** STEP 334413.0 ON h100-st-p548xlarge-52 CANCELLED AT 2025-02-15T00:03:41 *** +W0215 00:03:41.226000 129764909156160 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers +W0215 00:03:41.227000 129764909156160 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 676287 closing signal SIGTERM +W0215 00:03:41.227000 135003158742848 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers +W0215 00:03:41.227000 132163522049856 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers +W0215 00:03:41.227000 132163522049856 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 59877 closing signal SIGTERM +W0215 00:03:41.227000 132163522049856 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 59878 closing signal SIGTERM +W0215 00:03:41.227000 129122235602752 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers +W0215 00:03:41.227000 129122235602752 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 173834 closing signal SIGTERM +W0215 00:03:41.227000 128138573829952 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers +W0215 00:03:41.227000 128138573829952 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 59442 closing signal SIGTERM +W0215 00:03:41.227000 129122235602752 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 173835 closing signal SIGTERM +W0215 00:03:41.227000 129373585499968 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers +W0215 00:03:41.227000 129122235602752 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 173837 closing signal SIGTERM +W0215 00:03:41.227000 129801103271744 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers +W0215 00:03:41.228000 129801103271744 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 57799 closing signal SIGTERM +W0215 00:03:41.228000 125204035315520 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers +W0215 00:03:41.228000 125204035315520 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2240632 closing signal SIGTERM +W0215 00:03:41.228000 129801103271744 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 57803 closing signal SIGTERM +W0215 00:03:41.228000 129801103271744 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 57804 closing signal SIGTERM +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + result = agent.run() + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + ^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + result = f(*args, **kwargs) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run + result = self._invoke_run(role) + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run + run_result = self._monitor_workers(self._worker_group) + run_result = self._monitor_workers(self._worker_group) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers + result = self._pcontext.wait(0) + result = self._pcontext.wait(0) + ^^^^^^^^^^^^^^^^^^^^^^ + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait + return self._poll() + ^^^^^^^^^^^^ + return self._poll() + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll + self.close() # terminate all running procs + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close + self.close() # terminate all running procs + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close + self._close(death_sig=death_sig, timeout=timeout) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close + self._close(death_sig=death_sig, timeout=timeout) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close + handler.proc.wait(time_to_wait) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait + handler.proc.wait(time_to_wait) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait + return self._wait(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait + return self._wait(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait + time.sleep(delay) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler + time.sleep(delay) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 57370 got signal: 15 + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 1994566 got signal: 15 +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run + run_result = self._monitor_workers(self._worker_group) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers + result = self._pcontext.wait(0) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait + return self._poll() + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll + self.close() # terminate all running procs + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close + self._close(death_sig=death_sig, timeout=timeout) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close + handler.proc.wait(time_to_wait) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait + return self._wait(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait + time.sleep(delay) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 58655 got signal: 15 +srun: error: h100-st-p548xlarge-210: task 7: Terminated +srun: error: h100-st-p548xlarge-346: task 12: Terminated +srun: error: h100-st-p548xlarge-390: task 15: Terminated +srun: error: h100-st-p548xlarge-384: task 14: Terminated +srun: error: h100-st-p548xlarge-238: task 10: Terminated +srun: error: h100-st-p548xlarge-160: task 3: Exited with exit code 1 +srun: error: h100-st-p548xlarge-208: task 5: Terminated +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run + run_result = self._monitor_workers(self._worker_group) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers + result = self._pcontext.wait(0) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait + return self._poll() + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll + self.close() # terminate all running procs + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close + self._close(death_sig=death_sig, timeout=timeout) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close + handler.proc.wait(time_to_wait) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait + return self._wait(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait + time.sleep(delay) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 58181 got signal: 15 +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run + run_result = self._monitor_workers(self._worker_group) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers + result = self._pcontext.wait(0) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait + return self._poll() + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll + self.close() # terminate all running procs + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close + self._close(death_sig=death_sig, timeout=timeout) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close + handler.proc.wait(time_to_wait) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait + return self._wait(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait + time.sleep(delay) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 173041 got signal: 15 +srun: error: h100-st-p548xlarge-195: task 4: Exited with exit code 1 +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run + run_result = self._monitor_workers(self._worker_group) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers + result = self._pcontext.wait(0) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait + return self._poll() + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll + self.close() # terminate all running procs + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close + self._close(death_sig=death_sig, timeout=timeout) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close + handler.proc.wait(time_to_wait) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait + return self._wait(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait + time.sleep(delay) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 2240545 got signal: 15 +srun: error: h100-st-p548xlarge-209: task 6: Exited with exit code 1 +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run + run_result = self._monitor_workers(self._worker_group) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers + result = self._pcontext.wait(0) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait + return self._poll() + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll + self.close() # terminate all running procs + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close + self._close(death_sig=death_sig, timeout=timeout) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close + handler.proc.wait(time_to_wait) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait + return self._wait(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait + time.sleep(delay) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 56178 got signal: 15 +srun: error: h100-st-p548xlarge-153: task 2: Exited with exit code 1 +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run + run_result = self._monitor_workers(self._worker_group) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers + result = self._pcontext.wait(0) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait + return self._poll() + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll + self.close() # terminate all running procs + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close + self._close(death_sig=death_sig, timeout=timeout) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close + handler.proc.wait(time_to_wait) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait + return self._wait(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait + time.sleep(delay) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 675475 got signal: 15 +srun: error: h100-st-p548xlarge-215: task 8: Exited with exit code 1 +srun: error: h100-st-p548xlarge-377: task 13: Exited with exit code 1 +srun: error: h100-st-p548xlarge-278: task 11: Exited with exit code 1 +srun: error: h100-st-p548xlarge-217: task 9: Exited with exit code 1 +srun: error: h100-st-p548xlarge-52: task 0: Exited with exit code 1 +srun: Force Terminated StepId=334413.0 +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/upload_pretrain.py", line 16, in + upload_folder( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/huggingface_hub/utils/_validators.py", line 114, in _inner_fn + return fn(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/huggingface_hub/hf_api.py", line 1398, in _inner + return fn(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/huggingface_hub/hf_api.py", line 4824, in upload_folder + add_operations = _prepare_upload_folder_additions( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/huggingface_hub/hf_api.py", line 9227, in _prepare_upload_folder_additions + raise ValueError(f"Provided path: '{folder_path}' is not a directory") +ValueError: Provided path: '/fsx-project/xichenpan/PISCES/latent_query_ar_20241220' is not a directory + Reformatting /fsx_0/user/zhaojiang/data/ShareGPT4V/blip_laion_cc_sbu_558k.json: 0%| | 0/558128 [00:00 + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import trainfrom llava.train.train import train + +from llava.train.train import train + from llava.train.train import train + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM +from .model import LlavaLlamaForCausalLM + +from .model import LlavaLlamaForCausalLM + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + + +from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig +from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in +from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + + +from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM +from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit +from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + +from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + +from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + + + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in +from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + + + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportErrorImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py): +cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)ImportErrorImportError +ImportError: : : cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + + +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .model import LlavaLlamaForCausalLM + from .model import LlavaLlamaForCausalLM +from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_ditfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn +from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + ImportErrorfrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina: +cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): +Traceback (most recent call last): +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import trainfrom llava.train.train import trainfrom llava.train.train import trainfrom llava.train.train import train + + + + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from .model import LlavaLlamaForCausalLM +from .model import LlavaLlamaForCausalLM + from .model import LlavaLlamaForCausalLM +from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in +from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from llava.train.train import train + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + + +from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM +from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in +from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_ditfrom .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit +from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_ditfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + +from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn +from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + +from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportErrorImportErrorImportError: : cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py): cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)ImportErrorcannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + +: +ImportErrorcannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)ImportError: +: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from llava.train.train import train +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from .model import LlavaLlamaForCausalLM + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig +from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM +Traceback (most recent call last): + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM +from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_ditfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.train.train import train + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn +from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom .model import LlavaLlamaForCausalLMfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)ImportError +: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + Traceback (most recent call last): +from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig +from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom llava.train.train import train + +from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .model import LlavaLlamaForCausalLMfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train from llava.train.train import train +from llava.train.train import train + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + from .model import LlavaLlamaForCausalLM + from .model import LlavaLlamaForCausalLM File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit +from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminaImportError +ImportError: : cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import trainfrom llava.train.train import trainfrom llava.train.train import train + + + from llava.train.train import train + from llava.train.train import train + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM +from .model import LlavaLlamaForCausalLM + + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in +from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM + from .model import LlavaLlamaForCausalLM + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in +from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in +from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + +from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_ditfrom .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + +from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit +from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from llava.train.train import train + from llava.train.train import train + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + + +from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in +from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in +ImportErrorImportError: : cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + +ImportErrorImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py): +cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + +from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from llava.train.train import train +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_ditfrom .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn +from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + from llava.train.train import train + from llava.train.train import train File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + +from llava.train.train import train + from llava.train.train import train + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + from .model import LlavaLlamaForCausalLM File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + + from .model import LlavaLlamaForCausalLM + from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig from .model import LlavaLlamaForCausalLM +from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig +from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM +from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + +from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_ditfrom .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + +from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_ditfrom .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn +from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +ImportError: ImportErrorImportErrorcannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py): +: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)ImportErrorcannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +: +cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd + from .model import LlavaLlamaForCausalLM +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_ditfrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + +ImportError File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in +: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +Traceback (most recent call last): +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + from llava.train.train import trainTraceback (most recent call last): + + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + from .model import LlavaLlamaForCausalLM + from llava.train.train import trainfrom .model import LlavaLlamaForCausalLM + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in +from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .model import LlavaLlamaForCausalLM File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit +from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit +from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn +from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn +from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in +ImportErrorImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py): +cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in +cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +Traceback (most recent call last): +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import trainfrom llava.train.train import train + + from llava.train.train import train + from llava.train.train import train + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM + +from .model import LlavaLlamaForCausalLM + from .model import LlavaLlamaForCausalLM File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + +from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + +from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM +from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_ditfrom .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_ditfrom .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + + +from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit +from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + +from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn +from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + + + +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: ImportErrorImportErrorcannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)ImportError: : +cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py): cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + +cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnImportError +: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from llava.train.train import train File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .model import LlavaLlamaForCausalLM + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +Traceback (most recent call last): +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + from llava.train.train import trainfrom llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from llava.train.train import train + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + +from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn +from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in +from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + +from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportErrorImportError: ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py): + +cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + ImportErrorfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn: +cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_ditfrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in +from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_ditfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .model import LlavaLlamaForCausalLM + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_ditfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_ditfrom .model import LlavaLlamaForCausalLM + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import trainfrom llava.train.train import train + + from llava.train.train import train + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM +from .model import LlavaLlamaForCausalLM + +from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig +from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from llava.train.train import train + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + + +from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_ditfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_ditfrom .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + +from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn +from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn +from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in +from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + +from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportErrorImportError: ImportError: ImportErrorcannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py): +: +cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +from llava.train.train import trainfrom llava.train.train import train + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM +from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig +from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .model import LlavaLlamaForCausalLM + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit +from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in +from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn +from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in +ImportError: ImportErrorcannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py): +cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError from .model import LlavaLlamaForCausalLM: +cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.train.train import train +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.train.train import train + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from .model import LlavaLlamaForCausalLM File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +Traceback (most recent call last): +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import trainfrom llava.train.train import train + + from llava.train.train import train + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM +from .model import LlavaLlamaForCausalLM + + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in +Traceback (most recent call last): + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig +from .model import LlavaLlamaForCausalLM +from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + +from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + + +from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in +from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +Traceback (most recent call last): +Traceback (most recent call last): +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import trainfrom llava.train.train import trainfrom llava.train.train import train + + + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM + + from .model import LlavaLlamaForCausalLM + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig +from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in +from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + +from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM +from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_ditfrom .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit +from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + + +from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + +from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportErrorImportError: : ImportErrorImportErrorcannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py): : ImportError +cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py): + +cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + +ImportError File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in +: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import trainfrom llava.train.train import train + + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train +from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM + + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in +from .model import LlavaLlamaForCausalLM File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + + Traceback (most recent call last): +from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in +from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .model import LlavaLlamaForCausalLMfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + + + + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in +from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + + + + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in +from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit +from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit +from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn +from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn +from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in +from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + + +from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + +ImportErrorImportError ImportErrorImportError: : from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit: : cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)ImportErrorcannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: +cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +: + +cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +W0215 00:09:39.923000 125687188592448 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 65880 closing signal SIGTERM +W0215 00:09:39.924000 125687188592448 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 65882 closing signal SIGTERM +W0215 00:09:39.924000 125687188592448 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 65884 closing signal SIGTERM +W0215 00:09:39.924000 125687188592448 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 65885 closing signal SIGTERM +W0215 00:09:39.924000 125687188592448 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 65886 closing signal SIGTERM +W0215 00:09:39.924000 125687188592448 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 65887 closing signal SIGTERM +W0215 00:09:39.925000 135699732072256 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2247311 closing signal SIGTERM +W0215 00:09:39.925000 135699732072256 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2247312 closing signal SIGTERM +W0215 00:09:39.926000 135699732072256 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2247313 closing signal SIGTERM +W0215 00:09:39.926000 135699732072256 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2247314 closing signal SIGTERM +W0215 00:09:39.926000 135699732072256 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2247315 closing signal SIGTERM +W0215 00:09:39.927000 135699732072256 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2247316 closing signal SIGTERM +W0215 00:09:39.927000 135699732072256 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2247317 closing signal SIGTERM +W0215 00:09:39.929000 131474921789248 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1591807 closing signal SIGTERM +W0215 00:09:39.930000 131474921789248 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1591808 closing signal SIGTERM +W0215 00:09:39.930000 131474921789248 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1591809 closing signal SIGTERM +W0215 00:09:39.930000 131474921789248 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1591810 closing signal SIGTERM +W0215 00:09:39.931000 131474921789248 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1591811 closing signal SIGTERM +W0215 00:09:39.931000 131474921789248 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1591812 closing signal SIGTERM +W0215 00:09:39.931000 131474921789248 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1591813 closing signal SIGTERM +W0215 00:09:39.924000 134036258109248 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 67995 closing signal SIGTERM +W0215 00:09:39.937000 134036258109248 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 67996 closing signal SIGTERM +W0215 00:09:39.937000 134036258109248 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 67998 closing signal SIGTERM +W0215 00:09:39.938000 134036258109248 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 67999 closing signal SIGTERM +W0215 00:09:39.938000 134036258109248 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 68000 closing signal SIGTERM +W0215 00:09:39.939000 134036258109248 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 68001 closing signal SIGTERM +W0215 00:09:39.939000 134036258109248 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 68002 closing signal SIGTERM +W0215 00:09:40.130000 139955662657344 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 683747 closing signal SIGTERM +W0215 00:09:40.131000 139955662657344 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 683749 closing signal SIGTERM +W0215 00:09:40.132000 139955662657344 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 683750 closing signal SIGTERM +W0215 00:09:40.132000 139955662657344 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 683751 closing signal SIGTERM +W0215 00:09:40.133000 139955662657344 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 683752 closing signal SIGTERM +W0215 00:09:40.134000 139955662657344 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 683753 closing signal SIGTERM +W0215 00:09:40.134000 139955662657344 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 683754 closing signal SIGTERM +W0215 00:09:40.230000 126060026709824 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 65545 closing signal SIGTERM +W0215 00:09:40.231000 126060026709824 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 65546 closing signal SIGTERM +W0215 00:09:40.231000 126060026709824 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 65547 closing signal SIGTERM +W0215 00:09:40.231000 126060026709824 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 65549 closing signal SIGTERM +W0215 00:09:40.233000 126060026709824 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 65550 closing signal SIGTERM +W0215 00:09:40.233000 126060026709824 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 65551 closing signal SIGTERM +W0215 00:09:40.233000 126060026709824 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 65552 closing signal SIGTERM +W0215 00:09:40.238000 135680444311360 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 180776 closing signal SIGTERM +W0215 00:09:40.238000 135680444311360 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 180777 closing signal SIGTERM +W0215 00:09:40.238000 135680444311360 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 180778 closing signal SIGTERM +W0215 00:09:40.238000 135680444311360 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 180780 closing signal SIGTERM +W0215 00:09:40.238000 135680444311360 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 180781 closing signal SIGTERM +W0215 00:09:40.239000 135680444311360 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 180782 closing signal SIGTERM +W0215 00:09:40.239000 135680444311360 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 180783 closing signal SIGTERM +W0215 00:09:40.530000 138417597019968 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3616177 closing signal SIGTERM +W0215 00:09:40.529000 125742087296832 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 996891 closing signal SIGTERM +W0215 00:09:40.531000 125742087296832 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 996892 closing signal SIGTERM +W0215 00:09:40.531000 138417597019968 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3616178 closing signal SIGTERM +W0215 00:09:40.531000 138417597019968 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3616179 closing signal SIGTERM +W0215 00:09:40.532000 138417597019968 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3616180 closing signal SIGTERM +W0215 00:09:40.532000 138417597019968 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3616181 closing signal SIGTERM +W0215 00:09:40.532000 125742087296832 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 996893 closing signal SIGTERM +W0215 00:09:40.532000 138417597019968 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3616182 closing signal SIGTERM +W0215 00:09:40.533000 138417597019968 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3616184 closing signal SIGTERM +W0215 00:09:40.533000 125742087296832 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 996894 closing signal SIGTERM +W0215 00:09:40.533000 125742087296832 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 996895 closing signal SIGTERM +W0215 00:09:40.534000 125742087296832 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 996896 closing signal SIGTERM +W0215 00:09:40.535000 125742087296832 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 996898 closing signal SIGTERM +W0215 00:09:40.627000 130406372398912 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2166249 closing signal SIGTERM +W0215 00:09:40.629000 130406372398912 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2166250 closing signal SIGTERM +W0215 00:09:40.630000 130406372398912 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2166251 closing signal SIGTERM +W0215 00:09:40.630000 130926856595264 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 65683 closing signal SIGTERM +W0215 00:09:40.631000 130406372398912 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2166252 closing signal SIGTERM +W0215 00:09:40.631000 130926856595264 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 65684 closing signal SIGTERM +W0215 00:09:40.631000 130406372398912 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2166253 closing signal SIGTERM +W0215 00:09:40.632000 130926856595264 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 65685 closing signal SIGTERM +W0215 00:09:40.632000 130406372398912 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2166255 closing signal SIGTERM +W0215 00:09:40.632000 130926856595264 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 65686 closing signal SIGTERM +W0215 00:09:40.633000 130406372398912 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2166256 closing signal SIGTERM +W0215 00:09:40.633000 130926856595264 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 65687 closing signal SIGTERM +W0215 00:09:40.634000 130926856595264 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 65688 closing signal SIGTERM +W0215 00:09:40.635000 130926856595264 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 65689 closing signal SIGTERM +W0215 00:09:40.834000 132690238252864 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 65529 closing signal SIGTERM +W0215 00:09:40.835000 132690238252864 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 65530 closing signal SIGTERM +W0215 00:09:40.835000 132690238252864 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 65532 closing signal SIGTERM +W0215 00:09:40.835000 132690238252864 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 65533 closing signal SIGTERM +W0215 00:09:40.835000 132690238252864 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 65534 closing signal SIGTERM +W0215 00:09:40.835000 132690238252864 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 65535 closing signal SIGTERM +W0215 00:09:40.835000 132690238252864 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 65536 closing signal SIGTERM +W0215 00:09:41.041000 125602418833216 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2002088 closing signal SIGTERM +W0215 00:09:41.043000 125602418833216 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2002089 closing signal SIGTERM +W0215 00:09:41.044000 125602418833216 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2002090 closing signal SIGTERM +W0215 00:09:41.045000 125602418833216 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2002091 closing signal SIGTERM +W0215 00:09:41.046000 125602418833216 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2002092 closing signal SIGTERM +W0215 00:09:41.047000 125602418833216 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2002093 closing signal SIGTERM +W0215 00:09:41.046000 139539798456128 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 65523 closing signal SIGTERM +W0215 00:09:41.047000 139539798456128 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 65525 closing signal SIGTERM +W0215 00:09:41.045000 128498242037568 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 65852 closing signal SIGTERM +W0215 00:09:41.047000 125602418833216 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2002094 closing signal SIGTERM +W0215 00:09:41.048000 139539798456128 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 65526 closing signal SIGTERM +W0215 00:09:41.048000 139539798456128 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 65527 closing signal SIGTERM +W0215 00:09:41.048000 128498242037568 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 65853 closing signal SIGTERM +W0215 00:09:41.049000 139539798456128 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 65528 closing signal SIGTERM +W0215 00:09:41.049000 128498242037568 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 65854 closing signal SIGTERM +W0215 00:09:41.049000 139539798456128 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 65529 closing signal SIGTERM +W0215 00:09:41.049000 139539798456128 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 65530 closing signal SIGTERM +W0215 00:09:41.050000 128498242037568 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 65855 closing signal SIGTERM +W0215 00:09:41.050000 128498242037568 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 65856 closing signal SIGTERM +W0215 00:09:41.051000 128498242037568 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 65857 closing signal SIGTERM +W0215 00:09:41.052000 128498242037568 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 65859 closing signal SIGTERM +W0215 00:09:41.137000 135672173561664 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3451604 closing signal SIGTERM +W0215 00:09:41.139000 135672173561664 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3451605 closing signal SIGTERM +W0215 00:09:41.140000 135672173561664 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3451606 closing signal SIGTERM +W0215 00:09:41.140000 135672173561664 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3451607 closing signal SIGTERM +W0215 00:09:41.140000 135672173561664 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3451608 closing signal SIGTERM +W0215 00:09:41.141000 135672173561664 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3451609 closing signal SIGTERM +W0215 00:09:41.141000 135672173561664 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3451611 closing signal SIGTERM +E0215 00:09:42.155000 125687188592448 torch/distributed/elastic/multiprocessing/api.py:833] failed (exitcode: 1) local_rank: 1 (pid: 65881) of binary: /data/home/zhaojiang/miniconda3/envs/test/bin/python3.11 +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 264, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +llava/train/train_mem.py FAILED +------------------------------------------------------------ +Failures: +[1]: + time : 2025-02-15_00:09:39 + host : h100-st-p548xlarge-217.ar-ai-use2.hpcaas + rank : 75 (local_rank: 3) + exitcode : 1 (pid: 65883) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2025-02-15_00:09:39 + host : h100-st-p548xlarge-217.ar-ai-use2.hpcaas + rank : 73 (local_rank: 1) + exitcode : 1 (pid: 65881) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ +srun: error: h100-st-p548xlarge-217: task 9: Exited with exit code 1 +srun: Terminating StepId=334413.1 +slurmstepd: error: *** STEP 334413.1 ON h100-st-p548xlarge-52 CANCELLED AT 2025-02-15T00:09:42 *** +W0215 00:09:42.489000 139955662657344 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers +W0215 00:09:42.490000 139955662657344 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 683749 closing signal SIGTERM +W0215 00:09:42.490000 139955662657344 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 683750 closing signal SIGTERM +W0215 00:09:42.490000 128498242037568 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers +W0215 00:09:42.490000 134036258109248 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers +W0215 00:09:42.490000 130406372398912 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers +W0215 00:09:42.490000 128498242037568 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 65856 closing signal SIGTERM +W0215 00:09:42.490000 130406372398912 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2166255 closing signal SIGTERM +W0215 00:09:42.490000 128498242037568 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 65857 closing signal SIGTERM +W0215 00:09:42.490000 126060026709824 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers +W0215 00:09:42.490000 134036258109248 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 67995 closing signal SIGTERM +W0215 00:09:42.490000 130926856595264 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers +W0215 00:09:42.490000 130926856595264 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 65683 closing signal SIGTERM +W0215 00:09:42.490000 126060026709824 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 65545 closing signal SIGTERM +W0215 00:09:42.490000 130406372398912 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2166256 closing signal SIGTERM +W0215 00:09:42.490000 132690238252864 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers +W0215 00:09:42.490000 126060026709824 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 65546 closing signal SIGTERM +W0215 00:09:42.490000 132690238252864 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 65536 closing signal SIGTERM +W0215 00:09:42.490000 135680444311360 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers +W0215 00:09:42.490000 138417597019968 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers +W0215 00:09:42.490000 135680444311360 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 180778 closing signal SIGTERM +W0215 00:09:42.490000 130926856595264 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 65686 closing signal SIGTERM +W0215 00:09:42.490000 135699732072256 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers +W0215 00:09:42.490000 138417597019968 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3616178 closing signal SIGTERM +W0215 00:09:42.490000 125742087296832 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers +W0215 00:09:42.490000 134036258109248 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 68000 closing signal SIGTERM +W0215 00:09:42.490000 125742087296832 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 996891 closing signal SIGTERM +W0215 00:09:42.490000 125602418833216 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers +W0215 00:09:42.490000 125602418833216 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2002090 closing signal SIGTERM +W0215 00:09:42.490000 135680444311360 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 180781 closing signal SIGTERM +W0215 00:09:42.490000 134036258109248 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 68002 closing signal SIGTERM +W0215 00:09:42.490000 139539798456128 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers +W0215 00:09:42.490000 135680444311360 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 180782 closing signal SIGTERM +W0215 00:09:42.490000 139539798456128 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 65523 closing signal SIGTERM +W0215 00:09:42.490000 138417597019968 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3616184 closing signal SIGTERM +W0215 00:09:42.490000 125602418833216 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2002092 closing signal SIGTERM +W0215 00:09:42.490000 125742087296832 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 996894 closing signal SIGTERM +W0215 00:09:42.490000 135672173561664 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers +W0215 00:09:42.490000 131474921789248 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers +W0215 00:09:42.490000 135672173561664 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3451605 closing signal SIGTERM +W0215 00:09:42.490000 131474921789248 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1591807 closing signal SIGTERM +W0215 00:09:42.490000 139539798456128 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 65526 closing signal SIGTERM +W0215 00:09:42.490000 139539798456128 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 65527 closing signal SIGTERM +W0215 00:09:42.491000 131474921789248 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1591809 closing signal SIGTERM +W0215 00:09:42.491000 135672173561664 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3451609 closing signal SIGTERM +W0215 00:09:42.491000 131474921789248 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1591811 closing signal SIGTERM +W0215 00:09:42.491000 135672173561664 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 3451611 closing signal SIGTERM +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run + run_result = self._monitor_workers(self._worker_group) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers + result = self._pcontext.wait(0) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait + return self._poll() + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll + self.close() # terminate all running procs + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close + self._close(death_sig=death_sig, timeout=timeout) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close + handler.proc.wait(time_to_wait) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait + return self._wait(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait + time.sleep(delay) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 2246670 got signal: 15 +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run + run_result = self._monitor_workers(self._worker_group) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers + result = self._pcontext.wait(0) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait + return self._poll() + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll + self.close() # terminate all running procs + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close + self._close(death_sig=death_sig, timeout=timeout) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close + handler.proc.wait(time_to_wait) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait + return self._wait(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait + time.sleep(delay) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 1591739 got signal: 15 +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run + run_result = self._monitor_workers(self._worker_group) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers + result = self._pcontext.wait(0) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait + return self._poll() + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll + self.close() # terminate all running procs + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close + self._close(death_sig=death_sig, timeout=timeout) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close + handler.proc.wait(time_to_wait) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait + return self._wait(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait + time.sleep(delay) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 65473 got signal: 15 +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run + run_result = self._monitor_workers(self._worker_group) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers + result = self._pcontext.wait(0) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait + return self._poll() + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll + self.close() # terminate all running procs + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close + self._close(death_sig=death_sig, timeout=timeout) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close + handler.proc.wait(time_to_wait) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait + return self._wait(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait + time.sleep(delay) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 180707 got signal: 15 +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent + result = agent.run() + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + ^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + return launch_agent(self._config, self._entrypoint, list(args)) + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run + result = agent.run() + ^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run + run_result = self._monitor_workers(self._worker_group) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = self._invoke_run(role) + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers + run_result = self._monitor_workers(self._worker_group) + result = self._pcontext.wait(0) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + return self._poll() + ^^^^^^^^^^^^^^^^^^ + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers + result = self._pcontext.wait(0) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait + self.close() # terminate all running procs + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close + return self._poll() + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll + self._close(death_sig=death_sig, timeout=timeout) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close + self.close() # terminate all running procs + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close + handler.proc.wait(time_to_wait) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait + self._close(death_sig=death_sig, timeout=timeout) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close + return self._wait(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait + handler.proc.wait(time_to_wait) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait + return self._wait(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait + time.sleep(delay) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 683673 got signal: 15 + time.sleep(delay) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 67927 got signal: 15 +srun: error: h100-st-p548xlarge-278: task 11: Exited with exit code 1 +srun: error: h100-st-p548xlarge-153: task 2: Exited with exit code 1 +srun: error: h100-st-p548xlarge-390: task 15: Exited with exit code 1 +srun: error: h100-st-p548xlarge-377: task 13: Exited with exit code 1 +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run + run_result = self._monitor_workers(self._worker_group) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers + result = self._pcontext.wait(0) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait + return self._poll() + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll + self.close() # terminate all running procs + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close + self._close(death_sig=death_sig, timeout=timeout) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close + handler.proc.wait(time_to_wait) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait + return self._wait(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait + time.sleep(delay) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 996228 got signal: 15 +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run + run_result = self._monitor_workers(self._worker_group) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers + result = self._pcontext.wait(0) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait + return self._poll() + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll + self.close() # terminate all running procs + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close + self._close(death_sig=death_sig, timeout=timeout) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close + handler.proc.wait(time_to_wait) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait + return self._wait(timeout=timeout) +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + time.sleep(delay) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 65615 got signal: 15 + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run + run_result = self._monitor_workers(self._worker_group) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers + result = self._pcontext.wait(0) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait + return self._poll() + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll + self.close() # terminate all running procs + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close + self._close(death_sig=death_sig, timeout=timeout) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close + handler.proc.wait(time_to_wait) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait + return self._wait(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait + time.sleep(delay) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 65456 got signal: 15 +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run + run_result = self._monitor_workers(self._worker_group) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers + result = self._pcontext.wait(0) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait + return self._poll() + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll + self.close() # terminate all running procs + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close + self._close(death_sig=death_sig, timeout=timeout) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close + handler.proc.wait(time_to_wait) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait + return self._wait(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait + time.sleep(delay) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 3616064 got signal: 15 +srun: error: h100-st-p548xlarge-215: task 8: Exited with exit code 1 +srun: error: h100-st-p548xlarge-52: task 0: Exited with exit code 1 +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + elastic_launch( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + result = agent.run() + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent + ^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = agent.run() + ^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run + run_result = self._monitor_workers(self._worker_group) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + run_result = self._monitor_workers(self._worker_group) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers + result = self._pcontext.wait(0) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait + result = self._pcontext.wait(0) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait + return self._poll() + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll + self.close() # terminate all running procs + return self._poll() + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll + self._close(death_sig=death_sig, timeout=timeout) + self.close() # terminate all running procs + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close + handler.proc.wait(time_to_wait) + self._close(death_sig=death_sig, timeout=timeout) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait + handler.proc.wait(time_to_wait) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait + return self._wait(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait + time.sleep(delay) + return self._wait(timeout=timeout) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 65459 got signal: 15 + time.sleep(delay) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 2166157 got signal: 15 +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run + run_result = self._monitor_workers(self._worker_group) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers + result = self._pcontext.wait(0) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait + return self._poll() + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll + self.close() # terminate all running procs + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close + self._close(death_sig=death_sig, timeout=timeout) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close + handler.proc.wait(time_to_wait) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait + return self._wait(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait + time.sleep(delay) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 3451534 got signal: 15 +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run + run_result = self._monitor_workers(self._worker_group) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers + result = self._pcontext.wait(0) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait + return self._poll() + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll + self.close() # terminate all running procs + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close + self._close(death_sig=death_sig, timeout=timeout) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close + handler.proc.wait(time_to_wait) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait + return self._wait(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait + time.sleep(delay) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 2002019 got signal: 15 +srun: error: h100-st-p548xlarge-160: task 3: Exited with exit code 1 +srun: error: h100-st-p548xlarge-210: task 7: Exited with exit code 1 +srun: error: h100-st-p548xlarge-238: task 10: Exited with exit code 1 +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ +srun: error: h100-st-p548xlarge-209: task 6: Exited with exit code 1 + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run + run_result = self._monitor_workers(self._worker_group) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers + result = self._pcontext.wait(0) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait + return self._poll() + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll + self.close() # terminate all running procs + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close + self._close(death_sig=death_sig, timeout=timeout) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close + handler.proc.wait(time_to_wait) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait + return self._wait(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait + time.sleep(delay) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 65783 got signal: 15 +srun: error: h100-st-p548xlarge-346: task 12: Exited with exit code 1 +srun: error: h100-st-p548xlarge-208: task 5: Exited with exit code 1 +srun: error: h100-st-p548xlarge-384: task 14: Exited with exit code 1 +srun: error: h100-st-p548xlarge-195: task 4: Exited with exit code 1 +srun: error: h100-st-p548xlarge-150: task 1: Exited with exit code 1