Upload 258 files

.gitattributes

@@ -33,3 +33,12 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+assets/comp_effic.png filter=lfs diff=lfs merge=lfs -text
+assets/data_for_diff_stage.jpg filter=lfs diff=lfs merge=lfs -text
+assets/i2v_res.png filter=lfs diff=lfs merge=lfs -text
+assets/t2v_res.jpg filter=lfs diff=lfs merge=lfs -text
+assets/vben_vs_sota.png filter=lfs diff=lfs merge=lfs -text
+assets/video_dit_arch.jpg filter=lfs diff=lfs merge=lfs -text
+assets/video_vae_res.jpg filter=lfs diff=lfs merge=lfs -text
+preprocessing/matanyone/tutorial_multi_targets.mp4 filter=lfs diff=lfs merge=lfs -text
+preprocessing/matanyone/tutorial_single_target.mp4 filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,42 @@
+.*
+*.py[cod]
+# *.jpg
+*.jpeg
+# *.png
+*.gif
+*.bmp
+*.mp4
+*.mov
+*.mkv
+*.log
+*.zip
+*.pt
+*.pth
+*.ckpt
+*.safetensors
+*.json
+# *.txt
+*.backup
+*.pkl
+*.html
+*.pdf
+*.whl
+*.exe
+cache
+__pycache__/
+storage/
+samples/
+!.gitignore
+!requirements.txt
+.DS_Store
+*DS_Store
+google/
+Wan2.1-T2V-14B/
+Wan2.1-T2V-1.3B/
+Wan2.1-I2V-14B-480P/
+Wan2.1-I2V-14B-720P/
+outputs/
+gradio_outputs/
+ckpts/
+loras/
+loras_i2v/

LICENSE.txt

@@ -0,0 +1,17 @@
+FREE for Non Commercial USE
+
+You are free to:
+- Share — copy and redistribute the material in any medium or format
+- Adapt — remix, transform, and build upon the material
+The licensor cannot revoke these freedoms as long as you follow the license terms.
+
+Under the following terms:
+- Attribution — You must give appropriate credit , provide a link to the license, and indicate if changes were made . You may do so in any reasonable manner, but not in any way that suggests the licensor endorses you or your use.
+NonCommercial — You may not use the material for commercial purposes .
+
+- No additional restrictions — You may not apply legal terms or technological measures that legally restrict others from doing anything the license permits.
+Notices:
+
+- You do not have to comply with the license for elements of the material in the public domain or where your use is permitted by an applicable exception or limitation .
+
+No warranties are given. The license may not give you all of the permissions necessary for your intended use. For example, other rights such as publicity, privacy, or moral rights may limit how you use the material.

README.md

@@ -1,13 +1,175 @@
----
-title: Wan2GP
-emoji: 🐨
-colorFrom: green
-colorTo: gray
-sdk: gradio
-sdk_version: 5.34.2
-app_file: app.py
-pinned: false
-short_description: Wan2GP
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# WanGP
+
+-----
+<p align="center">
+<b>WanGP by DeepBeepMeep : The best Open Source Video Generative Models Accessible to the GPU Poor</b>
+</p>
+
+WanGP supports the Wan (and derived models), Hunyuan Video and LTV Video models with:
+- Low VRAM requirements (as low as 6 GB of VRAM is sufficient for certain models)
+- Support for old GPUs (RTX 10XX, 20xx, ...)
+- Very Fast on the latest GPUs
+- Easy to use Full Web based interface
+- Auto download of the required model adapted to your specific architecture
+- Tools integrated to facilitate Video Generation : Mask Editor, Prompt Enhancer, Temporal and Spatial Generation
+- Loras Support to customize each model
+- Queuing system : make your shopping list of videos to generate and come back later 
+
+**Discord Server to get Help from Other Users and show your Best Videos:** https://discord.gg/g7efUW9jGV
+
+**Follow DeepBeepMeep on Twitter/X to get the Latest News**: https://x.com/deepbeepmeep
+
+## 🔥 Latest Updates
+### June 23 2025: WanGP v6.3, Vace Unleashed. Thought we couldnt squeeze Vace even more ?
+- Multithreaded preprocessing when possible for faster generations
+- Multithreaded frames Lanczos Upsampling as a bonus
+- A new Vace preprocessor : *Flow* to extract fluid motion
+- Multi Vace Controlnets: you can now transfer several properties at the same time. This opens new possibilities to explore, for instance if you transfer *Human Movement* and *Shapes* at the same time for some reasons the lighting of your character will take into account much more the environment of your character.
+- Injected Frames Outpainting, in case you missed it in WanGP 6.21
+
+Don't know how to use all of the Vace features ? Check the Vace Guide embedded in WanGP as it has also been updated.
+
+
+### June 19 2025: WanGP v6.2, Vace even more Powercharged
+👋 Have I told you that I am a big fan of Vace ? Here are more goodies to unleash its power: 
+- If you ever wanted to watch Star Wars in 4:3, just use the new *Outpainting* feature and it will add the missing bits of image at the top and the bottom of the screen. The best thing is *Outpainting* can be combined with all the other Vace modifications, for instance you can change the main character of your favorite movie at the same time  
+- More processing can combined at the same time  (for instance the depth process can be applied outside the mask)
+- Upgraded the depth extractor to Depth Anything 2 which is much more detailed
+
+As a bonus, I have added two finetunes based on the Safe-Forcing technology (which requires only 4 steps to generate a video): Wan 2.1 text2video Self-Forcing and Vace Self-Forcing. I know there is Lora around but the quality of the Lora is worse (at least with Vace) compared to the full model. Don't hesitate to share your opinion about this on the discord server. 
+### June 17 2025: WanGP v6.1, Vace Powercharged
+👋 Lots of improvements for Vace the Mother of all Models:
+- masks can now be combined with on the fly processing of a control video, for instance you can extract the motion of a specific person defined by a mask
+- on the fly modification of masks : reversed masks (with the same mask you can modify the background instead of the people covered by the masks), enlarged masks (you can cover more area if for instance the person you are trying to inject is larger than the one in the mask), ...
+- view these modified masks directly inside WanGP during the video generation to check they are really as expected
+- multiple frames injections: multiples frames can be injected at any location of the video
+- expand past videos in on click: just select one generated video to expand it
+
+Of course all these new stuff work on all Vace finetunes (including Vace Fusionix).
+
+Thanks also to Reevoy24 for adding a Notfication sound at the end of a generation and for fixing the background color of the current generation summary.
+
+### June 12 2025: WanGP v6.0
+👋 *Finetune models*: You find the 20 models supported by WanGP not sufficient ? Too impatient to wait for the next release to get the support for a newly released model ? Your prayers have been answered: if a new model is compatible with a model architecture supported by WanGP, you can add by yourself the support for this model in WanGP by just creating a finetune model definition. You can then store this model in the cloud (for instance in Huggingface) and the very light finetune definition file can be easily shared with other users. WanGP will download automatically the finetuned model for them.
+
+To celebrate the new finetunes support, here are a few finetune gifts (directly accessible from the model selection menu):
+- *Fast Hunyuan Video* : generate model t2v in only 6 steps
+- *Hunyuan Vido AccVideo* : generate model t2v in only 5 steps
+- *Wan FusioniX*: it is a combo of AccVideo / CausVid ans other models and can generate high quality Wan videos in only 8 steps
+
+One more thing...
+
+The new finetune system can be used to combine complementaty models : what happens when you combine  Fusionix Text2Video and Vace Control Net ?
+
+You get **Vace FusioniX**: the Ultimate Vace Model, Fast (10 steps, no need for guidance) and with a much better quality Video than the original slower model (despite being the best Control Net out there). Here goes one more finetune...
+
+Check the *Finetune Guide* to create finetune models definitions and share them on the WanGP discord server.
+
+### June 11 2025: WanGP v5.5
+👋 *Hunyuan Video Custom Audio*: it is similar to Hunyuan Video Avatar except there isn't any lower limit on the number of frames and you can use your reference images in a different context than the image itself\
+*Hunyuan Video Custom Edit*: Hunyuan Video Controlnet, use it to do inpainting and replace a person in a video while still keeping his poses. Similar to Vace but less restricted than the Wan models in terms of content...
+
+
+### June 6 2025: WanGP v5.41
+👋 Bonus release: Support for **AccVideo** Lora to speed up x2 Video generations in Wan models. Check the Loras documentation to get the usage instructions of AccVideo.\
+You will need to do a *pip install -r requirements.txt*
+
+### June 6 2025: WanGP v5.4
+👋 World Exclusive : **Hunyuan Video Avatar** Support ! You won't need 80 GB of VRAM nor 32 GB oF VRAM, just 10 GB of VRAM will be sufficient to generate up to 15s of high quality speech / song driven Video at a high speed with no quality degradation. Support for TeaCache included.\
+Here is a link to the original repo where you will find some very interesting documentation and examples. https://github.com/Tencent-Hunyuan/HunyuanVideo-Avatar. Kudos to the Hunyuan Video Avatar team for the best model of its kind.\
+Also many thanks to Reevoy24 for his repackaging / completing the documentation
+
+### May 28 2025: WanGP v5.31
+👋 Added **Phantom 14B**, a model that you can use to transfer objects / people in the video. My preference goes to Vace that remains the king of controlnets.
+VACE improvements: Better sliding window transitions, image mask support in Matanyone, new Extend Video feature, and enhanced background removal options.
+
+### May 26, 2025: WanGP v5.3
+👋 Settings management revolution! Now you can:
+- Select any generated video and click *Use Selected Video Settings* to instantly reuse its configuration
+- Drag & drop videos to automatically extract their settings metadata
+- Export/import settings as JSON files for easy sharing and backup
+
+### May 20, 2025: WanGP v5.2
+👋 **CausVid support** - Generate videos in just 4-12 steps with the new distilled Wan model! Also added experimental MoviiGen for 1080p generation (20GB+ VRAM required). Check the Loras documentation to get the usage instructions of CausVid.
+
+### May 18, 2025: WanGP v5.1
+👋 **LTX Video 13B Distilled** - Generate high-quality videos in less than one minute!
+
+### May 17, 2025: WanGP v5.0
+👋 **One App to Rule Them All!** Added Hunyuan Video and LTX Video support, plus Vace 14B and integrated prompt enhancer.
+
+See full changelog: **[Changelog](docs/CHANGELOG.md)**
+
+## 📋 Table of Contents
+
+- [🚀 Quick Start](#-quick-start)
+- [📦 Installation](#-installation)
+- [🎯 Usage](#-usage)
+- [📚 Documentation](#-documentation)
+- [🔗 Related Projects](#-related-projects)
+
+## 🚀 Quick Start
+
+**One-click installation:** Get started instantly with [Pinokio App](https://pinokio.computer/)
+
+**Manual installation:**
+```bash
+git clone https://github.com/deepbeepmeep/Wan2GP.git
+cd Wan2GP
+conda create -n wan2gp python=3.10.9
+conda activate wan2gp
+pip install torch==2.7.0 torchvision torchaudio --index-url https://download.pytorch.org/whl/test/cu124
+pip install -r requirements.txt
+```
+
+**Run the application:**
+```bash
+python wgp.py  # Text-to-video (default)
+python wgp.py --i2v  # Image-to-video
+```
+
+**Update the application:**
+If using Pinokio use Pinokio to update otherwise:
+Get in the directory where WanGP is installed and:
+```bash
+git pull
+pip install -r requirements.txt
+```
+
+## 📦 Installation
+
+For detailed installation instructions for different GPU generations:
+- **[Installation Guide](docs/INSTALLATION.md)** - Complete setup instructions for RTX 10XX to RTX 50XX
+
+## 🎯 Usage
+
+### Basic Usage
+- **[Getting Started Guide](docs/GETTING_STARTED.md)** - First steps and basic usage
+- **[Models Overview](docs/MODELS.md)** - Available models and their capabilities
+
+### Advanced Features
+- **[Loras Guide](docs/LORAS.md)** - Using and managing Loras for customization
+- **[Finetunes](docs/FINETUNES.md)** - Add manually new models to WanGP
+- **[VACE ControlNet](docs/VACE.md)** - Advanced video control and manipulation
+- **[Command Line Reference](docs/CLI.md)** - All available command line options
+
+## 📚 Documentation
+
+- **[Changelog](docs/CHANGELOG.md)** - Latest updates and version history
+- **[Troubleshooting](docs/TROUBLESHOOTING.md)** - Common issues and solutions
+
+## 🔗 Related Projects
+
+### Other Models for the GPU Poor
+- **[HuanyuanVideoGP](https://github.com/deepbeepmeep/HunyuanVideoGP)** - One of the best open source Text to Video generators
+- **[Hunyuan3D-2GP](https://github.com/deepbeepmeep/Hunyuan3D-2GP)** - Image to 3D and text to 3D tool
+- **[FluxFillGP](https://github.com/deepbeepmeep/FluxFillGP)** - Inpainting/outpainting tools based on Flux
+- **[Cosmos1GP](https://github.com/deepbeepmeep/Cosmos1GP)** - Text to world generator and image/video to world
+- **[OminiControlGP](https://github.com/deepbeepmeep/OminiControlGP)** - Flux-derived application for object transfer
+- **[YuE GP](https://github.com/deepbeepmeep/YuEGP)** - Song generator with instruments and singer's voice
+
+---
+
+<p align="center">
+Made with ❤️ by DeepBeepMeep
+</p> 

configs/fantasy.json

@@ -0,0 +1,15 @@
+{
+  "_class_name": "WanModel",
+  "_diffusers_version": "0.30.0",
+  "dim": 5120,
+  "eps": 1e-06,
+  "ffn_dim": 13824,
+  "freq_dim": 256,
+  "in_dim": 36,
+  "model_type": "i2v",
+  "num_heads": 40,
+  "num_layers": 40,
+  "out_dim": 16,
+  "text_len": 512,
+  "fantasytalking_dim": 2048
+}

configs/flf2v_720p.json

@@ -0,0 +1,14 @@
+{
+  "_class_name": "WanModel",
+  "_diffusers_version": "0.30.0",
+  "dim": 5120,
+  "eps": 1e-06,
+  "ffn_dim": 13824,
+  "freq_dim": 256,
+  "in_dim": 36,
+  "model_type": "i2v",
+  "num_heads": 40,
+  "num_layers": 40,
+  "out_dim": 16,
+  "text_len": 512
+}

configs/i2v.json

@@ -0,0 +1,14 @@
+{
+  "_class_name": "WanModel",
+  "_diffusers_version": "0.30.0",
+  "dim": 5120,
+  "eps": 1e-06,
+  "ffn_dim": 13824,
+  "freq_dim": 256,
+  "in_dim": 36,
+  "model_type": "i2v",
+  "num_heads": 40,
+  "num_layers": 40,
+  "out_dim": 16,
+  "text_len": 512
+}

configs/i2v_720p.json

@@ -0,0 +1,14 @@
+{
+  "_class_name": "WanModel",
+  "_diffusers_version": "0.30.0",
+  "dim": 5120,
+  "eps": 1e-06,
+  "ffn_dim": 13824,
+  "freq_dim": 256,
+  "in_dim": 36,
+  "model_type": "i2v",
+  "num_heads": 40,
+  "num_layers": 40,
+  "out_dim": 16,
+  "text_len": 512
+}

configs/phantom_1.3B.json

@@ -0,0 +1,14 @@
+{
+  "_class_name": "WanModel",
+  "_diffusers_version": "0.30.0",
+  "dim": 1536,
+  "eps": 1e-06,
+  "ffn_dim": 8960,
+  "freq_dim": 256,
+  "in_dim": 16,
+  "model_type": "t2v",
+  "num_heads": 12,
+  "num_layers": 30,
+  "out_dim": 16,
+  "text_len": 512
+}

configs/phantom_14B.json

@@ -0,0 +1,14 @@
+{
+  "_class_name": "WanModel",
+  "_diffusers_version": "0.30.0",
+  "dim": 5120,
+  "eps": 1e-06,
+  "ffn_dim": 13824,
+  "freq_dim": 256,
+  "in_dim": 16,
+  "model_type": "t2v",
+  "num_heads": 40,
+  "num_layers": 40,
+  "out_dim": 16,
+  "text_len": 512
+}

configs/sky_df_1.3.json

@@ -0,0 +1,14 @@
+{
+  "_class_name": "WanModel",
+  "_diffusers_version": "0.30.0",
+  "dim": 1536,
+  "eps": 1e-06,
+  "ffn_dim": 8960,
+  "freq_dim": 256,
+  "in_dim": 16,
+  "model_type": "t2v",
+  "num_heads": 12,
+  "num_layers": 30,
+  "out_dim": 16,
+  "text_len": 512
+}

configs/sky_df_14B.json

@@ -0,0 +1,14 @@
+{
+  "_class_name": "WanModel",
+  "_diffusers_version": "0.30.0",
+  "dim": 5120,
+  "eps": 1e-06,
+  "ffn_dim": 13824,
+  "freq_dim": 256,
+  "in_dim": 16,
+  "model_type": "t2v",
+  "num_heads": 40,
+  "num_layers": 40,
+  "out_dim": 16,
+  "text_len": 512
+}

configs/t2v.json

@@ -0,0 +1,14 @@
+{
+  "_class_name": "WanModel",
+  "_diffusers_version": "0.30.0",
+  "dim": 5120,
+  "eps": 1e-06,
+  "ffn_dim": 13824,
+  "freq_dim": 256,
+  "in_dim": 16,
+  "model_type": "t2v",
+  "num_heads": 40,
+  "num_layers": 40,
+  "out_dim": 16,
+  "text_len": 512
+}

configs/t2v_1.3B.json

@@ -0,0 +1,14 @@
+{
+  "_class_name": "WanModel",
+  "_diffusers_version": "0.30.0",
+  "dim": 1536,
+  "eps": 1e-06,
+  "ffn_dim": 8960,
+  "freq_dim": 256,
+  "in_dim": 16,
+  "model_type": "t2v",
+  "num_heads": 12,
+  "num_layers": 30,
+  "out_dim": 16,
+  "text_len": 512
+}

configs/vace_1.3B.json

@@ -0,0 +1,16 @@
+{
+  "_class_name": "VaceWanModel",
+  "_diffusers_version": "0.30.0",
+  "dim": 1536,
+  "eps": 1e-06,
+  "ffn_dim": 8960,
+  "freq_dim": 256,
+  "in_dim": 16,
+  "model_type": "t2v",
+  "num_heads": 12,
+  "num_layers": 30,
+  "out_dim": 16,
+  "text_len": 512,
+  "vace_layers": [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28],
+  "vace_in_dim": 96
+}

configs/vace_14B.json

@@ -0,0 +1,16 @@
+{
+  "_class_name": "VaceWanModel",
+  "_diffusers_version": "0.30.0",
+  "dim": 5120,
+  "eps": 1e-06,
+  "ffn_dim": 13824,
+  "freq_dim": 256,
+  "in_dim": 16,
+  "model_type": "t2v",
+  "num_heads": 40,
+  "num_layers": 40,
+  "out_dim": 16,
+  "text_len": 512,
+  "vace_layers": [0, 5, 10, 15, 20, 25, 30, 35],
+  "vace_in_dim": 96
+}

docs/CHANGELOG.md

@@ -0,0 +1,209 @@
+# Changelog
+
+## 🔥 Latest News
+### June 19 2025: WanGP v6.2, Vace even more Powercharged
+Have I told you that I am a big fan of Vace ? Here are more goodies to unleash its power: 
+- If you ever wanted to watch Star Wars in 4:3, just use the new *Outpainting* feature and it will add the missing bits of image at the top and the bottom of the screen. The best thing is *Outpainting* can be combined with all the other Vace modifications, for instance you can change the main character of your favorite movie at the same time  
+- More processing can combined at the same time  (for instance the depth process can be applied outside the mask)
+- Upgraded the depth extractor to Depth Anything 2 which is much more detailed
+
+As a bonus, I have added two finetunes based on the Safe-Forcing technology (which requires only 4 steps to generate a video): Wan 2.1 text2video Self-Forcing and Vace Self-Forcing. I know there is Lora around but the quality of the Lora is worse (at least with Vace) compared to the full model. Don't hesitate to share your opinion about this on the discord server. 
+
+### June 17 2025: WanGP v6.1, Vace Powercharged
+Lots of improvements for Vace the Mother of all Models:
+- masks can now be combined with on the fly processing of a control video, for instance you can extract the motion of a specific person defined by a mask
+- on the fly modification of masks : reversed masks (with the same mask you can modify the background instead of the people covered by the masks), enlarged masks (you can cover more area if for instance the person you are trying to inject is larger than the one in the mask), ...
+- view these modified masks directly inside WanGP during the video generation to check they are really as expected
+- multiple frames injections: multiples frames can be injected at any location of the video
+- expand past videos in on click: just select one generated video to expand it
+
+Of course all these new stuff work on all Vace finetunes (including Vace Fusionix).
+
+Thanks also to Reevoy24 for adding a Notfication sound at the end of a generation and for fixing the background color of the current generation summary.
+
+### June 12 2025: WanGP v6.0
+👋 *Finetune models*: You find the 20 models supported by WanGP not sufficient ? Too impatient to wait for the next release to get the support for a newly released model ? Your prayers have been answered: if a new model is compatible with a model architecture supported by WanGP, you can add by yourself the support for this model in WanGP by just creating a finetune model definition. You can then store this model in the cloud (for instance in Huggingface) and the very light finetune definition file can be easily shared with other users. WanGP will download automatically the finetuned model for them.
+
+To celebrate the new finetunes support, here are a few finetune gifts (directly accessible from the model selection menu):
+- *Fast Hunyuan Video* : generate model t2v in only 6 steps
+- *Hunyuan Vido AccVideo* : generate model t2v in only 5 steps
+- *Wan FusioniX*: it is a combo of AccVideo / CausVid ans other models and can generate high quality Wan videos in only 8 steps
+
+One more thing...
+
+The new finetune system can be used to combine complementaty models : what happens when you combine  Fusionix Text2Video and Vace Control Net ?
+
+You get **Vace FusioniX**: the Ultimate Vace Model, Fast (10 steps, no need for guidance) and with a much better quality Video than the original slower model (despite being the best Control Net out there). Here goes one more finetune...
+
+Check the *Finetune Guide* to create finetune models definitions and share them on the WanGP discord server.
+
+### June 12 2025: WanGP v5.6
+👋 *Finetune models*: You find the 20 models supported by WanGP not sufficient ? Too impatient to wait for the next release to get the support for a newly released model ? Your prayers have been answered: if a new model is compatible with a model architecture supported by WanGP, you can add by yourself the support for this model in WanGP by just creating a finetune model definition. You can then store this model in the cloud (for instance in Huggingface) and the very light finetune definition file can be easily shared with other users. WanGP will download automatically the finetuned model for them.
+
+To celebrate the new finetunes support, here are a few finetune gifts (directly accessible from the model selection menu):
+- *Fast Hunyuan Video* : generate model t2v in only 6 steps
+- *Hunyuan Vido AccVideo* : generate model t2v in only 5 steps
+- *Wan FusioniX*: it is a combo of AccVideo / CausVid ans other models and can generate high quality Wan videos in only 8 steps
+
+One more thing...
+
+The new finetune system can be used to combine complementaty models : what happens when you combine  Fusionix Text2Video and Vace Control Net ?
+
+You get **Vace FusioniX**: the Ultimate Vace Model, Fast (10 steps, no need for guidance) and with a much better quality Video than the original slower model (despite being the best Control Net out there). Here goes one more finetune...
+
+Check the *Finetune Guide* to create finetune models definitions and share them on the WanGP discord server.
+
+### June 11 2025: WanGP v5.5
+👋 *Hunyuan Video Custom Audio*: it is similar to Hunyuan Video Avatar excpet there isn't any lower limit on the number of frames and you can use your reference images in a different context than the image itself\
+*Hunyuan Video Custom Edit*: Hunyuan Video Controlnet, use it to do inpainting and replace a person in a video while still keeping his poses. Similar to Vace but less restricted than the Wan models in terms of content...
+
+### June 6 2025: WanGP v5.41
+👋 Bonus release: Support for **AccVideo** Lora to speed up x2 Video generations in Wan models. Check the Loras documentation to get the usage instructions of AccVideo.
+
+### June 6 2025: WanGP v5.4
+👋 World Exclusive : Hunyuan Video Avatar Support ! You won't need 80 GB of VRAM nor 32 GB oF VRAM, just 10 GB of VRAM will be sufficient to generate up to 15s of high quality speech / song driven Video at a high speed with no quality degradation. Support for TeaCache included.
+
+### May 26, 2025: WanGP v5.3
+👋 Happy with a Video generation and want to do more generations using the same settings but you can't remember what you did or you find it too hard to copy/paste one per one each setting from the file metadata? Rejoice! There are now multiple ways to turn this tedious process into a one click task:
+- Select one Video recently generated in the Video Gallery and click *Use Selected Video Settings*
+- Click *Drop File Here* and select a Video you saved somewhere, if the settings metadata have been saved with the Video you will be able to extract them automatically
+- Click *Export Settings to File* to save on your harddrive the current settings. You will be able to use them later again by clicking *Drop File Here* and select this time a Settings json file
+
+### May 23, 2025: WanGP v5.21
+👋 Improvements for Vace: better transitions between Sliding Windows, Support for Image masks in Matanyone, new Extend Video for Vace, different types of automated background removal
+
+### May 20, 2025: WanGP v5.2
+👋 Added support for Wan CausVid which is a distilled Wan model that can generate nice looking videos in only 4 to 12 steps. The great thing is that Kijai (Kudos to him!) has created a CausVid Lora that can be combined with any existing Wan t2v model 14B like Wan Vace 14B. See [LORAS.md](LORAS.md) for instructions on how to use CausVid.
+
+Also as an experiment I have added support for the MoviiGen, the first model that claims to be capable of generating 1080p videos (if you have enough VRAM (20GB...) and be ready to wait for a long time...). Don't hesitate to share your impressions on the Discord server.
+
+### May 18, 2025: WanGP v5.1
+👋 Bonus Day, added LTX Video 13B Distilled: generate in less than one minute, very high quality Videos!
+
+### May 17, 2025: WanGP v5.0
+👋 One App to Rule Them All! Added support for the other great open source architectures:
+- **Hunyuan Video**: text 2 video (one of the best, if not the best t2v), image 2 video and the recently released Hunyuan Custom (very good identity preservation when injecting a person into a video)
+- **LTX Video 13B** (released last week): very long video support and fast 720p generation. Wan GP version has been greatly optimized and reduced LTX Video VRAM requirements by 4!
+
+Also:
+- Added support for the best Control Video Model, released 2 days ago: Vace 14B
+- New Integrated prompt enhancer to increase the quality of the generated videos
+
+*You will need one more `pip install -r requirements.txt`*
+
+### May 5, 2025: WanGP v4.5
+👋 FantasySpeaking model, you can animate a talking head using a voice track. This works not only on people but also on objects. Also better seamless transitions between Vace sliding windows for very long videos. New high quality processing features (mixed 16/32 bits calculation and 32 bits VAE)
+
+### April 27, 2025: WanGP v4.4
+👋 Phantom model support, very good model to transfer people or objects into video, works quite well at 720p and with the number of steps > 30
+
+### April 25, 2025: WanGP v4.3
+👋 Added preview mode and support for Sky Reels v2 Diffusion Forcing for high quality "infinite length videos". Note that Skyreel uses causal attention that is only supported by Sdpa attention so even if you choose another type of attention, some of the processes will use Sdpa attention.
+
+### April 18, 2025: WanGP v4.2
+👋 FLF2V model support, official support from Wan for image2video start and end frames specialized for 720p.
+
+### April 17, 2025: WanGP v4.1
+👋 Recam Master model support, view a video from a different angle. The video to process must be at least 81 frames long and you should set at least 15 steps denoising to get good results.
+
+### April 13, 2025: WanGP v4.0
+👋 Lots of goodies for you!
+- A new UI, tabs were replaced by a Dropdown box to easily switch models
+- A new queuing system that lets you stack in a queue as many text2video, image2video tasks, ... as you want. Each task can rely on complete different generation parameters (different number of frames, steps, loras, ...). Many thanks to **Tophness** for being a big contributor on this new feature
+- Temporal upsampling (Rife) and spatial upsampling (Lanczos) for a smoother video (32 fps or 64 fps) and to enlarge your video by x2 or x4. Check these new advanced options.
+- Wan Vace Control Net support: with Vace you can inject in the scene people or objects, animate a person, perform inpainting or outpainting, continue a video, ... See [VACE.md](VACE.md) for introduction guide.
+- Integrated *Matanyone* tool directly inside WanGP so that you can create easily inpainting masks used in Vace
+- Sliding Window generation for Vace, create windows that can last dozens of seconds
+- New optimizations for old generation GPUs: Generate 5s (81 frames, 15 steps) of Vace 1.3B with only 5GB and in only 6 minutes on a RTX 2080Ti and 5s of t2v 14B in less than 10 minutes.
+
+### March 27, 2025
+👋 Added support for the new Wan Fun InP models (image2video). The 14B Fun InP has probably better end image support but unfortunately existing loras do not work so well with it. The great novelty is the Fun InP image2 1.3B model: Image 2 Video is now accessible to even lower hardware configuration. It is not as good as the 14B models but very impressive for its size. Many thanks to the VideoX-Fun team (https://github.com/aigc-apps/VideoX-Fun)
+
+### March 26, 2025
+👋 Good news! Official support for RTX 50xx please check the [installation instructions](INSTALLATION.md).
+
+### March 24, 2025: Wan2.1GP v3.2
+👋 
+- Added Classifier-Free Guidance Zero Star. The video should match better the text prompt (especially with text2video) at no performance cost: many thanks to the **CFG Zero * Team**. Don't hesitate to give them a star if you appreciate the results: https://github.com/WeichenFan/CFG-Zero-star
+- Added back support for PyTorch compilation with Loras. It seems it had been broken for some time
+- Added possibility to keep a number of pregenerated videos in the Video Gallery (useful to compare outputs of different settings)
+
+*You will need one more `pip install -r requirements.txt`*
+
+### March 19, 2025: Wan2.1GP v3.1
+👋 Faster launch and RAM optimizations (should require less RAM to run)
+
+*You will need one more `pip install -r requirements.txt`*
+
+### March 18, 2025: Wan2.1GP v3.0
+👋 
+- New Tab based interface, you can switch from i2v to t2v conversely without restarting the app
+- Experimental Dual Frames mode for i2v, you can also specify an End frame. It doesn't always work, so you will need a few attempts.
+- You can save default settings in the files *i2v_settings.json* and *t2v_settings.json* that will be used when launching the app (you can also specify the path to different settings files)
+- Slight acceleration with loras
+
+*You will need one more `pip install -r requirements.txt`*
+
+Many thanks to *Tophness* who created the framework (and did a big part of the work) of the multitabs and saved settings features
+
+### March 18, 2025: Wan2.1GP v2.11
+👋 Added more command line parameters to prefill the generation settings + customizable output directory and choice of type of metadata for generated videos. Many thanks to *Tophness* for his contributions.
+
+*You will need one more `pip install -r requirements.txt` to reflect new dependencies*
+
+### March 18, 2025: Wan2.1GP v2.1
+👋 More Loras!: added support for 'Safetensors' and 'Replicate' Lora formats.
+
+*You will need to refresh the requirements with a `pip install -r requirements.txt`*
+
+### March 17, 2025: Wan2.1GP v2.0
+👋 The Lora festival continues:
+- Clearer user interface
+- Download 30 Loras in one click to try them all (expand the info section)
+- Very easy to use Loras as now Lora presets can input the subject (or other needed terms) of the Lora so that you don't have to modify manually a prompt
+- Added basic macro prompt language to prefill prompts with different values. With one prompt template, you can generate multiple prompts.
+- New Multiple images prompts: you can now combine any number of images with any number of text prompts (need to launch the app with --multiple-images)
+- New command lines options to launch directly the 1.3B t2v model or the 14B t2v model
+
+### March 14, 2025: Wan2.1GP v1.7
+👋 
+- Lora Fest special edition: very fast loading/unload of loras for those Loras collectors around. You can also now add/remove loras in the Lora folder without restarting the app.
+- Added experimental Skip Layer Guidance (advanced settings), that should improve the image quality at no extra cost. Many thanks to the *AmericanPresidentJimmyCarter* for the original implementation
+
+*You will need to refresh the requirements `pip install -r requirements.txt`*
+
+### March 13, 2025: Wan2.1GP v1.6
+👋 Better Loras support, accelerated loading Loras.
+
+*You will need to refresh the requirements `pip install -r requirements.txt`*
+
+### March 10, 2025: Wan2.1GP v1.5
+👋 Official Teacache support + Smart Teacache (find automatically best parameters for a requested speed multiplier), 10% speed boost with no quality loss, improved lora presets (they can now include prompts and comments to guide the user)
+
+### March 7, 2025: Wan2.1GP v1.4
+👋 Fix PyTorch compilation, now it is really 20% faster when activated
+
+### March 4, 2025: Wan2.1GP v1.3
+👋 Support for Image to Video with multiples images for different images/prompts combinations (requires *--multiple-images* switch), and added command line *--preload x* to preload in VRAM x MB of the main diffusion model if you find there is too much unused VRAM and you want to (slightly) accelerate the generation process.
+
+*If you upgrade you will need to do a `pip install -r requirements.txt` again.*
+
+### March 4, 2025: Wan2.1GP v1.2
+👋 Implemented tiling on VAE encoding and decoding. No more VRAM peaks at the beginning and at the end
+
+### March 3, 2025: Wan2.1GP v1.1
+👋 Added Tea Cache support for faster generations: optimization of kijai's implementation (https://github.com/kijai/ComfyUI-WanVideoWrapper/) of teacache (https://github.com/ali-vilab/TeaCache)
+
+### March 2, 2025: Wan2.1GP by DeepBeepMeep v1
+👋 Brings:
+- Support for all Wan including the Image to Video model
+- Reduced memory consumption by 2, with possibility to generate more than 10s of video at 720p with a RTX 4090 and 10s of video at 480p with less than 12GB of VRAM. Many thanks to REFLEx (https://github.com/thu-ml/RIFLEx) for their algorithm that allows generating nice looking video longer than 5s.
+- The usual perks: web interface, multiple generations, loras support, sage attention, auto download of models, ...
+
+## Original Wan Releases
+
+### February 25, 2025
+👋 We've released the inference code and weights of Wan2.1.
+
+### February 27, 2025
+👋 Wan2.1 has been integrated into [ComfyUI](https://comfyanonymous.github.io/ComfyUI_examples/wan/). Enjoy! 

docs/CLI.md

@@ -0,0 +1,226 @@
+--vace-1-3B--vace-1-3B# Command Line Reference
+
+This document covers all available command line options for WanGP.
+
+## Basic Usage
+
+```bash
+# Default launch 
+python wgp.py
+
+# Specific model modes
+python wgp.py --i2v           # Image-to-video
+python wgp.py --t2v           # Text-to-video (default)
+python wgp.py --t2v-14B       # 14B text-to-video model
+python wgp.py --t2v-1-3B      # 1.3B text-to-video model
+python wgp.py --i2v-14B       # 14B image-to-video model
+python wgp.py --i2v-1-3B      # Fun InP 1.3B image-to-video model
+python wgp.py --vace-1-3B     # VACE ControlNet 1.3B model
+```
+
+## Model and Performance Options
+
+### Model Configuration
+```bash
+--quantize-transformer BOOL   # Enable/disable transformer quantization (default: True)
+--compile                     # Enable PyTorch compilation (requires Triton)
+--attention MODE              # Force attention mode: sdpa, flash, sage, sage2
+--profile NUMBER              # Performance profile 1-5 (default: 4)
+--preload NUMBER              # Preload N MB of diffusion model in VRAM
+--fp16                        # Force fp16 instead of bf16 models
+--gpu DEVICE                  # Run on specific GPU device (e.g., "cuda:1")
+```
+
+### Performance Profiles
+- **Profile 1**: Load entire current model in VRAM and keep all unused models in reserved RAM for fast VRAM tranfers 
+- **Profile 2**: Load model parts as needed, keep all unused models in reserved RAM for fast VRAM tranfers
+- **Profile 3**: Load entire current model in VRAM (requires 24GB for 14B model)
+- **Profile 4**: Default and recommended, load model parts as needed, most flexible option
+- **Profile 5**: Minimum RAM usage
+
+### Memory Management
+```bash
+--perc-reserved-mem-max FLOAT # Max percentage of RAM for reserved memory (< 0.5)
+```
+
+## Lora Configuration
+
+```bash
+--lora-dir PATH              # Path to Wan t2v loras directory
+--lora-dir-i2v PATH          # Path to Wan i2v loras directory
+--lora-dir-hunyuan PATH      # Path to Hunyuan t2v loras directory
+--lora-dir-hunyuan-i2v PATH  # Path to Hunyuan i2v loras directory
+--lora-dir-ltxv PATH         # Path to LTX Video loras directory
+--lora-preset PRESET         # Load lora preset file (.lset) on startup
+--check-loras                # Filter incompatible loras (slower startup)
+```
+
+## Generation Settings
+
+### Basic Generation
+```bash
+--seed NUMBER                # Set default seed value
+--frames NUMBER              # Set default number of frames to generate
+--steps NUMBER               # Set default number of denoising steps
+--advanced                   # Launch with advanced mode enabled
+```
+
+### Advanced Generation
+```bash
+--teacache MULTIPLIER        # TeaCache speed multiplier: 0, 1.5, 1.75, 2.0, 2.25, 2.5
+```
+
+## Interface and Server Options
+
+### Server Configuration
+```bash
+--server-port PORT           # Gradio server port (default: 7860)
+--server-name NAME           # Gradio server name (default: localhost)
+--listen                     # Make server accessible on network
+--share                      # Create shareable HuggingFace URL for remote access
+--open-browser               # Open browser automatically when launching
+```
+
+### Interface Options
+```bash
+--lock-config                # Prevent modifying video engine configuration from interface
+--theme THEME_NAME           # UI theme: "default" or "gradio"
+```
+
+## File and Directory Options
+
+```bash
+--settings PATH              # Path to folder containing default settings for all models
+--verbose LEVEL              # Information level 0-2 (default: 1)
+```
+
+## Examples
+
+### Basic Usage Examples
+```bash
+# Launch with specific model and loras
+python wgp.py --t2v-14B --lora-preset mystyle.lset
+
+# High-performance setup with compilation
+python wgp.py --compile --attention sage2 --profile 3
+
+# Low VRAM setup
+python wgp.py --t2v-1-3B --profile 4 --attention sdpa
+
+# Multiple images with custom lora directory
+python wgp.py --i2v --multiple-images --lora-dir /path/to/shared/loras
+```
+
+### Server Configuration Examples
+```bash
+# Network accessible server
+python wgp.py --listen --server-port 8080
+
+# Shareable server with custom theme
+python wgp.py --share --theme gradio --open-browser
+
+# Locked configuration for public use
+python wgp.py --lock-config --share
+```
+
+### Advanced Performance Examples
+```bash
+# Maximum performance (requires high-end GPU)
+python wgp.py --compile --attention sage2 --profile 3 --preload 2000
+
+# Optimized for RTX 2080Ti
+python wgp.py --profile 4 --attention sdpa --teacache 2.0
+
+# Memory-efficient setup
+python wgp.py --fp16 --profile 4 --perc-reserved-mem-max 0.3
+```
+
+### TeaCache Configuration
+```bash
+# Different speed multipliers
+python wgp.py --teacache 1.5   # 1.5x speed, minimal quality loss
+python wgp.py --teacache 2.0   # 2x speed, some quality loss
+python wgp.py --teacache 2.5   # 2.5x speed, noticeable quality loss
+python wgp.py --teacache 0     # Disable TeaCache
+```
+
+## Attention Modes
+
+### SDPA (Default)
+```bash
+python wgp.py --attention sdpa
+```
+- Available by default with PyTorch
+- Good compatibility with all GPUs
+- Moderate performance
+
+### Sage Attention
+```bash
+python wgp.py --attention sage
+```
+- Requires Triton installation
+- 30% faster than SDPA
+- Small quality cost
+
+### Sage2 Attention
+```bash
+python wgp.py --attention sage2
+```
+- Requires Triton and SageAttention 2.x
+- 40% faster than SDPA
+- Best performance option
+
+### Flash Attention
+```bash
+python wgp.py --attention flash
+```
+- May require CUDA kernel compilation
+- Good performance
+- Can be complex to install on Windows
+
+## Troubleshooting Command Lines
+
+### Fallback to Basic Setup
+```bash
+# If advanced features don't work
+python wgp.py --attention sdpa --profile 4 --fp16
+```
+
+### Debug Mode
+```bash
+# Maximum verbosity for troubleshooting
+python wgp.py --verbose 2 --check-loras
+```
+
+### Memory Issue Debugging
+```bash
+# Minimal memory usage
+python wgp.py --profile 4 --attention sdpa --perc-reserved-mem-max 0.2
+```
+
+
+
+## Configuration Files
+
+### Settings Files
+Load custom settings:
+```bash
+python wgp.py --settings /path/to/settings/folder
+```
+
+### Lora Presets
+Create and share lora configurations:
+```bash
+# Load specific preset
+python wgp.py --lora-preset anime_style.lset
+
+# With custom lora directory
+python wgp.py --lora-preset mystyle.lset --lora-dir /shared/loras
+```
+
+## Environment Variables
+
+While not command line options, these environment variables can affect behavior:
+- `CUDA_VISIBLE_DEVICES` - Limit visible GPUs
+- `PYTORCH_CUDA_ALLOC_CONF` - CUDA memory allocation settings
+- `TRITON_CACHE_DIR` - Triton cache directory (for Sage attention) 

docs/FINETUNES.md

@@ -0,0 +1,85 @@
+# FINETUNES
+
+A Finetuned model is model that shares the same architecture of one specific model but has derived weights from this model. Some finetuned models have been created by combining multiple finetuned models.
+
+As there are potentially an infinite number of finetunes, specific finetuned models are not known by default by WanGP, however you can create a finetuned model definition that will tell WanGP about the existence of this finetuned model and WanGP will do as usual all the work for you: autodownload the model and build the user interface.
+
+Finetune models definitions are light json files that can be easily shared. You can find some of them on the WanGP *discord* server https://discord.gg/g7efUW9jGV
+
+Finetuned models have been tested so far with Wan2.1 text2video, Wan2.1 image2video,  Hunyuan Video text2video. There isn't currently any support for LTX Video finetunes.
+
+## Create a new Finetune Model Definition
+All the finetune models definitions are json files stored in the **finetunes** sub folder. All the corresponding finetune model weights will be stored in the *ckpts* subfolder and will sit next to the base models.
+
+WanGP comes with a few prebuilt finetune models that you can use as starting points and to get an idea of the structure of the definition file.
+
+A definition is built from a *settings file* that can contains all the default parameters for a video generation. On top of this file a subtree named **model** contains all the information regarding the finetune (URLs to download model, corresponding base model id, ...).
+
+You can obtain a settings file in several ways:
+- In the subfolder **settings**, get the json file that corresponds to the base model of your finetune (see the next section for the list of ids of base models)
+- From the user interface, go to the base model and click **export settings**
+
+Here are steps:
+1) Create a *settings file*
+2) Add a **model** subtree with the finetune description
+3) Save this file in the subfolder **finetunes**. The name used for the file will be used as its id. It is a good practise to prefix the name of this file with the base model. For instance for a finetune named **Fast*** based on  Hunyuan Text 2 Video model *hunyuan_t2v_fast.json*. In this example the Id is *hunyuan_t2v_fast*.
+4) Restart WanGP
+
+## Architecture Models Ids
+A finetune is derived from a base model and will inherit all the user interface and corresponding model capabilities, here are Architecture Ids:
+- *t2v*: Wan 2.1 Video text 2 
+- *i2v*: Wan 2.1 Video image 2 480p
+- *i2v_720p*: Wan 2.1 Video image 2 720p
+- *vace_14B*: Wan 2.1 Vace 14B
+- *hunyuan*: Hunyuan Video text 2 video
+- *hunyuan_i2v*: Hunyuan Video image 2 video
+
+## The Model Subtree
+- *name* : name of the finetune used to select
+- *architecture* : architecture Id of the base model of the finetune (see previous section)
+- *description*: description of the finetune that will appear at the top
+- *URLs*: URLs of all the finetune versions (quantized / non quantized). WanGP will pick the version that is the closest to the user preferences. You will need to follow a naming convention to help WanGP identify the content of each version (see next section). Right now WanGP supports only 8 bits quantized model that have been quantized using **quanto**. WanGP offers a command switch to build easily such a quantized model (see below). *URLs* can contain also paths to local file to allow testing.
+- *modules*: this a list of modules to be combined with the models referenced by the URLs. A module is a model extension that is merged with a model to expand its capabilities. So far the only module supported is Vace 14B  (its id is *vace_14B*). For instance the full Vace model is the fusion of a Wan text 2 video and the Vace module.
+- *preload_URLs* : URLs of files to download no matter what (used to load quantization maps for instance)
+- *auto_quantize*: if set to True and no quantized model URL is provided, WanGP will perform on the fly quantization if the user expects a quantized model
+
+Example of **model** subtree
+```
+	"model":
+	{
+		"name": "Wan text2video FusioniX 14B",
+		"architecture" : "t2v",
+		"description": "A powerful merged text-to-video model based on the original WAN 2.1 T2V model, enhanced using multiple open-source components and LoRAs to boost motion realism, temporal consistency, and expressive detail. multiple open-source models and LoRAs to boost temporal quality, expressiveness, and motion realism.",
+		"URLs": [
+			"https://huggingface.co/DeepBeepMeep/Wan2.1/resolve/main/Wan14BT2VFusioniX_fp16.safetensors",
+			"https://huggingface.co/DeepBeepMeep/Wan2.1/resolve/main/Wan14BT2VFusioniX_quanto_fp16_int8.safetensors",
+			"https://huggingface.co/DeepBeepMeep/Wan2.1/resolve/main/Wan14BT2VFusioniX_quanto_bf16_int8.safetensors"
+		],
+        "preload_URLs": [
+        ],
+		"auto_quantize": true
+	},
+```
+
+## Finetune Model Naming Convention
+If a model is not quantized, it is assumed to be mostly 16 bits (with maybe a few 32 bits weights), so *bf16* or *fp16* should appear somewhere in the name. If you need examples just look at the **ckpts** subfolder, the naming convention for the base models is the same.
+
+If a model is quantized the term *quanto* should also be included since WanGP supports for the moment only *quanto* quantized model, most specically you should replace *fp16* by *quanto_fp16_int8* or *bf6* by *quanto_bf16_int8*. 
+
+Please note it is important than *bf16", "fp16* and *quanto* are all in lower cases letters.
+
+## Creating a Quanto Quantized file
+If you launch the app with the *--save-quantized* switch, WanGP will create a quantized file in the **ckpts** subfolder just after the model has been loaded. Please note that the model will *bf16* or *fp16* quantized depending on what you chose in the configuration menu.
+
+1) Make sure that in the finetune definition json file there is only a URL or filepath that points to the non quantized model
+2) Launch WanGP *python wgp.py --save-quantized*
+3) In the configuration menu *Transformer Data Type* property choose either *BF16* of *FP16*
+4) Launch a video generation (settings used do not matter). As soon as the model is loaded, a new quantized model will be created in the **ckpts** subfolder if it doesn't already exist.
+5) WanGP will update automatically the finetune definition file with the local path of the newly created quantized file (the list "URLs" will have an extra value such as *"ckpts/finetune_quanto_fp16_int8.safetensors"*
+6) Remove *--save-quantized*, restart WanGP and select *Scaled Int8 Quantization* in the *Transformer Model Quantization* property
+7) Launch a new generation and verify in the terminal window that the right quantized model is loaded
+8) In order to share the finetune definition file you will need to store the fine model weights in the cloud. You can upload them for instance on *Huggingface*. You can now replace in the finetune definition file the local path by a URL (on Huggingface to get the URL of the model file click *Copy download link* when accessing the model properties)
+
+You need to create a quantized model specifically for *bf16* or *fp16* as they can not converted on the fly. However there is no need for a non quantized model as they can be converted on the fly while being loaded.
+
+Wan models supports both *fp16* and *bf16* data types albeit *fp16* delivers in theory better quality. On the contrary Hunyuan and LTXV supports only *bf16*.

docs/GETTING_STARTED.md

@@ -0,0 +1,194 @@
+# Getting Started with WanGP
+
+This guide will help you get started with WanGP video generation quickly and easily.
+
+## Prerequisites
+
+Before starting, ensure you have:
+- A compatible GPU (RTX 10XX or newer recommended)
+- Python 3.10.9 installed
+- At least 6GB of VRAM for basic models
+- Internet connection for model downloads
+
+## Quick Setup
+
+### Option 1: One-Click Installation (Recommended)
+Use [Pinokio App](https://pinokio.computer/) for the easiest installation experience.
+
+### Option 2: Manual Installation
+```bash
+git clone https://github.com/deepbeepmeep/Wan2GP.git
+cd Wan2GP
+conda create -n wan2gp python=3.10.9
+conda activate wan2gp
+pip install torch==2.6.0 torchvision torchaudio --index-url https://download.pytorch.org/whl/test/cu124
+pip install -r requirements.txt
+```
+
+For detailed installation instructions, see [INSTALLATION.md](INSTALLATION.md).
+
+## First Launch
+
+### Basic Launch
+```bash
+python wgp.py
+```
+This launches the WanGP generator with default settings. You will be able to pick from a Drop Down menu which model you want to use.
+
+### Alternative Modes
+```bash
+python wgp.py --i2v        # Wan Image-to-video mode
+python wgp.py --t2v-1-3B   # Wan Smaller, faster model
+```
+
+## Understanding the Interface
+
+When you launch WanGP, you'll see a web interface with several sections:
+
+### Main Generation Panel
+- **Model Selection**: Dropdown to choose between different models
+- **Prompt**: Text description of what you want to generate
+- **Generate Button**: Start the video generation process
+
+### Advanced Settings (click checkbox to enable)
+- **Generation Settings**: Steps, guidance, seeds
+- **Loras**: Additional style customizations
+- **Sliding Window**: For longer videos
+
+## Your First Video
+
+Let's generate a simple text-to-video:
+
+1. **Launch WanGP**: `python wgp.py`
+2. **Open Browser**: Navigate to `http://localhost:7860`
+3. **Enter Prompt**: "A cat walking in a garden"
+4. **Click Generate**: Wait for the video to be created
+5. **View Result**: The video will appear in the output section
+
+### Recommended First Settings
+- **Model**: Wan 2.1 text2video 1.3B (faster, lower VRAM)
+- **Frames**: 49 (about 2 seconds)
+- **Steps**: 20 (good balance of speed/quality)
+
+## Model Selection
+
+### Text-to-Video Models
+- **Wan 2.1 T2V 1.3B**: Fastest, lowest VRAM (6GB), good quality
+- **Wan 2.1 T2V 14B**: Best quality, requires more VRAM (12GB+)
+- **Hunyuan Video**: Excellent quality, slower generation
+- **LTX Video**: Good for longer videos
+
+### Image-to-Video Models
+- **Wan Fun InP 1.3B**: Fast image animation
+- **Wan Fun InP 14B**: Higher quality image animation
+- **VACE**: Advanced control over video generation
+
+### Choosing the Right Model
+- **Low VRAM (6-8GB)**: Use 1.3B models
+- **Medium VRAM (10-12GB)**: Use 14B models or Hunyuan
+- **High VRAM (16GB+)**: Any model, longer videos
+
+## Basic Settings Explained
+
+### Generation Settings
+- **Frames**: Number of frames (more = longer video)
+  - 25 frames ≈ 1 second
+  - 49 frames ≈ 2 seconds
+  - 73 frames ≈ 3 seconds
+
+- **Steps**: Quality vs Speed tradeoff
+  - 15 steps: Fast, lower quality
+  - 20 steps: Good balance
+  - 30+ steps: High quality, slower
+
+- **Guidance Scale**: How closely to follow the prompt
+  - 3-5: More creative interpretation
+  - 7-10: Closer to prompt description
+  - 12+: Very literal interpretation
+
+### Seeds
+- **Random Seed**: Different result each time
+- **Fixed Seed**: Reproducible results
+- **Use same seed + prompt**: Generate variations
+
+## Common Beginner Issues
+
+### "Out of Memory" Errors
+1. Use smaller models (1.3B instead of 14B)
+2. Reduce frame count
+3. Lower resolution in advanced settings
+4. Enable quantization (usually on by default)
+
+### Slow Generation
+1. Use 1.3B models for speed
+2. Reduce number of steps
+3. Install Sage attention (see [INSTALLATION.md](INSTALLATION.md))
+4. Enable TeaCache: `python wgp.py --teacache 2.0`
+
+### Poor Quality Results
+1. Increase number of steps (25-30)
+2. Improve prompt description
+3. Use 14B models if you have enough VRAM
+4. Enable Skip Layer Guidance in advanced settings
+
+## Writing Good Prompts
+
+### Basic Structure
+```
+[Subject] [Action] [Setting] [Style/Quality modifiers]
+```
+
+### Examples
+```
+A red sports car driving through a mountain road at sunset, cinematic, high quality
+
+A woman with long hair walking on a beach, waves in the background, realistic, detailed
+
+A cat sitting on a windowsill watching rain, cozy atmosphere, soft lighting
+```
+
+### Tips
+- Be specific about what you want
+- Include style descriptions (cinematic, realistic, etc.)
+- Mention lighting and atmosphere
+- Describe the setting in detail
+- Use quality modifiers (high quality, detailed, etc.)
+
+## Next Steps
+
+Once you're comfortable with basic generation:
+
+1. **Explore Advanced Features**:
+   - [Loras Guide](LORAS.md) - Customize styles and characters
+   - [VACE ControlNet](VACE.md) - Advanced video control
+   - [Command Line Options](CLI.md) - Optimize performance
+
+2. **Improve Performance**:
+   - Install better attention mechanisms
+   - Optimize memory settings
+   - Use compilation for speed
+
+3. **Join the Community**:
+   - [Discord Server](https://discord.gg/g7efUW9jGV) - Get help and share videos
+   - Share your best results
+   - Learn from other users
+
+## Troubleshooting First Steps
+
+### Installation Issues
+- Ensure Python 3.10.9 is used
+- Check CUDA version compatibility
+- See [INSTALLATION.md](INSTALLATION.md) for detailed steps
+
+### Generation Issues
+- Check GPU compatibility
+- Verify sufficient VRAM
+- Try basic settings first
+- See [TROUBLESHOOTING.md](TROUBLESHOOTING.md) for specific issues
+
+### Performance Issues
+- Use appropriate model for your hardware
+- Enable performance optimizations
+- Check [CLI.md](CLI.md) for optimization flags
+
+Remember: Start simple and gradually explore more advanced features as you become comfortable with the basics! 

docs/INSTALLATION.md

@@ -0,0 +1,170 @@
+# Installation Guide
+
+This guide covers installation for different GPU generations and operating systems.
+
+## Requirements
+
+- Python 3.10.9
+- Conda or Python venv
+- Compatible GPU (RTX 10XX or newer recommended)
+
+## Installation for RTX 10XX to RTX 40XX (Stable)
+
+This installation uses PyTorch 2.6.0 which is well-tested and stable.
+
+### Step 1: Download and Setup Environment
+
+```shell
+# Clone the repository
+git clone https://github.com/deepbeepmeep/Wan2GP.git
+cd Wan2GP
+
+# Create Python 3.10.9 environment using conda
+conda create -n wan2gp python=3.10.9
+conda activate wan2gp
+```
+
+### Step 2: Install PyTorch
+
+```shell
+# Install PyTorch 2.6.0 with CUDA 12.4
+pip install torch==2.6.0 torchvision torchaudio --index-url https://download.pytorch.org/whl/test/cu124  
+```
+
+### Step 3: Install Dependencies
+
+```shell
+# Install core dependencies
+pip install -r requirements.txt
+```
+
+### Step 4: Optional Performance Optimizations
+
+#### Sage Attention (30% faster)
+
+```shell
+# Windows only: Install Triton
+pip install triton-windows 
+
+# For both Windows and Linux
+pip install sageattention==1.0.6 
+```
+
+#### Sage 2 Attention (40% faster)
+
+```shell
+# Windows
+pip install triton-windows 
+pip install https://github.com/woct0rdho/SageAttention/releases/download/v2.1.1-windows/sageattention-2.1.1+cu126torch2.6.0-cp310-cp310-win_amd64.whl
+
+# Linux (manual compilation required)
+git clone https://github.com/thu-ml/SageAttention
+cd SageAttention 
+pip install -e .
+```
+
+#### Flash Attention
+
+```shell
+# May require CUDA kernel compilation on Windows
+pip install flash-attn==2.7.2.post1
+```
+
+## Installation for RTX 50XX (Beta)
+
+RTX 50XX GPUs require PyTorch 2.7.0 (beta). This version may be less stable.
+
+⚠️ **Important:** Use Python 3.10 for compatibility with pip wheels.
+
+### Step 1: Setup Environment
+
+```shell
+# Clone and setup (same as above)
+git clone https://github.com/deepbeepmeep/Wan2GP.git
+cd Wan2GP
+conda create -n wan2gp python=3.10.9
+conda activate wan2gp
+```
+
+### Step 2: Install PyTorch Beta
+
+```shell
+# Install PyTorch 2.7.0 with CUDA 12.8
+pip install torch==2.7.0 torchvision torchaudio --index-url https://download.pytorch.org/whl/test/cu128
+```
+
+### Step 3: Install Dependencies
+
+```shell
+pip install -r requirements.txt
+```
+
+### Step 4: Optional Optimizations for RTX 50XX
+
+#### Sage Attention
+
+```shell
+# Windows
+pip install triton-windows 
+pip install sageattention==1.0.6 
+
+# Linux
+pip install sageattention==1.0.6
+```
+
+#### Sage 2 Attention
+
+```shell
+# Windows
+pip install triton-windows 
+pip install https://github.com/woct0rdho/SageAttention/releases/download/v2.1.1-windows/sageattention-2.1.1+cu128torch2.7.0-cp310-cp310-win_amd64.whl 
+
+# Linux (manual compilation)
+git clone https://github.com/thu-ml/SageAttention
+cd SageAttention 
+pip install -e .
+```
+
+## Attention Modes
+
+WanGP supports several attention implementations:
+
+- **SDPA** (default): Available by default with PyTorch
+- **Sage**: 30% speed boost with small quality cost
+- **Sage2**: 40% speed boost 
+- **Flash**: Good performance, may be complex to install on Windows
+
+## Performance Profiles
+
+Choose a profile based on your hardware:
+
+- **Profile 3 (LowRAM_HighVRAM)**: Loads entire model in VRAM, requires 24GB VRAM for 8-bit quantized 14B model
+- **Profile 4 (LowRAM_LowVRAM)**: Default, loads model parts as needed, slower but lower VRAM requirement
+
+## Troubleshooting
+
+### Sage Attention Issues
+
+If Sage attention doesn't work:
+
+1. Check if Triton is properly installed
+2. Clear Triton cache
+3. Fallback to SDPA attention:
+   ```bash
+   python wgp.py --attention sdpa
+   ```
+
+### Memory Issues
+
+- Use lower resolution or shorter videos
+- Enable quantization (default)
+- Use Profile 4 for lower VRAM usage
+- Consider using 1.3B models instead of 14B models
+
+### GPU Compatibility
+
+- RTX 10XX, 20XX: Supported with SDPA attention
+- RTX 30XX, 40XX: Full feature support
+- RTX 50XX: Beta support with PyTorch 2.7.0
+
+For more troubleshooting, see [TROUBLESHOOTING.md](TROUBLESHOOTING.md) 

docs/LORAS.md

@@ -0,0 +1,252 @@
+# Loras Guide
+
+Loras (Low-Rank Adaptations) allow you to customize video generation models by adding specific styles, characters, or effects to your videos.
+
+## Directory Structure
+
+Loras are organized in different folders based on the model they're designed for:
+
+### Text-to-Video Models
+- `loras/` - General t2v loras
+- `loras/1.3B/` - Loras specifically for 1.3B models
+- `loras/14B/` - Loras specifically for 14B models
+
+### Image-to-Video Models
+- `loras_i2v/` - Image-to-video loras
+
+### Other Models
+- `loras_hunyuan/` - Hunyuan Video t2v loras
+- `loras_hunyuan_i2v/` - Hunyuan Video i2v loras
+- `loras_ltxv/` - LTX Video loras
+
+## Custom Lora Directory
+
+You can specify custom lora directories when launching the app:
+
+```bash
+# Use shared lora directory for both t2v and i2v
+python wgp.py --lora-dir /path/to/shared/loras --lora-dir-i2v /path/to/shared/loras
+
+# Specify different directories for different models
+python wgp.py --lora-dir-hunyuan /path/to/hunyuan/loras --lora-dir-ltxv /path/to/ltx/loras
+```
+
+## Using Loras
+
+### Basic Usage
+
+1. Place your lora files in the appropriate directory
+2. Launch WanGP
+3. In the Advanced Tab, select the "Loras" section
+4. Check the loras you want to activate
+5. Set multipliers for each lora (default is 1.0)
+
+### Lora Multipliers
+
+Multipliers control the strength of each lora's effect:
+
+#### Simple Multipliers
+```
+1.2 0.8
+```
+- First lora: 1.2 strength
+- Second lora: 0.8 strength
+
+#### Time-based Multipliers
+For dynamic effects over generation steps, use comma-separated values:
+```
+0.9,0.8,0.7
+1.2,1.1,1.0
+```
+- For 30 steps: steps 0-9 use first value, 10-19 use second, 20-29 use third
+- First lora: 0.9 → 0.8 → 0.7
+- Second lora: 1.2 → 1.1 → 1.0
+
+## Lora Presets
+
+Presets are combinations of loras with predefined multipliers and prompts.
+
+### Creating Presets
+1. Configure your loras and multipliers
+2. Write a prompt with comments (lines starting with #)
+3. Save as a preset with `.lset` extension
+
+### Example Preset
+```
+# Use the keyword "ohnvx" to trigger the lora
+A ohnvx character is driving a car through the city
+```
+
+### Using Presets
+```bash
+# Load preset on startup
+python wgp.py --lora-preset mypreset.lset
+```
+
+### Managing Presets
+- Edit, save, or delete presets directly from the web interface
+- Presets include comments with usage instructions
+- Share `.lset` files with other users
+
+## Supported Formats
+
+WanGP supports multiple lora formats:
+- **Safetensors** (.safetensors)
+- **Replicate** format
+- **Standard PyTorch** (.pt, .pth)
+
+## Safe-Forcing lightx2v Lora (Video Generation Accelerator)
+
+Safeforcing Lora has been created by Kijai from the Safe-Forcing lightx2v distilled Wan model and can generate videos with only 2 steps and offers also a 2x speed improvement since it doesnt require classifier free guidance. It works on both t2v and i2v models
+
+### Setup Instructions
+1. Download the Lora:
+   ```
+   https://huggingface.co/Kijai/WanVideo_comfy/blob/main/Wan21_T2V_14B_lightx2v_cfg_step_distill_lora_rank32.safetensors
+   ```
+2. Place in your `loras/` directory
+
+### Usage
+1. Select a Wan t2v or i2v model (e.g., Wan 2.1 text2video 13B or Vace 13B)
+2. Enable Advanced Mode
+3. In Advanced Generation Tab:
+   - Set Guidance Scale = 1
+   - Set Shift Scale = 5
+4. In Advanced Lora Tab:
+   - Select the Lora above
+   - Set multiplier to 1
+5. Set generation steps to 2-8
+6. Generate!
+
+## CausVid Lora (Video Generation Accelerator)
+
+CausVid is a distilled Wan model that generates videos in 4-12 steps with 2x speed improvement.
+
+### Setup Instructions
+1. Download the CausVid Lora:
+   ```
+   https://huggingface.co/Kijai/WanVideo_comfy/blob/main/Wan21_CausVid_14B_T2V_lora_rank32.safetensors
+   ```
+2. Place in your `loras/` directory
+
+### Usage
+1. Select a Wan t2v model (e.g., Wan 2.1 text2video 13B or Vace 13B)
+2. Enable Advanced Mode
+3. In Advanced Generation Tab:
+   - Set Guidance Scale = 1
+   - Set Shift Scale = 7
+4. In Advanced Lora Tab:
+   - Select CausVid Lora
+   - Set multiplier to 0.3
+5. Set generation steps to 12
+6. Generate!
+
+### CausVid Step/Multiplier Relationship
+- **12 steps**: 0.3 multiplier (recommended)
+- **8 steps**: 0.5-0.7 multiplier
+- **4 steps**: 0.8-1.0 multiplier
+
+*Note: Lower steps = lower quality (especially motion)*
+
+
+
+## AccVid Lora (Video Generation Accelerator)
+
+AccVid is a distilled Wan model that generates videos with a 2x speed improvement since classifier free guidance is no longer needed (that is cfg = 1).
+
+### Setup Instructions
+1. Download the AccVid Lora:
+
+- for t2v models:
+   ```
+   https://huggingface.co/Kijai/WanVideo_comfy/blob/main/Wan21_AccVid_T2V_14B_lora_rank32_fp16.safetensors
+   ```
+
+- for i2v models:
+   ```
+   https://huggingface.co/Kijai/WanVideo_comfy/blob/main/Wan21_AccVid_I2V_480P_14B_lora_rank32_fp16.safetensors
+   ```
+
+2. Place in your `loras/` directory or `loras_i2v/` directory
+
+### Usage
+1. Select a Wan t2v model (e.g., Wan 2.1 text2video 13B or Vace 13B) or Wan i2v model
+2. Enable Advanced Mode
+3. In Advanced Generation Tab:
+   - Set Guidance Scale = 1
+   - Set Shift Scale = 5
+4. The number steps remain unchanged compared to what you would use with the original model but it will be two times faster since classifier free guidance is not needed
+
+
+https://huggingface.co/Kijai/WanVideo_comfy/blob/main/Wan21_T2V_14B_lightx2v_cfg_step_distill_lora_rank32.safetensors
+
+## Performance Tips
+
+### Fast Loading/Unloading
+- Loras can be added/removed without restarting the app
+- Use the "Refresh" button to detect new loras
+- Enable `--check-loras` to filter incompatible loras (slower startup)
+
+### Memory Management
+- Loras are loaded on-demand to save VRAM
+- Multiple loras can be used simultaneously
+- Time-based multipliers don't use extra memory
+
+## Finding Loras
+
+### Sources
+- **[Civitai](https://civitai.com/)** - Large community collection
+- **HuggingFace** - Official and community loras
+- **Discord Server** - Community recommendations
+
+### Creating Loras
+- **Kohya** - Popular training tool
+- **OneTrainer** - Alternative training solution
+- **Custom datasets** - Train on your own content
+
+## Macro System (Advanced)
+
+Create multiple prompts from templates using macros:
+
+```
+! {Subject}="cat","woman","man", {Location}="forest","lake","city", {Possessive}="its","her","his"
+In the video, a {Subject} is presented. The {Subject} is in a {Location} and looks at {Possessive} watch.
+```
+
+This generates:
+1. "In the video, a cat is presented. The cat is in a forest and looks at its watch."
+2. "In the video, a woman is presented. The woman is in a lake and looks at her watch."
+3. "In the video, a man is presented. The man is in a city and looks at his watch."
+
+## Troubleshooting
+
+### Lora Not Working
+1. Check if lora is compatible with your model size (1.3B vs 14B)
+2. Verify lora format is supported
+3. Try different multiplier values
+4. Check the lora was trained for your model type (t2v vs i2v)
+
+### Performance Issues
+1. Reduce number of active loras
+2. Lower multiplier values
+3. Use `--check-loras` to filter incompatible files
+4. Clear lora cache if issues persist
+
+### Memory Errors
+1. Use fewer loras simultaneously
+2. Reduce model size (use 1.3B instead of 14B)
+3. Lower video resolution or frame count
+4. Enable quantization if not already active
+
+## Command Line Options
+
+```bash
+# Lora-related command line options
+--lora-dir path                    # Path to t2v loras directory
+--lora-dir-i2v path               # Path to i2v loras directory  
+--lora-dir-hunyuan path           # Path to Hunyuan t2v loras
+--lora-dir-hunyuan-i2v path       # Path to Hunyuan i2v loras
+--lora-dir-ltxv path              # Path to LTX Video loras
+--lora-preset preset              # Load preset on startup
+--check-loras                     # Filter incompatible loras
+``` 

docs/MODELS.md

@@ -0,0 +1,275 @@
+# Models Overview
+
+WanGP supports multiple video generation models, each optimized for different use cases and hardware configurations. 
+
+
+## Wan 2.1 Text2Video Models
+Please note that that the term *Text2Video* refers to the underlying Wan architecture but as it has been greatly improved overtime many derived Text2Video models can now  generate videos using images.
+
+#### Wan 2.1 Text2Video 1.3B
+- **Size**: 1.3 billion parameters
+- **VRAM**: 6GB minimum
+- **Speed**: Fast generation
+- **Quality**: Good quality for the size
+- **Best for**: Quick iterations, lower-end hardware
+- **Command**: `python wgp.py --t2v-1-3B`
+
+#### Wan 2.1 Text2Video 14B
+- **Size**: 14 billion parameters  
+- **VRAM**: 12GB+ recommended
+- **Speed**: Slower but higher quality
+- **Quality**: Excellent detail and coherence
+- **Best for**: Final production videos
+- **Command**: `python wgp.py --t2v-14B`
+
+#### Wan Vace 1.3B
+- **Type**: ControlNet for advanced video control
+- **VRAM**: 6GB minimum
+- **Features**: Motion transfer, object injection, inpainting
+- **Best for**: Advanced video manipulation
+- **Command**: `python wgp.py --vace-1.3B`
+
+#### Wan Vace 14B
+- **Type**: Large ControlNet model
+- **VRAM**: 12GB+ recommended
+- **Features**: All Vace features with higher quality
+- **Best for**: Professional video editing workflows
+
+#### MoviiGen (Experimental)
+- **Resolution**: Claims 1080p capability
+- **VRAM**: 20GB+ required
+- **Speed**: Very slow generation
+- **Features**: Should generate cinema like video, specialized for 2.1 / 1 ratios
+- **Status**: Experimental, feedback welcome
+
+<BR>
+
+## Wan 2.1 Image-to-Video Models
+
+#### Wan 2.1 Image2Video 14B
+- **Size**: 14 billion parameters  
+- **VRAM**: 12GB+ recommended
+- **Speed**: Slower but higher quality
+- **Quality**: Excellent detail and coherence
+- **Best for**: Most Loras available work with this model
+- **Command**: `python wgp.py --i2v-14B`
+
+#### FLF2V
+- **Type**: Start/end frame specialist
+- **Resolution**: Optimized for 720p
+- **Official**: Wan team supported
+- **Use case**: Image-to-video with specific endpoints
+
+
+<BR>
+
+## Wan 2.1 Specialized Models
+
+#### FantasySpeaking
+- **Type**: Talking head animation
+- **Input**: Voice track + image
+- **Works on**: People and objects
+- **Use case**: Lip-sync and voice-driven animation
+
+#### Phantom
+- **Type**: Person/object transfer
+- **Resolution**: Works well at 720p
+- **Requirements**: 30+ steps for good results
+- **Best for**: Transferring subjects between videos
+
+#### Recam Master
+- **Type**: Viewpoint change
+- **Requirements**: 81+ frame input videos, 15+ denoising steps
+- **Use case**: View same scene from different angles
+
+#### Sky Reels v2
+- **Type**: Diffusion Forcing model
+- **Specialty**: "Infinite length" videos
+- **Features**: High quality continuous generation
+
+
+<BR>
+
+## Wan Fun InP Models
+
+#### Wan Fun InP 1.3B
+- **Size**: 1.3 billion parameters
+- **VRAM**: 6GB minimum
+- **Quality**: Good for the size, accessible to lower hardware
+- **Best for**: Entry-level image animation
+- **Command**: `python wgp.py --i2v-1-3B`
+
+#### Wan Fun InP 14B
+- **Size**: 14 billion parameters
+- **VRAM**: 12GB+ recommended
+- **Quality**: Better end image support
+- **Limitation**: Existing loras don't work as well
+
+<BR>
+
+## Wan Special Loras
+### Safe-Forcing lightx2v Lora
+- **Type**: Distilled model (Lora implementation)
+- **Speed**: 4-8 steps generation, 2x faster (no classifier free guidance)
+- **Compatible**: Works with t2v and i2v Wan 14B models
+- **Setup**: Requires Safe-Forcing lightx2v Lora (see [LORAS.md](LORAS.md))
+
+
+### Causvid Lora
+- **Type**: Distilled model (Lora implementation)
+- **Speed**: 4-12 steps generation, 2x faster (no classifier free guidance)
+- **Compatible**: Works with Wan 14B models
+- **Setup**: Requires CausVid Lora (see [LORAS.md](LORAS.md))
+
+
+<BR>
+
+## Hunyuan Video Models
+
+#### Hunyuan Video Text2Video
+- **Quality**: Among the best open source t2v models
+- **VRAM**: 12GB+ recommended
+- **Speed**: Slower generation but excellent results
+- **Features**: Superior text adherence and video quality, up to 10s of video
+- **Best for**: High-quality text-to-video generation
+
+#### Hunyuan Video Custom
+- **Specialty**: Identity preservation
+- **Use case**: Injecting specific people into videos
+- **Quality**: Excellent for character consistency
+- **Best for**: Character-focused video generation
+
+#### Hunyuan Video Avater
+- **Specialty**: Generate up to 15s of high quality speech / song driven Video .
+- **Use case**: Injecting specific people into videos
+- **Quality**: Excellent for character consistency
+- **Best for**: Character-focused video generation, Video synchronized with voice
+
+
+<BR>
+
+## LTX Video Models
+
+#### LTX Video 13B
+- **Specialty**: Long video generation
+- **Resolution**: Fast 720p generation
+- **VRAM**: Optimized by WanGP (4x reduction in requirements)
+- **Best for**: Longer duration videos
+
+#### LTX Video 13B Distilled
+- **Speed**: Generate in less than one minute
+- **Quality**: Very high quality despite speed
+- **Best for**: Rapid prototyping and quick results
+
+<BR>
+
+## Model Selection Guide
+
+### By Hardware (VRAM)
+
+#### 6-8GB VRAM
+- Wan 2.1 T2V 1.3B
+- Wan Fun InP 1.3B
+- Wan Vace 1.3B
+
+#### 10-12GB VRAM
+- Wan 2.1 T2V 14B
+- Wan Fun InP 14B
+- Hunyuan Video (with optimizations)
+- LTX Video 13B
+
+#### 16GB+ VRAM
+- All models supported
+- Longer videos possible
+- Higher resolutions
+- Multiple simultaneous Loras
+
+#### 20GB+ VRAM
+- MoviiGen (experimental 1080p)
+- Very long videos
+- Maximum quality settings
+
+### By Use Case
+
+#### Quick Prototyping
+1. **LTX Video 13B Distilled** - Fastest, high quality
+2. **Wan 2.1 T2V 1.3B** - Fast, good quality
+3. **CausVid Lora** - 4-12 steps, very fast
+
+#### Best Quality
+1. **Hunyuan Video** - Overall best t2v quality
+2. **Wan 2.1 T2V 14B** - Excellent Wan quality
+3. **Wan Vace 14B** - Best for controlled generation
+
+#### Advanced Control
+1. **Wan Vace 14B/1.3B** - Motion transfer, object injection
+2. **Phantom** - Person/object transfer
+3. **FantasySpeaking** - Voice-driven animation
+
+#### Long Videos
+1. **LTX Video 13B** - Specialized for length
+2. **Sky Reels v2** - Infinite length videos
+3. **Wan Vace + Sliding Windows** - Up to 1 minute
+
+#### Lower Hardware
+1. **Wan Fun InP 1.3B** - Image-to-video
+2. **Wan 2.1 T2V 1.3B** - Text-to-video
+3. **Wan Vace 1.3B** - Advanced control
+
+<BR>
+
+## Performance Comparison
+
+### Speed (Relative)
+1. **CausVid Lora** (4-12 steps) - Fastest
+2. **LTX Video Distilled** - Very fast
+3. **Wan 1.3B models** - Fast
+4. **Wan 14B models** - Medium
+5. **Hunyuan Video** - Slower
+6. **MoviiGen** - Slowest
+
+### Quality (Subjective)
+1. **Hunyuan Video** - Highest overall
+2. **Wan 14B models** - Excellent
+3. **LTX Video models** - Very good
+4. **Wan 1.3B models** - Good
+5. **CausVid** - Good (varies with steps)
+
+### VRAM Efficiency
+1. **Wan 1.3B models** - Most efficient
+2. **LTX Video** (with WanGP optimizations)
+3. **Wan 14B models**
+4. **Hunyuan Video**
+5. **MoviiGen** - Least efficient
+
+<BR>
+
+## Model Switching
+
+WanGP allows switching between models without restarting:
+
+1. Use the dropdown menu in the web interface
+2. Models are loaded on-demand
+3. Previous model is unloaded to save VRAM
+4. Settings are preserved when possible
+
+<BR>
+
+## Tips for Model Selection
+
+### First Time Users
+Start with **Wan 2.1 T2V 1.3B** to learn the interface and test your hardware.
+
+### Production Work
+Use **Hunyuan Video** or **Wan 14B** models for final output quality.
+
+### Experimentation
+**CausVid Lora** or **LTX Distilled** for rapid iteration and testing.
+
+### Specialized Tasks
+- **VACE** for advanced control
+- **FantasySpeaking** for talking heads
+- **LTX Video** for long sequences
+
+### Hardware Optimization
+Always start with the largest model your VRAM can handle, then optimize settings for speed vs quality based on your needs. 

docs/TROUBLESHOOTING.md

@@ -0,0 +1,338 @@
+# Troubleshooting Guide
+
+This guide covers common issues and their solutions when using WanGP.
+
+## Installation Issues
+
+### PyTorch Installation Problems
+
+#### CUDA Version Mismatch
+**Problem**: PyTorch can't detect GPU or CUDA errors
+**Solution**: 
+```bash
+# Check your CUDA version
+nvidia-smi
+
+# Install matching PyTorch version
+# For CUDA 12.4 (RTX 10XX-40XX)
+pip install torch==2.6.0 torchvision torchaudio --index-url https://download.pytorch.org/whl/test/cu124
+
+# For CUDA 12.8 (RTX 50XX)
+pip install torch==2.7.0 torchvision torchaudio --index-url https://download.pytorch.org/whl/test/cu128
+```
+
+#### Python Version Issues
+**Problem**: Package compatibility errors
+**Solution**: Ensure you're using Python 3.10.9
+```bash
+python --version  # Should show 3.10.9
+conda create -n wan2gp python=3.10.9
+```
+
+### Dependency Installation Failures
+
+#### Triton Installation (Windows)
+**Problem**: `pip install triton-windows` fails
+**Solution**:
+1. Update pip: `pip install --upgrade pip`
+2. Try pre-compiled wheel
+3. Fallback to SDPA attention: `python wgp.py --attention sdpa`
+
+#### SageAttention Compilation Issues
+**Problem**: SageAttention installation fails
+**Solution**:
+1. Install Visual Studio Build Tools (Windows)
+2. Use pre-compiled wheels when available
+3. Fallback to basic attention modes
+
+## Memory Issues
+
+### CUDA Out of Memory
+
+#### During Model Loading
+**Problem**: "CUDA out of memory" when loading model
+**Solutions**:
+```bash
+# Use smaller model
+python wgp.py --t2v-1-3B
+
+# Enable quantization (usually default)
+python wgp.py --quantize-transformer True
+
+# Use memory-efficient profile
+python wgp.py --profile 4
+
+# Reduce preloaded model size
+python wgp.py --preload 0
+```
+
+#### During Video Generation
+**Problem**: Memory error during generation
+**Solutions**:
+1. Reduce frame count (shorter videos)
+2. Lower resolution in advanced settings
+3. Use lower batch size
+4. Clear GPU cache between generations
+
+### System RAM Issues
+
+#### High RAM Usage
+**Problem**: System runs out of RAM
+**Solutions**:
+```bash
+# Limit reserved memory
+python wgp.py --perc-reserved-mem-max 0.3
+
+# Use minimal RAM profile
+python wgp.py --profile 5
+
+# Enable swap file (OS level)
+```
+
+## Performance Issues
+
+### Slow Generation Speed
+
+#### General Optimization
+```bash
+# Enable compilation (requires Triton)
+python wgp.py --compile
+
+# Use faster attention
+python wgp.py --attention sage2
+
+# Enable TeaCache
+python wgp.py --teacache 2.0
+
+# Use high-performance profile
+python wgp.py --profile 3
+```
+
+#### GPU-Specific Optimizations
+
+**RTX 10XX/20XX Series**:
+```bash
+python wgp.py --attention sdpa --profile 4 --teacache 1.5
+```
+
+**RTX 30XX/40XX Series**:
+```bash
+python wgp.py --compile --attention sage --profile 3 --teacache 2.0
+```
+
+**RTX 50XX Series**:
+```bash
+python wgp.py --attention sage --profile 4 --fp16
+```
+
+### Attention Mechanism Issues
+
+#### Sage Attention Not Working
+**Problem**: Sage attention fails to compile or work
+**Diagnostic Steps**:
+1. Check Triton installation:
+   ```python
+   import triton
+   print(triton.__version__)
+   ```
+2. Clear Triton cache:
+   ```bash
+   # Windows
+   rmdir /s %USERPROFILE%\.triton
+   # Linux
+   rm -rf ~/.triton
+   ```
+3. Fallback solution:
+   ```bash
+   python wgp.py --attention sdpa
+   ```
+
+#### Flash Attention Issues
+**Problem**: Flash attention compilation fails
+**Solution**: 
+- Windows: Often requires manual CUDA kernel compilation
+- Linux: Usually works with `pip install flash-attn`
+- Fallback: Use Sage or SDPA attention
+
+## Model-Specific Issues
+
+### Lora Problems
+
+#### Loras Not Loading
+**Problem**: Loras don't appear in the interface
+**Solutions**:
+1. Check file format (should be .safetensors, .pt, or .pth)
+2. Verify correct directory:
+   ```
+   loras/          # For t2v models
+   loras_i2v/      # For i2v models
+   loras_hunyuan/  # For Hunyuan models
+   ```
+3. Click "Refresh" button in interface
+4. Use `--check-loras` to filter incompatible files
+
+#### Lora Compatibility Issues
+**Problem**: Lora causes errors or poor results
+**Solutions**:
+1. Check model size compatibility (1.3B vs 14B)
+2. Verify lora was trained for your model type
+3. Try different multiplier values
+4. Use `--check-loras` flag to auto-filter
+
+### VACE-Specific Issues
+
+#### Poor VACE Results
+**Problem**: VACE generates poor quality or unexpected results
+**Solutions**:
+1. Enable Skip Layer Guidance
+2. Use detailed prompts describing all elements
+3. Ensure proper mask creation with Matanyone
+4. Check reference image quality
+5. Use at least 15 steps, preferably 30+
+
+#### Matanyone Tool Issues
+**Problem**: Mask creation difficulties
+**Solutions**:
+1. Use negative point prompts to refine selection
+2. Create multiple sub-masks and combine them
+3. Try different background removal options
+4. Ensure sufficient contrast in source video
+
+## Network and Server Issues
+
+### Gradio Interface Problems
+
+#### Port Already in Use
+**Problem**: "Port 7860 is already in use"
+**Solution**:
+```bash
+# Use different port
+python wgp.py --server-port 7861
+
+# Or kill existing process
+# Windows
+netstat -ano | findstr :7860
+taskkill /PID <PID> /F
+
+# Linux
+lsof -i :7860
+kill <PID>
+```
+
+#### Interface Not Loading
+**Problem**: Browser shows "connection refused"
+**Solutions**:
+1. Check if server started successfully
+2. Try `http://127.0.0.1:7860` instead of `localhost:7860`
+3. Disable firewall temporarily
+4. Use `--listen` flag for network access
+
+### Remote Access Issues
+
+#### Sharing Not Working
+**Problem**: `--share` flag doesn't create public URL
+**Solutions**:
+1. Check internet connection
+2. Try different network
+3. Use `--listen` with port forwarding
+4. Check firewall settings
+
+## Quality Issues
+
+### Poor Video Quality
+
+#### General Quality Improvements
+1. Increase number of steps (25-30+)
+2. Use larger models (14B instead of 1.3B)
+3. Enable Skip Layer Guidance
+4. Improve prompt descriptions
+5. Use higher resolution settings
+
+#### Specific Quality Issues
+
+**Blurry Videos**:
+- Increase steps
+- Check source image quality (i2v)
+- Reduce TeaCache multiplier
+- Use higher guidance scale
+
+**Inconsistent Motion**:
+- Use longer overlap in sliding windows
+- Reduce window size
+- Improve prompt consistency
+- Check control video quality (VACE)
+
+**Color Issues**:
+- Check model compatibility
+- Adjust guidance scale
+- Verify input image color space
+- Try different VAE settings
+
+## Advanced Debugging
+
+### Enable Verbose Output
+```bash
+# Maximum verbosity
+python wgp.py --verbose 2
+
+# Check lora compatibility
+python wgp.py --check-loras --verbose 2
+```
+
+### Memory Debugging
+```bash
+# Monitor GPU memory
+nvidia-smi -l 1
+
+# Reduce memory usage
+python wgp.py --profile 4 --perc-reserved-mem-max 0.2
+```
+
+### Performance Profiling
+```bash
+# Test different configurations
+python wgp.py --attention sdpa --profile 4  # Baseline
+python wgp.py --attention sage --profile 3  # Performance
+python wgp.py --compile --teacache 2.0      # Maximum speed
+```
+
+## Getting Help
+
+### Before Asking for Help
+1. Check this troubleshooting guide
+2. Read the relevant documentation:
+   - [Installation Guide](INSTALLATION.md)
+   - [Getting Started](GETTING_STARTED.md)
+   - [Command Line Reference](CLI.md)
+3. Try basic fallback configuration:
+   ```bash
+   python wgp.py --attention sdpa --profile 4
+   ```
+
+### Community Support
+- **Discord Server**: https://discord.gg/g7efUW9jGV
+- Provide relevant information:
+  - GPU model and VRAM amount
+  - Python and PyTorch versions
+  - Complete error messages
+  - Command used to launch WanGP
+  - Operating system
+
+### Reporting Bugs
+When reporting issues:
+1. Include system specifications
+2. Provide complete error logs
+3. List the exact steps to reproduce
+4. Mention any modifications to default settings
+5. Include command line arguments used
+
+## Emergency Fallback
+
+If nothing works, try this minimal configuration:
+```bash
+# Absolute minimum setup
+python wgp.py --t2v-1-3B --attention sdpa --profile 4 --teacache 0 --fp16
+
+# If that fails, check basic PyTorch installation
+python -c "import torch; print(torch.cuda.is_available())"
+``` 

docs/VACE.md

@@ -0,0 +1,214 @@
+# VACE ControlNet Guide
+
+VACE is a powerful ControlNet that enables Video-to-Video and Reference-to-Video generation. It allows you to inject your own images into output videos, animate characters, perform inpainting/outpainting, and continue existing videos.
+
+## Overview
+
+VACE is probably one of the most powerful Wan models available. With it, you can:
+- Inject people or objects into scenes
+- Animate characters
+- Perform video inpainting and outpainting
+- Continue existing videos
+- Transfer motion from one video to another
+- Change the style of scenes while preserving the structure of the scenes
+
+
+## Getting Started
+
+### Model Selection
+1. Select either "Vace 1.3B" or "Vace 13B" from the dropdown menu
+2. Note: VACE works best with videos up to 7 seconds with the Riflex option enabled
+
+You can also use any derived Vace models such as Vace Fusionix or combine Vace with Loras accelerator such as Causvid.
+
+### Input Types
+
+#### 1. Control Video
+The Control Video is the source material that contains the instructions about what you want. So Vace expects in the Control Video some visual hints about the type of processing expected: for instance replacing an area by something else, converting an Open Pose wireframe into a human motion, colorizing an Area,  transferring the depth of an image area, ...
+
+For example, anywhere your control video contains the color 127 (grey), it will be considered as an area to be inpainting and replaced by the content of your text prompt and / or a reference image (see below). Likewise if the frames of a Control Video contains an Open Pose wireframe (basically some straight lines tied together that describes the pose of a person), Vace will automatically turn this Open Pose into a real human based on the text prompt and any reference Images (see below).
+
+You can either build yourself the Control Video with the annotators tools provided by the Vace team (see the Vace ressources at the bottom) or you can let WanGP (recommended option) generates on the fly a Vace formatted Control Video based on information you provide.
+
+WanGP wil need the following information to generate a Vace Control Video:
+- A *Control Video* : this video shouldn't have been altered by an annotator tool and can be taken straight from youtube or your camera
+- *Control Video Process* : This is the type of process you want to apply on the control video. For instance *Transfer Human Motion* will generate the Open Pose information from your video so that you can transfer this same motion to a generated character. If you want to do only *Spatial Outpainting* or *Temporal Inpainting / Outpainting* you may want to choose the *Keep Unchanged* process.
+- *Area Processed* : you can target the processing to a specific area. For instance even if there are multiple people in the Control Video you may want to replace only one them. If you decide to target an area you will need to provide a *Video Mask* as well. These types of videos can be easily created using the Matanyone tool embedded with WanGP (see the doc of Matanyone below). WanGP can apply different types of process, one the mask and another one on the outside the mask.
+
+Another nice thing is that you can combine all effects above with Outpainting since WanGP will create automatically an outpainting area in the Control Video if you ask for this. 
+
+By default WanGP will ask Vace to generate new frames in the "same spirit" of the control video if the latter is shorter than the number frames that you have requested.
+
+Be aware that the Control Video and Video Mask will be before anything happens resampled to the number of frames per second of Vace (usually 16) and resized to the output size you have requested.
+#### 2. Reference Images
+With Reference Images you can inject people or objects of your choice in the Video.
+You can also force Images to appear at a specific frame nos in the Video.
+
+If the Reference Image is a person or an object, it is recommended to turn on the background remover that will replace the background by the white color.
+This is not needed for a background image or an injected frame at a specific position.
+
+It is recommended to describe injected objects/people explicitly in your text prompt so that Vace can connect the Reference Images to the new generated video and this will increase the chance that you will find your injected people or objects.
+
+
+### Understanding Vace Control Video and Mask format
+As stated above WanGP will adapt the Control Video and the Video Mask to meet your instructions. You can preview the first frames of the new Control Video and of the Video Mask in the Generation Preview box (just click a thumbnail) to check that your request has been properly interpreted. You can as well ask WanGP to save in the main folder of WanGP the full generated Control Video and  Video Mask by launching the app with the *--save-masks* command.
+
+Look at the background colors of both the Control Video and the Video Mask:
+The Mask Video is the most important because depending on the color of its pixels, the Control Video will be interpreted differently. If an area in the Mask is black, the corresponding Control Video area will be kept as is. On the contrary if an area of the Mask is plain white, a Vace process will be applied on this area. If there isn't any Mask Video the Vace process will apply on the whole video frames. The nature of the process itself will depend on what there is in the Control Video for this area. 
+- if the area is grey (127) in the Control Video, this area will be replaced by new content based on the text prompt or image references
+- if an area represents a person in the wireframe Open Pose format, it will be replaced by a person animated with motion described by the Open Pose.The appearance of the person will depend on the text prompt or image references
+- if an area contains multiples shades of grey, these will be assumed to represent different levels of image depth and Vace will try to generate new content located at the same depth
+
+There are more Vace representations. For all the different mapping please refer the official Vace documentation.
+
+### Other Processing
+Most of the processing below and the ones related to Control Video can be combined together.
+- **Temporal Outpainting**\
+Temporal Outpainting requires an existing *Source Video* or *Control Video* and it amounts to adding missing frames. It is implicit if you use a Source Video that you want to continue (new frames will be added at the end of this Video) or if you provide a Control Video that contains fewer frames than the number that you have requested to generate.
+
+- **Temporal Inpainting**\
+With temporal inpainting you are asking Vace to generate missing frames that should exist between existing frames. There are two ways to do that:
+    - *Injected Reference Images* : Each Image is injected a position of your choice and Vace will fill the gaps between these frames
+    - *Frames to keep in Control Video* : If using a Control Video, you can ask WanGP to hide some of these frames to let Vace generate "alternate frames" for these parts of the Control Video.
+
+- **Spatial Outpainting**\
+This feature creates new content to the top, bottom, left or right of existing frames of a Control Video. You can set the amount of content for each direction by specifying a percentage of extra content in relation to the existing frame. Please note that the resulting video will target the resolution you specified. So if this Resolution corresponds to that of your Control Video you may lose details. Therefore it may be relevant to pick a higher resolution with Spatial Outpainting.\
+There are two ways to do Spatial Outpainting:
+    - *Injected Reference Frames* : new content will be added around Injected Frames
+    - *Control Video* : new content will be added on all the frames of the whole Control Video
+
+
+### Example 1 : Replace a Person in one video by another one by keeping the Background
+1) In Vace, select *Control Video Process*=**Transfer human pose**, *Area processed*=**Masked area** 
+2) In *Matanyone Video Mask Creator*, load your source video and create a mask where you targetted a specific person 
+3) Click *Export to Control Video Input and Video Mask Input* to transfer both the original video that now becomes the *Control Video* and the black & white mask that now defines the *Video Mask Area*
+4) Back in Vace, in *Reference Image* select **Inject Landscapes / People / Objects** and upload one or several pictures of the new person
+5) Generate
+
+This works also with several people at the same time (you just need to mask several people in *Matanyone*), you can also play with the slider *Expand / Shrink Mask* if the new person is larger than the original one and of course, you can also use the text *Prompt* if you dont want to use an image for the swap.
+
+
+### Example 2 : Change the Background behind some characters
+1) In Vace, select *Control Video Process*=**Inpainting**, *Area processed*=**Non Masked area** 
+2) In *Matanyone Video Mask Creator*, load your source video and create a mask where you targetted the people you want to keep 
+3) Click *Export to Control Video Input and Video Mask Input* to transfer both the original video that now becomes the *Control Video* and the black & white mask that now defines the *Video Mask Area*
+4) Generate
+
+If instead *Control Video Process*=**Depth**, then the background although it will be still different it will have a similar geometry than in the control video
+
+### Example 3 : Outpaint a Video to the Left and Inject a Character in this new area 
+1) In Vace, select *Control Video Process*=**Keep Unchanged** 
+2) *Control Video Outpainting in Percentage* enter the value 40 to the *Left* entry
+3) In *Reference Image* select **Inject Landscapes / People / Objects** and upload one or several pictures of a person
+4) Enter the *Prompt* such as "a person is coming from the left" (you will need of course a more accurate description)
+5) Generate
+
+
+
+### Creating Face / Object Replacement Masks
+Matanyone is a tool that will generate the Video Mask that needs to be combined with the Control Video. It is very useful as you just need to indicate in the first frame the area you want to mask and it will compute masked areas for the following frames by taking into account the motion.
+1. Load your video in Matanyone
+2. Click on the face or object in the first frame
+3. Validate the mask by clicking **Set Mask**
+4. Generate a copy of the control video (for easy transfers) and a new mask video by clicking "Generate Video Matting"
+5. Export to VACE with *Export to Control Video Input and Video Mask Input*
+
+### Advanced Matanyone Tips
+- **Negative Point Prompts**: Remove parts from current selection if the mask goes beyond the desired area
+- **Sub Masks**: Create multiple independent masks, then combine them. This may be useful if you are struggling to select exactly what you want.    
+
+
+
+## Window Sliding for Long Videos
+Generate videos up to 1 minute by merging multiple windows:
+The longer the video the greater the quality degradation. However the effect will be less visible if your generated video reuses mostly non altered control video.
+
+When this feature is enabled it is important to keep in mind that every positional argument of Vace (frames positions of *Injected Reference Frames*, *Frames to keep in Control Video*) are related to the first frame of the first Window. This is convenient as changing the size of a sliding window won't have any impact and this allows you define in advance the inject frames for all the windows.
+
+Likewise, if you use *Continue Video File* by providing a *Source Video*, this Source Video will be considered as the first window and the positional arguments will be calculated in relation to the first frame of this Source Video. Also the *overlap window size* parameter will correspond to the number of frames used of the Source Video that is temporally outpainted to produce new content.
+
+### How It Works
+- Each window uses the corresponding time segment of the Control Video
+- Example: 0-4s control video → first window, 4-8s → second window, etc.
+- Automatic overlap management ensures smooth transitions
+
+
+### Formula
+This formula gives the number of Generated Frames for a specific number of Sliding Windows :
+```
+Generated Frames = [Nb Windows - 1] × [Window Size - Overlap - Discard] + Window Size
+```
+
+### Multi-Line Prompts (Experimental)
+If you enable *Text Prompts separated by a Carriage Return will be used for a new Sliding Window*, you can define in advance a different prompt for each window.:
+- Each prompt is separated by a Carriage Return  
+- Each line of prompt will be used for a different window
+- If more windows than prompt lines, last line repeats
+
+## Recommended Settings
+
+### Quality Settings
+- **Skip Layer Guidance**: Turn ON with default configuration for better results (useless with FusioniX of Causvid are there is no cfg)
+- **Long Prompts**: Use detailed descriptions, especially for background elements not in reference images
+- **Steps**: Use at least 15 steps for good quality, 30+ for best results if you use the original Vace model. But only 8-10 steps are sufficient with Vace Funsionix or if you use Loras such as Causvid or Self-Forcing.
+
+### Sliding Window Settings
+For very long videos, configure sliding windows properly:
+
+- **Window Size**: Set appropriate duration for your content
+- **Overlap Frames**: Long enough for motion continuity, short enough to avoid blur propagation
+- **Discard Last Frames**: Remove at least 4 frames from each window (VACE 1.3B tends to blur final frames)
+- **Add Overlapped Noise**: May or may not reduce quality degradation over time
+
+### Background Removal
+WanGP includes automatic background removal options:
+- Use for reference images containing people/objects
+- **Don't use** this for landscape/setting reference images (the first reference image)
+- If you are not happy with the automatic background removal tool you can use the Image version of Matanyone for a precise background removal
+
+## External Resources
+
+### Official VACE Resources
+- **GitHub**: https://github.com/ali-vilab/VACE/tree/main/vace/gradios
+- **User Guide**: https://github.com/ali-vilab/VACE/blob/main/UserGuide.md
+- **Preprocessors**: Gradio tools for preparing materials
+
+### Recommended External Tools
+- **Annotation Tools**: For creating precise masks
+- **Video Editors**: For preparing control videos
+- **Background Removal**: For cleaning reference images
+
+## Troubleshooting
+
+### Poor Quality Results
+1. Use longer, more detailed prompts
+2. Enable Skip Layer Guidance
+3. Increase number of steps (30+)
+4. Check reference image quality
+5. Ensure proper mask creation
+
+### Inconsistent Windows
+1. Increase overlap frames
+2. Use consistent prompting across windows
+3. Add noise to overlapped frames
+4. Reduce discard frames if losing too much content
+
+### Memory Issues
+1. Use VACE 1.3B instead of 13B
+2. Reduce video length or resolution
+3. Decrease window size
+4. Enable quantization
+
+### Blurry Results
+1. Reduce overlap frames
+2. Increase discard last frames
+3. Use higher resolution reference images
+4. Check control video quality
+
+## Tips for Best Results
+1. **Detailed Prompts**: Describe everything in the scene, especially elements not in reference images
+2. **Quality Reference Images**: Use high-resolution, well-lit reference images
+3. **Proper Masking**: Take time to create precise masks with Matanyone
+4. **Iterative Approach**: Start with short videos, then extend successful results
+5. **Background Preparation**: Remove complex backgrounds from object/person reference images
+6. **Consistent Lighting**: Match lighting between reference images and intended scene 

fantasytalking/infer.py

@@ -0,0 +1,36 @@
+# Copyright Alibaba Inc. All Rights Reserved.
+
+from transformers import Wav2Vec2Model, Wav2Vec2Processor
+
+from .model import FantasyTalkingAudioConditionModel
+from .utils import get_audio_features
+import gc, torch
+
+def parse_audio(audio_path, num_frames, fps = 23, device = "cuda"):
+    fantasytalking = FantasyTalkingAudioConditionModel(None, 768, 2048).to(device)
+    from mmgp import offload
+    from accelerate import init_empty_weights
+    from fantasytalking.model import AudioProjModel
+
+    torch.set_grad_enabled(False) 
+
+    with init_empty_weights():
+        proj_model = AudioProjModel( 768, 2048)
+    offload.load_model_data(proj_model, "ckpts/fantasy_proj_model.safetensors")
+    proj_model.to("cpu").eval().requires_grad_(False)
+
+    wav2vec_model_dir = "ckpts/wav2vec"
+    wav2vec_processor = Wav2Vec2Processor.from_pretrained(wav2vec_model_dir)
+    wav2vec = Wav2Vec2Model.from_pretrained(wav2vec_model_dir, device_map="cpu").eval().requires_grad_(False)
+    wav2vec.to(device)
+    proj_model.to(device)
+    audio_wav2vec_fea = get_audio_features( wav2vec, wav2vec_processor, audio_path, fps, num_frames )
+
+    audio_proj_fea = proj_model(audio_wav2vec_fea)
+    pos_idx_ranges = fantasytalking.split_audio_sequence( audio_proj_fea.size(1), num_frames=num_frames )
+    audio_proj_split, audio_context_lens = fantasytalking.split_tensor_with_padding( audio_proj_fea, pos_idx_ranges, expand_length=4 )  # [b,21,9+8,768]    
+    wav2vec, proj_model= None, None
+    gc.collect()
+    torch.cuda.empty_cache()
+
+    return audio_proj_split, audio_context_lens

fantasytalking/model.py

@@ -0,0 +1,162 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from wan.modules.attention import pay_attention
+
+
+class AudioProjModel(nn.Module):
+    def __init__(self, audio_in_dim=1024, cross_attention_dim=1024):
+        super().__init__()
+        self.cross_attention_dim = cross_attention_dim
+        self.proj = torch.nn.Linear(audio_in_dim, cross_attention_dim, bias=False)
+        self.norm = torch.nn.LayerNorm(cross_attention_dim)
+
+    def forward(self, audio_embeds):
+        context_tokens = self.proj(audio_embeds)
+        context_tokens = self.norm(context_tokens)
+        return context_tokens  # [B,L,C]
+ 
+class WanCrossAttentionProcessor(nn.Module):
+    def __init__(self, context_dim, hidden_dim):
+        super().__init__()
+
+        self.context_dim = context_dim
+        self.hidden_dim = hidden_dim
+
+        self.k_proj = nn.Linear(context_dim, hidden_dim, bias=False)
+        self.v_proj = nn.Linear(context_dim, hidden_dim, bias=False)
+
+        nn.init.zeros_(self.k_proj.weight)
+        nn.init.zeros_(self.v_proj.weight)
+
+    def __call__(
+        self,
+        q: torch.Tensor,
+        audio_proj: torch.Tensor,
+        latents_num_frames: int = 21,
+        audio_context_lens = None
+    ) -> torch.Tensor:
+        """
+        audio_proj:   [B, 21, L3, C]
+        audio_context_lens: [B*21].
+        """
+        b, l, n, d = q.shape
+
+        if len(audio_proj.shape) == 4:
+            audio_q = q.view(b * latents_num_frames, -1, n, d)  # [b, 21, l1, n, d]
+            ip_key = self.k_proj(audio_proj).view(b * latents_num_frames, -1, n, d)
+            ip_value = self.v_proj(audio_proj).view(b * latents_num_frames, -1, n, d)
+            qkv_list = [audio_q, ip_key, ip_value]
+            del q, audio_q, ip_key, ip_value
+            audio_x = pay_attention(qkv_list, k_lens =audio_context_lens) #audio_context_lens
+            audio_x = audio_x.view(b, l, n, d)
+            audio_x = audio_x.flatten(2)
+        elif len(audio_proj.shape) == 3:
+            ip_key = self.k_proj(audio_proj).view(b, -1, n, d)
+            ip_value = self.v_proj(audio_proj).view(b, -1, n, d)
+            qkv_list = [q, ip_key, ip_value]
+            del q, ip_key, ip_value
+            audio_x = pay_attention(qkv_list, k_lens =audio_context_lens) #audio_context_lens
+            audio_x = audio_x.flatten(2)
+        return audio_x
+
+
+class FantasyTalkingAudioConditionModel(nn.Module):
+    def __init__(self, wan_dit, audio_in_dim: int, audio_proj_dim: int):
+        super().__init__()
+
+        self.audio_in_dim = audio_in_dim
+        self.audio_proj_dim = audio_proj_dim
+
+    def split_audio_sequence(self, audio_proj_length, num_frames=81):
+        """
+        Map the audio feature sequence to corresponding latent frame slices.
+
+        Args:
+            audio_proj_length (int): The total length of the audio feature sequence
+                                    (e.g., 173 in audio_proj[1, 173, 768]).
+            num_frames (int): The number of video frames in the training data (default: 81).
+
+        Returns:
+            list: A list of [start_idx, end_idx] pairs. Each pair represents the index range
+                (within the audio feature sequence) corresponding to a latent frame.
+        """
+        # Average number of tokens per original video frame
+        tokens_per_frame = audio_proj_length / num_frames
+
+        # Each latent frame covers 4 video frames, and we want the center
+        tokens_per_latent_frame = tokens_per_frame * 4
+        half_tokens = int(tokens_per_latent_frame / 2)
+
+        pos_indices = []
+        for i in range(int((num_frames - 1) / 4) + 1):
+            if i == 0:
+                pos_indices.append(0)
+            else:
+                start_token = tokens_per_frame * ((i - 1) * 4 + 1)
+                end_token = tokens_per_frame * (i * 4 + 1)
+                center_token = int((start_token + end_token) / 2) - 1
+                pos_indices.append(center_token)
+
+        # Build index ranges centered around each position
+        pos_idx_ranges = [[idx - half_tokens, idx + half_tokens] for idx in pos_indices]
+
+        # Adjust the first range to avoid negative start index
+        pos_idx_ranges[0] = [
+            -(half_tokens * 2 - pos_idx_ranges[1][0]),
+            pos_idx_ranges[1][0],
+        ]
+
+        return pos_idx_ranges
+
+    def split_tensor_with_padding(self, input_tensor, pos_idx_ranges, expand_length=0):
+        """
+        Split the input tensor into subsequences based on index ranges, and apply right-side zero-padding
+        if the range exceeds the input boundaries.
+
+        Args:
+            input_tensor (Tensor): Input audio tensor of shape [1, L, 768].
+            pos_idx_ranges (list): A list of index ranges, e.g. [[-7, 1], [1, 9], ..., [165, 173]].
+            expand_length (int): Number of tokens to expand on both sides of each subsequence.
+
+        Returns:
+            sub_sequences (Tensor): A tensor of shape [1, F, L, 768], where L is the length after padding.
+                                    Each element is a padded subsequence.
+            k_lens (Tensor): A tensor of shape [F], representing the actual (unpadded) length of each subsequence.
+                            Useful for ignoring padding tokens in attention masks.
+        """
+        pos_idx_ranges = [
+            [idx[0] - expand_length, idx[1] + expand_length] for idx in pos_idx_ranges
+        ]
+        sub_sequences = []
+        seq_len = input_tensor.size(1)  # 173
+        max_valid_idx = seq_len - 1  # 172
+        k_lens_list = []
+        for start, end in pos_idx_ranges:
+            # Calculate the fill amount
+            pad_front = max(-start, 0)
+            pad_back = max(end - max_valid_idx, 0)
+
+            # Calculate the start and end indices of the valid part
+            valid_start = max(start, 0)
+            valid_end = min(end, max_valid_idx)
+
+            # Extract the valid part
+            if valid_start <= valid_end:
+                valid_part = input_tensor[:, valid_start : valid_end + 1, :]
+            else:
+                valid_part = input_tensor.new_zeros((1, 0, input_tensor.size(2)))
+
+            # In the sequence dimension (the 1st dimension) perform padding
+            padded_subseq = F.pad(
+                valid_part,
+                (0, 0, 0, pad_back + pad_front, 0, 0),
+                mode="constant",
+                value=0,
+            )
+            k_lens_list.append(padded_subseq.size(-2) - pad_back - pad_front)
+
+            sub_sequences.append(padded_subseq)
+        return torch.stack(sub_sequences, dim=1), torch.tensor(
+            k_lens_list, dtype=torch.long
+        )

fantasytalking/utils.py

@@ -0,0 +1,52 @@
+# Copyright Alibaba Inc. All Rights Reserved.
+
+import imageio
+import librosa
+import numpy as np
+import torch
+from PIL import Image
+from tqdm import tqdm
+
+
+def resize_image_by_longest_edge(image_path, target_size):
+    image = Image.open(image_path).convert("RGB")
+    width, height = image.size
+    scale = target_size / max(width, height)
+    new_size = (int(width * scale), int(height * scale))
+    return image.resize(new_size, Image.LANCZOS)
+
+
+def save_video(frames, save_path, fps, quality=9, ffmpeg_params=None):
+    writer = imageio.get_writer(
+        save_path, fps=fps, quality=quality, ffmpeg_params=ffmpeg_params
+    )
+    for frame in tqdm(frames, desc="Saving video"):
+        frame = np.array(frame)
+        writer.append_data(frame)
+    writer.close()
+
+
+def get_audio_features(wav2vec, audio_processor, audio_path, fps, num_frames):
+    sr = 16000
+    audio_input, sample_rate = librosa.load(audio_path, sr=sr)  # 采样率为 16kHz
+
+    start_time = 0
+    # end_time = (0 + (num_frames - 1) * 1) / fps
+    end_time = num_frames / fps
+
+    start_sample = int(start_time * sr)
+    end_sample = int(end_time * sr)
+
+    try:
+        audio_segment = audio_input[start_sample:end_sample]
+    except:
+        audio_segment = audio_input
+
+    input_values = audio_processor(
+        audio_segment, sampling_rate=sample_rate, return_tensors="pt"
+    ).input_values.to("cuda")
+
+    with torch.no_grad():
+        fea = wav2vec(input_values).last_hidden_state
+
+    return fea

finetunes/hunyuan_t2v_accvideo.json

@@ -0,0 +1,30 @@
+{
+  "model": {
+    "name": "Hunyuan AccVideo 720p 13B",
+    "architecture": "hunyuan",
+    "description": " AccVideo is a novel efficient distillation method to accelerate video diffusion models with synthetic datset. Our method is 8.5x faster than HunyuanVideo.",
+    "URLs": [
+      "https://huggingface.co/DeepBeepMeep/HunyuanVideo/resolve/main/accvideo_hunyuan_video_720_quanto_int8.safetensors"
+    ],
+    "preload_URLs": [
+    ],
+    "auto_quantize": true
+  },
+  "negative_prompt": "",
+  "resolution": "832x480",
+  "video_length": 81,
+  "seed": 42,
+  "num_inference_steps": 5,
+  "flow_shift": 7,
+  "embedded_guidance_scale": 6,
+  "repeat_generation": 1,
+  "loras_multipliers": "",
+  "temporal_upsampling": "",
+  "spatial_upsampling": "",
+  "RIFLEx_setting": 0,
+  "slg_start_perc": 10,
+  "slg_end_perc": 90,
+  "prompt_enhancer": "",
+  "activated_loras": [
+  ]
+}

finetunes/hunyuan_t2v_fast.json

@@ -0,0 +1,31 @@
+{
+  "model": {
+    "name": "Hunyuan Fast Video 720p 13B",
+    "architecture": "hunyuan",
+    "description": "Fast Hunyuan is an accelerated HunyuanVideo model. It can sample high quality videos with 6 diffusion steps.",
+    "URLs": [
+      "https://huggingface.co/DeepBeepMeep/HunyuanVideo/resolve/main/fast_hunyuan_video_720_quanto_int8.safetensors"
+    ],
+    "preload_URLs": [
+      "https://huggingface.co/DeepBeepMeep/HunyuanVideo/resolve/main/fast_hunyuan_video_720_quanto_int8_map.json"
+    ],
+    "auto_quantize": true
+  },
+  "negative_prompt": "",
+  "resolution": "832x480",
+  "video_length": 81,
+  "seed": 42,
+  "num_inference_steps": 6,
+  "flow_shift": 17,
+  "embedded_guidance_scale": 6,
+  "repeat_generation": 1,
+  "loras_multipliers": "",
+  "temporal_upsampling": "",
+  "spatial_upsampling": "",
+  "RIFLEx_setting": 0,
+  "slg_start_perc": 10,
+  "slg_end_perc": 90,
+  "prompt_enhancer": "",
+  "activated_loras": [
+  ]
+}

finetunes/t2v_fusionix.json

@@ -0,0 +1,38 @@
+{
+	"model":
+	{
+		"name": "Wan2.1 text2video FusioniX 14B",
+		"architecture" : "t2v",
+		"description": "A powerful merged text-to-video model based on the original WAN 2.1 T2V model, enhanced using multiple open-source components and LoRAs to boost motion realism, temporal consistency, and expressive detail.",
+		"URLs": [
+			"https://huggingface.co/DeepBeepMeep/Wan2.1/resolve/main/Wan14BT2VFusioniX_fp16.safetensors",
+			"https://huggingface.co/DeepBeepMeep/Wan2.1/resolve/main/Wan14BT2VFusioniX_quanto_fp16_int8.safetensors",
+			"https://huggingface.co/DeepBeepMeep/Wan2.1/resolve/main/Wan14BT2VFusioniX_quanto_bf16_int8.safetensors"
+		],
+		"auto_quantize": true
+	},
+    "negative_prompt": "",
+    "prompt": "",
+    "resolution": "832x480",
+    "video_length": 81,
+    "seed": -1,
+    "num_inference_steps": 8,
+    "guidance_scale": 1,
+    "flow_shift": 5,
+    "embedded_guidance_scale": 6,
+    "repeat_generation": 1,
+    "multi_images_gen_type": 0,
+    "tea_cache_setting": 0,
+    "tea_cache_start_step_perc": 0,
+    "loras_multipliers": "",
+    "temporal_upsampling": "",
+    "spatial_upsampling": "",
+    "RIFLEx_setting": 0,
+    "slg_switch": 0,
+    "slg_start_perc": 10,
+    "slg_end_perc": 90,
+    "cfg_star_switch": 0,
+    "cfg_zero_step": -1,
+    "prompt_enhancer": "",
+    "activated_loras": []
+}

finetunes/t2v_sf.json

@@ -0,0 +1,38 @@
+{
+    "model": {
+        "name": "Wan2.1 text2video Self-Forcing 14B",
+        "architecture": "t2v",
+        "description": "This model is an advanced text-to-video generation model. This approach allows the model to generate videos with significantly fewer inference steps (4 or 8 steps) and without classifier-free guidance, substantially reducing video generation time while maintaining high quality outputs.",
+        "URLs": [
+            "https://huggingface.co/DeepBeepMeep/Wan2.1/resolve/main/wan2.1_StepDistill-CfgDistill_14B_bf16.safetensors",
+            "https://huggingface.co/DeepBeepMeep/Wan2.1/resolve/main/wan2.1_StepDistill-CfgDistill_14B_quanto_bf16_int8.safetensors",
+            "https://huggingface.co/DeepBeepMeep/Wan2.1/resolve/main/wan2.1_StepDistill-CfgDistill_14B_quanto_fp16_int8.safetensors"
+        ],
+		"author": "https://huggingface.co/lightx2v/Wan2.1-T2V-14B-StepDistill-CfgDistill",
+        "auto_quantize": true
+    },
+    "negative_prompt": "",
+    "prompt": "",
+    "resolution": "832x480",
+    "video_length": 81,
+    "seed": -1,
+    "num_inference_steps": 4,
+    "guidance_scale": 1,
+    "flow_shift": 3,
+    "embedded_guidance_scale": 6,
+    "repeat_generation": 1,
+    "multi_images_gen_type": 0,
+    "tea_cache_setting": 0,
+    "tea_cache_start_step_perc": 0,
+    "loras_multipliers": "",
+    "temporal_upsampling": "",
+    "spatial_upsampling": "",
+    "RIFLEx_setting": 0,
+    "slg_switch": 0,
+    "slg_start_perc": 10,
+    "slg_end_perc": 90,
+    "cfg_star_switch": 0,
+    "cfg_zero_step": -1,
+    "prompt_enhancer": "",
+    "activated_loras": []
+}

finetunes/vace_14B_fusionix.json

@@ -0,0 +1,40 @@
+{
+    "model": {
+        "name": "Vace FusioniX 14B",
+        "architecture": "vace_14B",
+        "modules": [
+            "vace_14B"
+        ],
+        "description": "Vace control model enhanced using multiple open-source components and LoRAs to boost motion realism, temporal consistency, and expressive detail.",
+        "URLs": [
+            "https://huggingface.co/DeepBeepMeep/Wan2.1/resolve/main/Wan14BT2VFusioniX_fp16.safetensors",
+            "https://huggingface.co/DeepBeepMeep/Wan2.1/resolve/main/Wan14BT2VFusioniX_quanto_bf16_int8.safetensors",
+            "https://huggingface.co/DeepBeepMeep/Wan2.1/resolve/main/Wan14BT2VFusioniX_quanto_fp16_int8.safetensors"
+        ],
+        "auto_quantize": true
+    },
+    "negative_prompt": "",
+    "prompt": "",
+    "resolution": "832x480",
+    "video_length": 81,
+    "seed": -1,
+    "num_inference_steps": 10,
+    "guidance_scale": 1,
+    "flow_shift": 5,
+    "embedded_guidance_scale": 6,
+    "repeat_generation": 1,
+    "multi_images_gen_type": 0,
+    "tea_cache_setting": 0,
+    "tea_cache_start_step_perc": 0,
+    "loras_multipliers": "",
+    "temporal_upsampling": "",
+    "spatial_upsampling": "",
+    "RIFLEx_setting": 0,
+    "slg_switch": 0,
+    "slg_start_perc": 10,
+    "slg_end_perc": 90,
+    "cfg_star_switch": 0,
+    "cfg_zero_step": -1,
+    "prompt_enhancer": "",
+    "activated_loras": []
+}

finetunes/vace_14B_sf.json

@@ -0,0 +1,41 @@
+{
+    "model": {
+        "name": "Vace Self-Forcing 14B",
+        "architecture": "vace_14B",
+        "modules": [
+            "vace_14B"
+        ],
+        "description": "This model is a combination of Vace and an advanced text-to-video generation model. This approach allows the model to generate videos with significantly fewer inference steps (4 or 8 steps) and without classifier-free guidance, substantially reducing video generation time while maintaining high quality outputs.",
+        "URLs": [
+            "https://huggingface.co/DeepBeepMeep/Wan2.1/resolve/main/wan2.1_StepDistill-CfgDistill_14B_bf16.safetensors",
+            "https://huggingface.co/DeepBeepMeep/Wan2.1/resolve/main/wan2.1_StepDistill-CfgDistill_14B_quanto_bf16_int8.safetensors",
+            "https://huggingface.co/DeepBeepMeep/Wan2.1/resolve/main/wan2.1_StepDistill-CfgDistill_14B_quanto_fp16_int8.safetensors"
+        ],
+		"author": "https://huggingface.co/lightx2v/Wan2.1-T2V-14B-StepDistill-CfgDistill",
+        "auto_quantize": true
+    },
+    "negative_prompt": "",
+    "prompt": "",
+    "resolution": "832x480",
+    "video_length": 81,
+    "seed": -1,
+    "num_inference_steps": 4,
+    "guidance_scale": 1,
+    "flow_shift": 3,
+    "embedded_guidance_scale": 6,
+    "repeat_generation": 1,
+    "multi_images_gen_type": 0,
+    "tea_cache_setting": 0,
+    "tea_cache_start_step_perc": 0,
+    "loras_multipliers": "",
+    "temporal_upsampling": "",
+    "spatial_upsampling": "",
+    "RIFLEx_setting": 0,
+    "slg_switch": 0,
+    "slg_start_perc": 10,
+    "slg_end_perc": 90,
+    "cfg_star_switch": 0,
+    "cfg_zero_step": -1,
+    "prompt_enhancer": "",
+    "activated_loras": []
+}

hyvideo/config.py

@@ -0,0 +1,534 @@
+import argparse
+from .constants import *
+import re
+from .modules.models import HUNYUAN_VIDEO_CONFIG
+
+
+def parse_args(namespace=None):
+    parser = argparse.ArgumentParser(description="HunyuanVideo inference script")
+
+    parser = add_network_args(parser)
+    parser = add_extra_models_args(parser)
+    parser = add_denoise_schedule_args(parser)
+    parser = add_inference_args(parser)
+    parser = add_parallel_args(parser)
+
+    args = parser.parse_args(namespace=namespace)
+    args = sanity_check_args(args)
+
+    return args
+
+
+def add_network_args(parser: argparse.ArgumentParser):
+    group = parser.add_argument_group(title="HunyuanVideo network args")
+
+
+    group.add_argument(
+        "--quantize-transformer",
+        action="store_true",
+        help="On the fly 'transformer' quantization"
+    )
+
+
+    group.add_argument(
+        "--lora-dir-i2v",
+        type=str,
+        default="loras_i2v",
+        help="Path to a directory that contains Loras for i2v"
+    )
+
+
+    group.add_argument(
+        "--lora-dir",
+        type=str,
+        default="",
+        help="Path to a directory that contains Loras"
+    )
+
+
+    group.add_argument(
+        "--lora-preset",
+        type=str,
+        default="",
+        help="Lora preset to preload"
+    )
+
+    # group.add_argument(
+    #     "--lora-preset-i2v",
+    #     type=str,
+    #     default="",
+    #     help="Lora preset to preload for i2v"
+    # )
+
+    group.add_argument(
+        "--profile",
+        type=str,
+        default=-1,
+        help="Profile No"
+    )
+
+    group.add_argument(
+        "--verbose",
+        type=str,
+        default=1,
+        help="Verbose level"
+    )
+
+    group.add_argument(
+        "--server-port",
+        type=str,
+        default=0,
+        help="Server port"
+    )
+
+    group.add_argument(
+        "--server-name",
+        type=str,
+        default="",
+        help="Server name"
+    )
+
+    group.add_argument(
+        "--open-browser",
+        action="store_true",
+        help="open browser"
+    )
+
+    group.add_argument(
+        "--t2v",
+        action="store_true",
+        help="text to video mode"
+    )
+
+    group.add_argument(
+        "--i2v",
+        action="store_true",
+        help="image to video mode"
+    )
+
+    group.add_argument(
+        "--compile",
+        action="store_true",
+        help="Enable pytorch compilation"
+    )
+
+    group.add_argument(
+        "--fast",
+        action="store_true",
+        help="use Fast HunyuanVideo model"
+    )
+
+    group.add_argument(
+        "--fastest",
+        action="store_true",
+        help="activate the best config"
+    )
+
+    group.add_argument(
+    "--attention",
+    type=str,
+    default="",
+    help="attention mode"
+    )
+
+    group.add_argument(
+    "--vae-config",
+    type=str,
+    default="",
+    help="vae config mode"
+    )    
+    
+    parser.add_argument(
+        "--share",
+        action="store_true",
+        help="Create a shared URL to access webserver remotely"
+    )
+
+    parser.add_argument(
+    "--lock-config",
+    action="store_true",
+    help="Prevent modifying the configuration from the web interface"
+    )
+
+    parser.add_argument(
+        "--preload",
+        type=str,
+        default="0",
+        help="Megabytes of the diffusion model to preload in VRAM"
+    )
+
+    parser.add_argument(
+        "--multiple-images",
+        action="store_true",
+        help="Allow inputting multiple images with image to video"
+    )
+
+
+    # Main model
+    group.add_argument(
+        "--model",
+        type=str,
+        choices=list(HUNYUAN_VIDEO_CONFIG.keys()),
+        default="HYVideo-T/2-cfgdistill",
+    )
+    group.add_argument(
+        "--latent-channels",
+        type=str,
+        default=16,
+        help="Number of latent channels of DiT. If None, it will be determined by `vae`. If provided, "
+        "it still needs to match the latent channels of the VAE model.",
+    )
+    group.add_argument(
+        "--precision",
+        type=str,
+        default="bf16",
+        choices=PRECISIONS,
+        help="Precision mode. Options: fp32, fp16, bf16. Applied to the backbone model and optimizer.",
+    )
+
+    # RoPE
+    group.add_argument(
+        "--rope-theta", type=int, default=256, help="Theta used in RoPE."
+    )
+    return parser
+
+
+def add_extra_models_args(parser: argparse.ArgumentParser):
+    group = parser.add_argument_group(
+        title="Extra models args, including vae, text encoders and tokenizers)"
+    )
+
+    # - VAE
+    group.add_argument(
+        "--vae",
+        type=str,
+        default="884-16c-hy",
+        choices=list(VAE_PATH),
+        help="Name of the VAE model.",
+    )
+    group.add_argument(
+        "--vae-precision",
+        type=str,
+        default="fp16",
+        choices=PRECISIONS,
+        help="Precision mode for the VAE model.",
+    )
+    group.add_argument(
+        "--vae-tiling",
+        action="store_true",
+        help="Enable tiling for the VAE model to save GPU memory.",
+    )
+    group.set_defaults(vae_tiling=True)
+
+    group.add_argument(
+        "--text-encoder",
+        type=str,
+        default="llm",
+        choices=list(TEXT_ENCODER_PATH),
+        help="Name of the text encoder model.",
+    )
+    group.add_argument(
+        "--text-encoder-precision",
+        type=str,
+        default="fp16",
+        choices=PRECISIONS,
+        help="Precision mode for the text encoder model.",
+    )
+    group.add_argument(
+        "--text-states-dim",
+        type=int,
+        default=4096,
+        help="Dimension of the text encoder hidden states.",
+    )
+    group.add_argument(
+        "--text-len", type=int, default=256, help="Maximum length of the text input."
+    )
+    group.add_argument(
+        "--tokenizer",
+        type=str,
+        default="llm",
+        choices=list(TOKENIZER_PATH),
+        help="Name of the tokenizer model.",
+    )
+    group.add_argument(
+        "--prompt-template",
+        type=str,
+        default="dit-llm-encode",
+        choices=PROMPT_TEMPLATE,
+        help="Image prompt template for the decoder-only text encoder model.",
+    )
+    group.add_argument(
+        "--prompt-template-video",
+        type=str,
+        default="dit-llm-encode-video",
+        choices=PROMPT_TEMPLATE,
+        help="Video prompt template for the decoder-only text encoder model.",
+    )
+    group.add_argument(
+        "--hidden-state-skip-layer",
+        type=int,
+        default=2,
+        help="Skip layer for hidden states.",
+    )
+    group.add_argument(
+        "--apply-final-norm",
+        action="store_true",
+        help="Apply final normalization to the used text encoder hidden states.",
+    )
+
+    # - CLIP
+    group.add_argument(
+        "--text-encoder-2",
+        type=str,
+        default="clipL",
+        choices=list(TEXT_ENCODER_PATH),
+        help="Name of the second text encoder model.",
+    )
+    group.add_argument(
+        "--text-encoder-precision-2",
+        type=str,
+        default="fp16",
+        choices=PRECISIONS,
+        help="Precision mode for the second text encoder model.",
+    )
+    group.add_argument(
+        "--text-states-dim-2",
+        type=int,
+        default=768,
+        help="Dimension of the second text encoder hidden states.",
+    )
+    group.add_argument(
+        "--tokenizer-2",
+        type=str,
+        default="clipL",
+        choices=list(TOKENIZER_PATH),
+        help="Name of the second tokenizer model.",
+    )
+    group.add_argument(
+        "--text-len-2",
+        type=int,
+        default=77,
+        help="Maximum length of the second text input.",
+    )
+
+    return parser
+
+
+def add_denoise_schedule_args(parser: argparse.ArgumentParser):
+    group = parser.add_argument_group(title="Denoise schedule args")
+
+    group.add_argument(
+        "--denoise-type",
+        type=str,
+        default="flow",
+        help="Denoise type for noised inputs.",
+    )
+
+    # Flow Matching
+    group.add_argument(
+        "--flow-shift",
+        type=float,
+        default=7.0,
+        help="Shift factor for flow matching schedulers.",
+    )
+    group.add_argument(
+        "--flow-reverse",
+        action="store_true",
+        help="If reverse, learning/sampling from t=1 -> t=0.",
+    )
+    group.add_argument(
+        "--flow-solver",
+        type=str,
+        default="euler",
+        help="Solver for flow matching.",
+    )
+    group.add_argument(
+        "--use-linear-quadratic-schedule",
+        action="store_true",
+        help="Use linear quadratic schedule for flow matching."
+        "Following MovieGen (https://ai.meta.com/static-resource/movie-gen-research-paper)",
+    )
+    group.add_argument(
+        "--linear-schedule-end",
+        type=int,
+        default=25,
+        help="End step for linear quadratic schedule for flow matching.",
+    )
+
+    return parser
+
+
+def add_inference_args(parser: argparse.ArgumentParser):
+    group = parser.add_argument_group(title="Inference args")
+
+    # ======================== Model loads ========================
+    group.add_argument(
+        "--model-base",
+        type=str,
+        default="ckpts",
+        help="Root path of all the models, including t2v models and extra models.",
+    )
+    group.add_argument(
+        "--dit-weight",
+        type=str,
+        default="ckpts/hunyuan-video-t2v-720p/transformers/mp_rank_00_model_states.pt",
+        help="Path to the HunyuanVideo model. If None, search the model in the args.model_root."
+        "1. If it is a file, load the model directly."
+        "2. If it is a directory, search the model in the directory. Support two types of models: "
+        "1) named `pytorch_model_*.pt`"
+        "2) named `*_model_states.pt`, where * can be `mp_rank_00`.",
+    )
+    group.add_argument(
+        "--model-resolution",
+        type=str,
+        default="540p",
+        choices=["540p", "720p"],
+        help="Root path of all the models, including t2v models and extra models.",
+    )
+    group.add_argument(
+        "--load-key",
+        type=str,
+        default="module",
+        help="Key to load the model states. 'module' for the main model, 'ema' for the EMA model.",
+    )
+    group.add_argument(
+        "--use-cpu-offload",
+        action="store_true",
+        help="Use CPU offload for the model load.",
+    )
+
+    # ======================== Inference general setting ========================
+    group.add_argument(
+        "--batch-size",
+        type=int,
+        default=1,
+        help="Batch size for inference and evaluation.",
+    )
+    group.add_argument(
+        "--infer-steps",
+        type=int,
+        default=50,
+        help="Number of denoising steps for inference.",
+    )
+    group.add_argument(
+        "--disable-autocast",
+        action="store_true",
+        help="Disable autocast for denoising loop and vae decoding in pipeline sampling.",
+    )
+    group.add_argument(
+        "--save-path",
+        type=str,
+        default="./results",
+        help="Path to save the generated samples.",
+    )
+    group.add_argument(
+        "--save-path-suffix",
+        type=str,
+        default="",
+        help="Suffix for the directory of saved samples.",
+    )
+    group.add_argument(
+        "--name-suffix",
+        type=str,
+        default="",
+        help="Suffix for the names of saved samples.",
+    )
+    group.add_argument(
+        "--num-videos",
+        type=int,
+        default=1,
+        help="Number of videos to generate for each prompt.",
+    )
+    # ---sample size---
+    group.add_argument(
+        "--video-size",
+        type=int,
+        nargs="+",
+        default=(720, 1280),
+        help="Video size for training. If a single value is provided, it will be used for both height "
+        "and width. If two values are provided, they will be used for height and width "
+        "respectively.",
+    )
+    group.add_argument(
+        "--video-length",
+        type=int,
+        default=129,
+        help="How many frames to sample from a video. if using 3d vae, the number should be 4n+1",
+    )
+    # --- prompt ---
+    group.add_argument(
+        "--prompt",
+        type=str,
+        default=None,
+        help="Prompt for sampling during evaluation.",
+    )
+    group.add_argument(
+        "--seed-type",
+        type=str,
+        default="auto",
+        choices=["file", "random", "fixed", "auto"],
+        help="Seed type for evaluation. If file, use the seed from the CSV file. If random, generate a "
+        "random seed. If fixed, use the fixed seed given by `--seed`. If auto, `csv` will use the "
+        "seed column if available, otherwise use the fixed `seed` value. `prompt` will use the "
+        "fixed `seed` value.",
+    )
+    group.add_argument("--seed", type=int, default=None, help="Seed for evaluation.")
+
+    # Classifier-Free Guidance
+    group.add_argument(
+        "--neg-prompt", type=str, default=None, help="Negative prompt for sampling."
+    )
+    group.add_argument(
+        "--cfg-scale", type=float, default=1.0, help="Classifier free guidance scale."
+    )
+    group.add_argument(
+        "--embedded-cfg-scale",
+        type=float,
+        default=6.0,
+        help="Embeded classifier free guidance scale.",
+    )
+
+    group.add_argument(
+        "--reproduce",
+        action="store_true",
+        help="Enable reproducibility by setting random seeds and deterministic algorithms.",
+    )
+
+    return parser
+
+
+def add_parallel_args(parser: argparse.ArgumentParser):
+    group = parser.add_argument_group(title="Parallel args")
+
+    # ======================== Model loads ========================
+    group.add_argument(
+        "--ulysses-degree",
+        type=int,
+        default=1,
+        help="Ulysses degree.",
+    )
+    group.add_argument(
+        "--ring-degree",
+        type=int,
+        default=1,
+        help="Ulysses degree.",
+    )
+
+    return parser
+
+
+def sanity_check_args(args):
+    # VAE channels
+    vae_pattern = r"\d{2,3}-\d{1,2}c-\w+"
+    if not re.match(vae_pattern, args.vae):
+        raise ValueError(
+            f"Invalid VAE model: {args.vae}. Must be in the format of '{vae_pattern}'."
+        )
+    vae_channels = int(args.vae.split("-")[1][:-1])
+    if args.latent_channels is None:
+        args.latent_channels = vae_channels
+    if vae_channels != args.latent_channels:
+        raise ValueError(
+            f"Latent channels ({args.latent_channels}) must match the VAE channels ({vae_channels})."
+        )
+    return args

hyvideo/constants.py

@@ -0,0 +1,164 @@
+import os
+import torch
+
+__all__ = [
+    "C_SCALE",
+    "PROMPT_TEMPLATE",
+    "MODEL_BASE",
+    "PRECISIONS",
+    "NORMALIZATION_TYPE",
+    "ACTIVATION_TYPE",
+    "VAE_PATH",
+    "TEXT_ENCODER_PATH",
+    "TOKENIZER_PATH",
+    "TEXT_PROJECTION",
+    "DATA_TYPE",
+    "NEGATIVE_PROMPT",
+    "NEGATIVE_PROMPT_I2V",
+    "FLOW_PATH_TYPE",
+    "FLOW_PREDICT_TYPE",
+    "FLOW_LOSS_WEIGHT",
+    "FLOW_SNR_TYPE",
+    "FLOW_SOLVER",
+]
+
+PRECISION_TO_TYPE = {
+    'fp32': torch.float32,
+    'fp16': torch.float16,
+    'bf16': torch.bfloat16,
+}
+
+# =================== Constant Values =====================
+# Computation scale factor, 1P = 1_000_000_000_000_000. Tensorboard will display the value in PetaFLOPS to avoid
+# overflow error when tensorboard logging values.
+C_SCALE = 1_000_000_000_000_000
+
+# When using decoder-only models, we must provide a prompt template to instruct the text encoder
+# on how to generate the text.
+# --------------------------------------------------------------------
+PROMPT_TEMPLATE_ENCODE = (
+    "<|start_header_id|>system<|end_header_id|>\n\nDescribe the image by detailing the color, shape, size, texture, "
+    "quantity, text, spatial relationships of the objects and background:<|eot_id|>"
+    "<|start_header_id|>user<|end_header_id|>\n\n{}<|eot_id|>"
+) 
+PROMPT_TEMPLATE_ENCODE_VIDEO = (
+    "<|start_header_id|>system<|end_header_id|>\n\nDescribe the video by detailing the following aspects: "
+    "1. The main content and theme of the video."
+    "2. The color, shape, size, texture, quantity, text, and spatial relationships of the objects."
+    "3. Actions, events, behaviors temporal relationships, physical movement changes of the objects."
+    "4. background environment, light, style and atmosphere."
+    "5. camera angles, movements, and transitions used in the video:<|eot_id|>"
+    "<|start_header_id|>user<|end_header_id|>\n\n{}<|eot_id|>"
+)  
+
+PROMPT_TEMPLATE_ENCODE_I2V = (
+    "<|start_header_id|>system<|end_header_id|>\n\n<image>\nDescribe the image by detailing the color, shape, size, texture, "
+    "quantity, text, spatial relationships of the objects and background:<|eot_id|>"
+    "<|start_header_id|>user<|end_header_id|>\n\n{}<|eot_id|>"
+    "<|start_header_id|>assistant<|end_header_id|>\n\n"
+)
+
+PROMPT_TEMPLATE_ENCODE_VIDEO_I2V = (
+    "<|start_header_id|>system<|end_header_id|>\n\n<image>\nDescribe the video by detailing the following aspects according to the reference image: "
+    "1. The main content and theme of the video."
+    "2. The color, shape, size, texture, quantity, text, and spatial relationships of the objects."
+    "3. Actions, events, behaviors temporal relationships, physical movement changes of the objects."
+    "4. background environment, light, style and atmosphere."
+    "5. camera angles, movements, and transitions used in the video:<|eot_id|>\n\n"
+    "<|start_header_id|>user<|end_header_id|>\n\n{}<|eot_id|>"
+    "<|start_header_id|>assistant<|end_header_id|>\n\n"
+)
+
+NEGATIVE_PROMPT = "Aerial view, aerial view, overexposed, low quality, deformation, a poor composition, bad hands, bad teeth, bad eyes, bad limbs, distortion"
+NEGATIVE_PROMPT_I2V = "deformation, a poor composition and deformed video, bad teeth, bad eyes, bad limbs"
+
+PROMPT_TEMPLATE = {
+    "dit-llm-encode": {
+        "template": PROMPT_TEMPLATE_ENCODE,
+        "crop_start": 36,
+    },
+    "dit-llm-encode-video": {
+        "template": PROMPT_TEMPLATE_ENCODE_VIDEO,
+        "crop_start": 95,
+    },
+    "dit-llm-encode-i2v": {
+        "template": PROMPT_TEMPLATE_ENCODE_I2V,
+        "crop_start": 36,
+        "image_emb_start": 5,
+        "image_emb_end": 581,
+        "image_emb_len": 576,
+        "double_return_token_id": 271
+    },
+    "dit-llm-encode-video-i2v": {
+        "template": PROMPT_TEMPLATE_ENCODE_VIDEO_I2V,
+        "crop_start": 103,
+        "image_emb_start": 5,
+        "image_emb_end": 581,
+        "image_emb_len": 576,
+        "double_return_token_id": 271
+    },
+}
+
+# ======================= Model ======================
+PRECISIONS = {"fp32", "fp16", "bf16"}
+NORMALIZATION_TYPE = {"layer", "rms"}
+ACTIVATION_TYPE = {"relu", "silu", "gelu", "gelu_tanh"}
+
+# =================== Model Path =====================
+MODEL_BASE = os.getenv("MODEL_BASE", "./ckpts")
+
+# =================== Data =======================
+DATA_TYPE = {"image", "video", "image_video"}
+
+# 3D VAE
+VAE_PATH = {"884-16c-hy": f"{MODEL_BASE}/hunyuan-video-t2v-720p/vae"}
+
+# Text Encoder
+TEXT_ENCODER_PATH = {
+    "clipL": f"{MODEL_BASE}/clip_vit_large_patch14",
+    "llm": f"{MODEL_BASE}/llava-llama-3-8b",
+    "llm-i2v": f"{MODEL_BASE}/llava-llama-3-8b",
+}
+
+# Tokenizer
+TOKENIZER_PATH = {
+    "clipL": f"{MODEL_BASE}/clip_vit_large_patch14",
+    "llm": f"{MODEL_BASE}/llava-llama-3-8b",
+    "llm-i2v": f"{MODEL_BASE}/llava-llama-3-8b",
+}
+
+TEXT_PROJECTION = {
+    "linear",  # Default, an nn.Linear() layer
+    "single_refiner",  # Single TokenRefiner. Refer to LI-DiT
+}
+
+# Flow Matching path type
+FLOW_PATH_TYPE = {
+    "linear",               # Linear trajectory between noise and data
+    "gvp",                  # Generalized variance-preserving SDE
+    "vp",                   # Variance-preserving SDE
+}
+
+# Flow Matching predict type
+FLOW_PREDICT_TYPE = {
+    "velocity",             # Predict velocity
+    "score",                # Predict score
+    "noise",                # Predict noise
+}
+
+# Flow Matching loss weight
+FLOW_LOSS_WEIGHT = {
+    "velocity",             # Weight loss by velocity
+    "likelihood",           # Weight loss by likelihood
+}
+
+# Flow Matching SNR type
+FLOW_SNR_TYPE = {
+    "lognorm",              # Log-normal SNR
+    "uniform",              # Uniform SNR
+}
+
+# Flow Matching solvers
+FLOW_SOLVER = {
+    "euler",                # Euler solver
+}

hyvideo/data_kits/audio_dataset.py

@@ -0,0 +1,170 @@
+import os
+import cv2
+import math
+import json
+import torch
+import random
+import librosa
+import traceback
+import torchvision
+import numpy as np
+import pandas as pd
+from PIL import Image
+from einops import rearrange
+from torch.utils.data import Dataset
+from decord import VideoReader, cpu
+from transformers import CLIPImageProcessor
+import torchvision.transforms as transforms
+from torchvision.transforms import ToPILImage
+
+
+
+def get_audio_feature(feature_extractor, audio_path):
+    audio_input, sampling_rate = librosa.load(audio_path, sr=16000)
+    assert sampling_rate == 16000
+
+    audio_features = []
+    window = 750*640
+    for i in range(0, len(audio_input), window):
+        audio_feature = feature_extractor(audio_input[i:i+window], 
+                                        sampling_rate=sampling_rate, 
+                                        return_tensors="pt", 
+                                        ).input_features
+        audio_features.append(audio_feature)
+
+    audio_features = torch.cat(audio_features, dim=-1)
+    return audio_features, len(audio_input) // 640
+
+
+class VideoAudioTextLoaderVal(Dataset):
+    def __init__(
+        self, 
+        image_size: int,
+        meta_file: str, 
+        **kwargs,
+    ):
+        super().__init__()
+        self.meta_file = meta_file
+        self.image_size = image_size
+        self.text_encoder = kwargs.get("text_encoder", None)            # llava_text_encoder
+        self.text_encoder_2 = kwargs.get("text_encoder_2", None)        # clipL_text_encoder
+        self.feature_extractor = kwargs.get("feature_extractor", None)
+        self.meta_files = []
+    
+        csv_data = pd.read_csv(meta_file)
+        for idx in range(len(csv_data)):
+            self.meta_files.append(
+                {
+                    "videoid": str(csv_data["videoid"][idx]),
+                    "image_path": str(csv_data["image"][idx]), 
+                    "audio_path": str(csv_data["audio"][idx]), 
+                    "prompt": str(csv_data["prompt"][idx]), 
+                    "fps": float(csv_data["fps"][idx])
+                }
+            )
+        
+        self.llava_transform = transforms.Compose(
+            [
+                transforms.Resize((336, 336), interpolation=transforms.InterpolationMode.BILINEAR), 
+                transforms.ToTensor(), 
+                transforms.Normalize((0.48145466, 0.4578275, 0.4082107), (0.26862954, 0.26130258, 0.27577711)),
+            ]
+        )
+        self.clip_image_processor = CLIPImageProcessor()
+        
+        self.device = torch.device("cuda")
+        self.weight_dtype = torch.float16
+
+        
+    def __len__(self):
+        return len(self.meta_files)
+
+    @staticmethod
+    def get_text_tokens(text_encoder, description, dtype_encode="video"):
+        text_inputs = text_encoder.text2tokens(description, data_type=dtype_encode)
+        text_ids = text_inputs["input_ids"].squeeze(0)
+        text_mask = text_inputs["attention_mask"].squeeze(0)
+        return text_ids, text_mask
+    
+    def get_batch_data(self, idx):
+        meta_file = self.meta_files[idx]
+        videoid = meta_file["videoid"]
+        image_path = meta_file["image_path"]
+        audio_path = meta_file["audio_path"]
+        prompt = "Authentic, Realistic, Natural, High-quality, Lens-Fixed, " + meta_file["prompt"]
+        fps = meta_file["fps"]
+        
+        img_size = self.image_size
+        ref_image = Image.open(image_path).convert('RGB')
+        
+        # Resize reference image
+        w, h = ref_image.size
+        scale = img_size / min(w, h)
+        new_w = round(w * scale / 64) * 64
+        new_h = round(h * scale / 64) * 64
+
+        if img_size == 704:
+            img_size_long = 1216
+        if new_w * new_h > img_size * img_size_long:
+            import math
+            scale = math.sqrt(img_size * img_size_long / w / h)
+            new_w = round(w * scale / 64) * 64
+            new_h = round(h * scale / 64) * 64
+
+        ref_image = ref_image.resize((new_w, new_h), Image.LANCZOS)
+        
+        ref_image = np.array(ref_image)
+        ref_image = torch.from_numpy(ref_image)
+         
+        audio_input, audio_len = get_audio_feature(self.feature_extractor, audio_path)
+        audio_prompts = audio_input[0]
+        
+        motion_bucket_id_heads = np.array([25] * 4)
+        motion_bucket_id_exps = np.array([30] * 4)
+        motion_bucket_id_heads = torch.from_numpy(motion_bucket_id_heads)
+        motion_bucket_id_exps = torch.from_numpy(motion_bucket_id_exps)
+        fps = torch.from_numpy(np.array(fps))
+        
+        to_pil = ToPILImage()
+        pixel_value_ref = rearrange(ref_image.clone().unsqueeze(0), "b h w c -> b c h w")   # (b c h w)
+        
+        pixel_value_ref_llava = [self.llava_transform(to_pil(image)) for image in pixel_value_ref]
+        pixel_value_ref_llava = torch.stack(pixel_value_ref_llava, dim=0)
+        pixel_value_ref_clip = self.clip_image_processor(
+            images=Image.fromarray((pixel_value_ref[0].permute(1,2,0)).data.cpu().numpy().astype(np.uint8)), 
+            return_tensors="pt"
+        ).pixel_values[0]
+        pixel_value_ref_clip = pixel_value_ref_clip.unsqueeze(0)
+        
+        # Encode text prompts
+   
+        text_ids, text_mask = self.get_text_tokens(self.text_encoder, prompt)
+        text_ids_2, text_mask_2 = self.get_text_tokens(self.text_encoder_2, prompt)
+        
+        # Output batch
+        batch = {
+            "text_prompt": prompt,                         # 
+            "videoid": videoid, 
+            "pixel_value_ref": pixel_value_ref.to(dtype=torch.float16),                 # 参考图，用于vae提特征 (1, 3, h, w), 取值范围(0, 255)
+            "pixel_value_ref_llava": pixel_value_ref_llava.to(dtype=torch.float16),     # 参考图，用于llava提特征 (1, 3, 336, 336), 取值范围 = CLIP取值范围
+            "pixel_value_ref_clip": pixel_value_ref_clip.to(dtype=torch.float16),       # 参考图，用于clip_image_encoder提特征 (1, 3, 244, 244), 取值范围 = CLIP取值范围
+            "audio_prompts": audio_prompts.to(dtype=torch.float16), 
+            "motion_bucket_id_heads": motion_bucket_id_heads.to(dtype=text_ids.dtype), 
+            "motion_bucket_id_exps": motion_bucket_id_exps.to(dtype=text_ids.dtype), 
+            "fps": fps.to(dtype=torch.float16), 
+            "text_ids": text_ids.clone(),                                               # 对应llava_text_encoder
+            "text_mask": text_mask.clone(),                                             # 对应llava_text_encoder
+            "text_ids_2": text_ids_2.clone(),                                           # 对应clip_text_encoder
+            "text_mask_2": text_mask_2.clone(),                                         # 对应clip_text_encoder
+            "audio_len": audio_len,
+            "image_path": image_path, 
+            "audio_path": audio_path, 
+        }
+        return batch
+    
+    def __getitem__(self, idx):
+        return self.get_batch_data(idx)
+    
+
+        
+        

hyvideo/data_kits/audio_preprocessor.py

@@ -0,0 +1,76 @@
+
+import os
+import cv2
+import json
+import time
+import decord
+import einops
+import librosa
+import torch
+import random
+import argparse
+import traceback
+import numpy as np
+from tqdm import tqdm
+from PIL import Image
+from einops import rearrange
+
+
+
+def get_facemask(ref_image, align_instance, area=1.25):
+    # ref_image: (b f c h w)
+    bsz, f, c, h, w = ref_image.shape
+    images = rearrange(ref_image, "b f c h w -> (b f) h w c").data.cpu().numpy().astype(np.uint8)
+    face_masks = []
+    for image in images:
+        image_pil = Image.fromarray(image).convert("RGB")
+        _, _, bboxes_list = align_instance(np.array(image_pil)[:,:,[2,1,0]], maxface=True)
+        try:
+            bboxSrc = bboxes_list[0]
+        except:
+            bboxSrc = [0, 0, w, h]
+        x1, y1, ww, hh = bboxSrc
+        x2, y2 = x1 + ww, y1 + hh
+        ww, hh = (x2-x1) * area, (y2-y1) * area
+        center = [(x2+x1)//2, (y2+y1)//2]
+        x1 = max(center[0] - ww//2, 0)
+        y1 = max(center[1] - hh//2, 0)
+        x2 = min(center[0] + ww//2, w)
+        y2 = min(center[1] + hh//2, h)
+        
+        face_mask = np.zeros_like(np.array(image_pil))
+        face_mask[int(y1):int(y2), int(x1):int(x2)] = 1.0
+        face_masks.append(torch.from_numpy(face_mask[...,:1]))
+    face_masks = torch.stack(face_masks, dim=0)     # (b*f, h, w, c)
+    face_masks = rearrange(face_masks, "(b f) h w c -> b c f h w", b=bsz, f=f)
+    face_masks = face_masks.to(device=ref_image.device, dtype=ref_image.dtype)
+    return face_masks
+
+
+def encode_audio(wav2vec, audio_feats, fps, num_frames=129):
+    if fps == 25:
+        start_ts = [0]
+        step_ts = [1]
+    elif fps == 12.5:
+        start_ts = [0]
+        step_ts = [2]
+    else:
+        start_ts = [0]
+        step_ts = [1]
+
+    num_frames = min(num_frames, 400)
+    audio_feats = wav2vec.encoder(audio_feats.unsqueeze(0)[:, :, :3000], output_hidden_states=True).hidden_states
+    audio_feats = torch.stack(audio_feats, dim=2)
+    audio_feats = torch.cat([torch.zeros_like(audio_feats[:,:4]), audio_feats], 1)
+    
+    audio_prompts = []
+    for bb in range(1):
+        audio_feats_list = []
+        for f in range(num_frames):
+            cur_t = (start_ts[bb] + f * step_ts[bb]) * 2
+            audio_clip = audio_feats[bb:bb+1, cur_t: cur_t+10]
+            audio_feats_list.append(audio_clip)
+        audio_feats_list = torch.stack(audio_feats_list, 1)
+        audio_prompts.append(audio_feats_list)
+    audio_prompts = torch.cat(audio_prompts)
+    return audio_prompts

hyvideo/data_kits/data_tools.py

@@ -0,0 +1,41 @@
+import os
+import cv2
+import torch
+import numpy as np
+import imageio
+import torchvision
+from einops import rearrange
+
+
+def save_videos_grid(videos: torch.Tensor, path: str, rescale=False, n_rows=6, fps=8, quality=8):
+    videos = rearrange(videos, "b c t h w -> t b c h w")
+    outputs = []
+    for x in videos:
+        x = torchvision.utils.make_grid(x, nrow=n_rows)
+        x = x.transpose(0, 1).transpose(1, 2).squeeze(-1)
+        if rescale:
+            x = (x + 1.0) / 2.0  # -1,1 -> 0,1
+        x = torch.clamp(x,0,1)
+        x = (x * 255).numpy().astype(np.uint8)
+        outputs.append(x)
+
+    os.makedirs(os.path.dirname(path), exist_ok=True)
+    imageio.mimsave(path, outputs, fps=fps, quality=quality)
+
+def pad_image(crop_img, size, color=(255, 255, 255), resize_ratio=1):
+    crop_h, crop_w = crop_img.shape[:2]
+    target_w, target_h = size
+    scale_h, scale_w = target_h / crop_h, target_w / crop_w
+    if scale_w > scale_h:
+        resize_h = int(target_h*resize_ratio)
+        resize_w = int(crop_w / crop_h * resize_h)
+    else:
+        resize_w = int(target_w*resize_ratio)
+        resize_h = int(crop_h / crop_w * resize_w)
+    crop_img = cv2.resize(crop_img, (resize_w, resize_h))
+    pad_left = (target_w - resize_w) // 2
+    pad_top = (target_h - resize_h) // 2
+    pad_right = target_w - resize_w - pad_left
+    pad_bottom = target_h - resize_h - pad_top
+    crop_img = cv2.copyMakeBorder(crop_img, pad_top, pad_bottom, pad_left, pad_right, cv2.BORDER_CONSTANT, value=color)
+    return crop_img

hyvideo/data_kits/face_align/__init__.py

@@ -0,0 +1 @@
+from .align import AlignImage

hyvideo/data_kits/face_align/align.py

@@ -0,0 +1,34 @@
+import os
+import sys
+import torch
+from .detface import DetFace
+
+class AlignImage(object):
+    def __init__(self, device='cuda', det_path=''):
+        self.facedet = DetFace(pt_path=det_path, confThreshold=0.5, nmsThreshold=0.45, device=device)
+
+    @torch.no_grad()
+    def __call__(self, im, maxface=False):
+        bboxes, kpss, scores = self.facedet.detect(im)
+        face_num = bboxes.shape[0]
+
+        five_pts_list = []
+        scores_list = []
+        bboxes_list = []
+        for i in range(face_num):
+            five_pts_list.append(kpss[i].reshape(5,2))
+            scores_list.append(scores[i])
+            bboxes_list.append(bboxes[i])
+
+        if maxface and face_num>1:
+            max_idx = 0
+            max_area = (bboxes[0, 2])*(bboxes[0, 3])
+            for i in range(1, face_num):
+                area = (bboxes[i,2])*(bboxes[i,3])
+                if area>max_area:
+                    max_idx = i
+            five_pts_list = [five_pts_list[max_idx]]
+            scores_list = [scores_list[max_idx]]
+            bboxes_list = [bboxes_list[max_idx]]
+
+        return five_pts_list, scores_list, bboxes_list