Create README.md

README.md

@@ -0,0 +1,32 @@
+---
+base_model:
+- HuggingFaceM4/Idefics3-8B-Llama3
+---
+# FP8-Dynamic quant for Idefics2-8B-Llama3, requires about ~12 GB 
+
+## vLLM usage
+```py
+vllm serve leon-se/Idefics3-8B-Llama3-FP8-Dynamic --trust-remote-code
+```
+
+## Code used for quantization
+```py
+from transformers import AutoProcessor, AutoModelForVision2Seq
+from llmcompressor.modifiers.quantization import QuantizationModifier
+from llmcompressor.transformers import oneshot
+
+model_name = "HuggingfaceM4/Idefics3-8B-Llama3"
+
+processor = AutoProcessor.from_pretrained(model_name)
+model = AutoModelForVision2Seq.from_pretrained(model_name, device_map="cuda", torch_dtype="auto", trust_remote_code=True)
+
+recipe = QuantizationModifier(
+    targets="Linear",
+    scheme="FP8_DYNAMIC",
+    ignore=["re:.*lm_head", "re:model.vision_model.*", "re:model.connector.*"],
+)
+
+SAVE_DIR = model_name.split("/")[1] + "-FP8-Dynamic"
+oneshot(model=model, recipe=recipe, output_dir=SAVE_DIR)
+processor.save_pretrained(SAVE_DIR)
+```