Update README.md

README.md

@@ -74,4 +74,53 @@ To solve the addition \(1.5 + 102.2\), follow these steps:
    \]
 
 So, the final answer is \(\boxed{103.7}\).<｜end▁of▁sentence｜>
+```
+
+## HQQ
+Run ~3.5x faster with <a href="https://github.com/mobiusml/hqq/">HQQ</a>. First, install the dependencies:
+```
+pip install hqq
+```
+
+```Python
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from hqq.models.hf.base import AutoHQQHFModel
+from hqq.core.quantize import *
+
+#Params
+device        = 'cuda:0'
+backend       = "torchao_int4" 
+compute_dtype = torch.bfloat16 if backend=="torchao_int4" else torch.float16
+model_id      = "mobiuslabsgmbh/DeepSeek-R1-ReDistill-Qwen-7B-v1.1"
+
+#Load
+tokenizer = AutoTokenizer.from_pretrained(model_id)
+model     = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=compute_dtype, attn_implementation="sdpa")
+
+#Quantize
+quant_config = BaseQuantizeConfig(nbits=4, group_size=64, axis=1)
+AutoHQQHFModel.quantize_model(model, quant_config=quant_config, compute_dtype=compute_dtype, device=device)
+
+#Optimize
+from hqq.utils.patching import prepare_for_inference
+prepare_for_inference(model, backend=backend, verbose=False)
+
+############################################################
+#Generate (streaming)
+from hqq.utils.generation_hf import HFGenerator
+gen = HFGenerator(model, tokenizer, max_new_tokens=8192, do_sample=True, compile='partial').warmup()
+
+prompt = "If A equals B, and C equals B - A, what would be the value of C?" 
+out    = gen.generate(prompt, print_tokens=True)
+
+############################################################
+# #Generate (simple)
+# from hqq.utils.generation_hf import patch_model_for_compiled_runtime
+# patch_model_for_compiled_runtime(model, tokenizer, warmup=True)
+
+# prompt = "If A equals B, and C equals B - A, what would be the value of C?" 
+# chat    = tokenizer.apply_chat_template([{"role":"user", "content":prompt}], tokenize=True, add_generation_prompt=True, return_tensors="pt")
+# outputs = model.generate(chat.to(device), max_new_tokens=8192, do_sample=True) 
+# print(tokenizer.decode(outputs[0]))
 ```