| from transformers import AutoModelForCausalLM, AutoTokenizer, GPTQConfig | |
| #model_name = "ibm-granite/granite-3.2-8b-instruct" | |
| # | |
| #for bits in [4, 8]: | |
| # tokenizer = AutoTokenizer.from_pretrained("ibm-granite/granite-3.2-8b-instruct") | |
| # | |
| # | |
| # gptq_config = GPTQConfig(bits=bits, tokenizer=tokenizer) | |
| # | |
| # quantized_model = AutoModelForCausalLM.from_pretrained("ibm-granite/granite-3.2-8b-instruct", device_map="auto", quantization_config=gptq_config) | |
| # | |
| # quantized_model.save_pretrained(f"ai-i9p/{model_name.split('/')[-1]}-GPTQ-Int{bits}") | |
| # tokenizer.save_pretrained(f"ai-i9p/{model_name.split('/')[-1]}-GPTQ-Int{bits}") | |
| # | |
| # quantized_model.to("cpu") | |
| # quantized_model.save_pretrained(f"ai-i9p/{model_name.split('/')[-1]}-GPTQ-Int{bits}") | |
| from awq import AutoAWQForCausalLM | |
| from transformers import AutoTokenizer | |
| model_path = 'Qwen/Qwen2.5-14B-Instruct' | |
| quant_path = 'Qwen2.5-14B-Instruct-awq' | |
| quant_config = { "zero_point": True, "q_group_size": 128, "w_bit": 4, "version": "GEMM" } | |
| # Load model | |
| model = AutoAWQForCausalLM.from_pretrained(model_path) | |
| tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True) | |
| # Quantize | |
| model.quantize(tokenizer, quant_config=quant_config) | |
| # Save quantized model | |
| model.save_quantized(quant_path) | |
| tokenizer.save_pretrained(quant_path) | |
| print(f'Model is quantized and saved at "{quant_path}"') |