File size: 1,371 Bytes
d75b820 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 |
from transformers import AutoModelForCausalLM, AutoTokenizer, GPTQConfig
#model_name = "ibm-granite/granite-3.2-8b-instruct"
#
#for bits in [4, 8]:
# tokenizer = AutoTokenizer.from_pretrained("ibm-granite/granite-3.2-8b-instruct")
#
#
# gptq_config = GPTQConfig(bits=bits, tokenizer=tokenizer)
#
# quantized_model = AutoModelForCausalLM.from_pretrained("ibm-granite/granite-3.2-8b-instruct", device_map="auto", quantization_config=gptq_config)
#
# quantized_model.save_pretrained(f"ai-i9p/{model_name.split('/')[-1]}-GPTQ-Int{bits}")
# tokenizer.save_pretrained(f"ai-i9p/{model_name.split('/')[-1]}-GPTQ-Int{bits}")
#
# quantized_model.to("cpu")
# quantized_model.save_pretrained(f"ai-i9p/{model_name.split('/')[-1]}-GPTQ-Int{bits}")
from awq import AutoAWQForCausalLM
from transformers import AutoTokenizer
model_path = 'Qwen/Qwen2.5-14B-Instruct'
quant_path = 'Qwen2.5-14B-Instruct-awq'
quant_config = { "zero_point": True, "q_group_size": 128, "w_bit": 4, "version": "GEMM" }
# Load model
model = AutoAWQForCausalLM.from_pretrained(model_path)
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
# Quantize
model.quantize(tokenizer, quant_config=quant_config)
# Save quantized model
model.save_quantized(quant_path)
tokenizer.save_pretrained(quant_path)
print(f'Model is quantized and saved at "{quant_path}"') |