Installation error
The load_in_4bit and load_in_8bit arguments are deprecated and will be removed in the future versions. Please, pass a BitsAndBytesConfig object in quantization_config argument instead.
Unused kwargs: ['_load_in_4bit', '_load_in_8bit', 'quant_method']. These kwargs are not used in <class 'transformers.utils.quantization_config.BitsAndBytesConfig'>.
ImportError Traceback (most recent call last)
in <cell line: 4>()
2
3 tokenizer = AutoTokenizer.from_pretrained("akshathmangudi/llama3.1-8b-quantized")
----> 4 model = AutoModelForCausalLM.from_pretrained(
5 "akshathmangudi/llama3.1-8b-quantized",
6 load_in_4bit=True, # Proper argument for 4-bit quantization
2 frames
/usr/local/lib/python3.10/dist-packages/transformers/models/auto/auto_factory.py in from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs)
562 elif type(config) in cls._model_mapping.keys():
563 model_class = _get_model_class(config, cls._model_mapping)
--> 564 return model_class.from_pretrained(
565 pretrained_model_name_or_path, *model_args, config=config, **hub_kwargs, **kwargs
566 )
/usr/local/lib/python3.10/dist-packages/transformers/modeling_utils.py in from_pretrained(cls, pretrained_model_name_or_path, config, cache_dir, ignore_mismatched_sizes, force_download, local_files_only, token, revision, use_safetensors, weights_only, *model_args, **kwargs)
3655
3656 if hf_quantizer is not None:
-> 3657 hf_quantizer.validate_environment(
3658 torch_dtype=torch_dtype, from_tf=from_tf, from_flax=from_flax, device_map=device_map
3659 )
/usr/local/lib/python3.10/dist-packages/transformers/quantizers/quantizer_bnb_4bit.py in validate_environment(self, *args, **kwargs)
72 )
73 if not is_bitsandbytes_available():
---> 74 raise ImportError(
75 "Using bitsandbytes 4-bit quantization requires the latest version of bitsandbytes: pip install -U bitsandbytes"
76 )
ImportError: Using bitsandbytes 4-bit quantization requires the latest version of bitsandbytes: pip install -U bitsandbytes
NOTE: If your import is failing due to a missing package, you can
manually install dependencies using either !pip or !apt.