Spaces:

rphrp1985
/

zerogpu

Running on Zero

rphrp1985 commited on 22 days ago

Commit

add0b80

verified ·

1 Parent(s): 6d1ecc6

Update app.py

Files changed (1) hide show

app.py CHANGED Viewed

@@ -127,7 +127,7 @@ CONTEXT_LENGTH = 16000
 device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 quantization_config = BitsAndBytesConfig(
-    load_in_8bit=True,
     bnb_4bit_compute_dtype=torch.bfloat16
 )
 tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)

 device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 quantization_config = BitsAndBytesConfig(
+    load_in_4bit=True,
     bnb_4bit_compute_dtype=torch.bfloat16
 )
 tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)