Update app.py

app.py

@@ -47,7 +47,13 @@ h1 {
 
 # Load the tokenizer and model
 tokenizer = AutoTokenizer.from_pretrained("CyberNative-AI/Colibri_8b_v0.1")
-model = AutoModelForCausalLM.from_pretrained("CyberNative-AI/Colibri_8b_v0.1", load_in_4bit=False, load_in_8bit=True, device_map="auto")
+#model = AutoModelForCausalLM.from_pretrained("CyberNative-AI/Colibri_8b_v0.1", load_in_4bit=True, load_in_8bit=False, device_map="auto")
+
+from transformers import BitsAndBytesConfig
+
+nf4_config = BitsAndBytesConfig(load_in_4bit=True, bnb_4bit_quant_type="nf4")
+model = AutoModelForCausalLM.from_pretrained(model_id, quantization_config=nf4_config)
+
 terminators = [
     tokenizer.eos_token_id,
     tokenizer.convert_tokens_to_ids("<|eot_id|>")