Update app.py

app.py

@@ -22,7 +22,7 @@ ROLE_TOKENS = {
 }
 
 CONTEXT_SIZE = 2000
-ENABLE_GPU = True
+ENABLE_GPU = False
 GPU_LAYERS = 70
 
 # Create a lock object
@@ -39,8 +39,8 @@ app.logger.setLevel(logging.DEBUG)  # Set the desired logging level
 #repo_name = "IlyaGusev/saiga2_13b_gguf"
 #model_name = "model-q4_K.gguf"
 
-repo_name = "IlyaGusev/saiga2_70b_gguf"
-model_name = "ggml-model-q4_1.gguf"
+repo_name = "IlyaGusev/saiga2_13b_gguf"
+model_name = "model-q8_0.gguf"
 
 #repo_name = "IlyaGusev/saiga2_7b_gguf"
 #model_name = "model-q4_K.gguf"
@@ -98,8 +98,8 @@ def init_model(context_size, enable_gpu=False, gpu_layer_number=35):
             logits_all=True,
             #n_threads=12,
             verbose=True,
-            n_gpu_layers=gpu_layer_number,
-            n_gqa=8       #must be set for 70b models
+            n_gpu_layers=gpu_layer_number#,
+            #n_gqa=8       #must be set for 70b models
         )
         return model
     else:
@@ -110,8 +110,8 @@ def init_model(context_size, enable_gpu=False, gpu_layer_number=35):
             #n_batch=100,
             logits_all=True,
             #n_threads=12,
-            verbose=True,
-            n_gqa=8       #must be set for 70b models
+            verbose=True#,
+            #n_gqa=8       #must be set for 70b models
         )
         return model
 
@@ -236,7 +236,7 @@ def generate_and_log_tokens(user_request, model, generator):
     global response_tokens
     for token in generate_tokens(model, generator):
         if token == b'': # or (max_new_tokens is not None and i >= max_new_tokens):
-            log(user_request, response_tokens.decode("utf-8", errors="ignore"))
+            #log(user_request, response_tokens.decode("utf-8", errors="ignore"))
             response_tokens = bytearray()
             break
         response_tokens.extend(token)
@@ -271,6 +271,8 @@ def generate_response():
     for message in messages:
         if message.get("from") == "assistant":
             message_tokens = get_message_tokens(model=model, role="bot", content=message.get("content", ""))
+        elif message.get("from") == "system":
+            message_tokens = get_message_tokens(model=model, role="system", content=message.get("content", ""))
         else:
             message_tokens = get_message_tokens(model=model, role="user", content=message.get("content", ""))