Update app.py

app.py

@@ -16,18 +16,14 @@ print("Your Computer IP Address is:" + IPAddr)
 
 
 DESCRIPTION = """
-# Breexe-8x7B-Instruct-v0_1
-
+# Breexe-8x7B-Instruct-v0.1
 Breexe-8x7B is a language model family that builds on top of [Mixtral-8x7B](https://huggingface.co/mistralai/Mixtral-8x7B-v0.1), 
 specifically intended for Traditional Chinese use. [Breexe-8x7B-Instruct-v0_1](https://huggingface.co/MediaTek-Research/Breexe-8x7B-Instruct-v0_1) demonstrates impressive performance in benchmarks for Traditional Chinese and English, on par with OpenAI's gpt3.5. 
-
 *A project by the members (in alphabetical order): Chan-Jan Hsu 許湛然, Chang-Le Liu 劉昶樂, Feng-Ting Liao 廖峰挺, Po-Chun Hsu 許博竣, Yi-Chang Chen 陳宜昌, and the supervisor Da-Shan Shiu 許大山.*
-
 **免責聲明: Breexe-8x7B-Instruct 並未針對問答進行安全保護，因此語言模型的任何回應不代表 MediaTek Research 立場。**
 """
 
 LICENSE = """
-
 """
 
 DEFAULT_SYSTEM_PROMPT = "You are a helpful AI assistant built by MediaTek Research. The user you are helping speaks Traditional Chinese and comes from Taiwan."
@@ -45,7 +41,7 @@ HEADERS = {
 MAX_SEC = 30
 MAX_INPUT_LENGTH = 5000
 
-tokenizer = AutoTokenizer.from_pretrained("MediaTek-Research/Breexe-8x7B-Instruct-v0_1")
+tokenizer = AutoTokenizer.from_pretrained("MediaTek-Research/Breexe-8x7B-Instruct-v0_1", use_auth_token=os.environ.get("HF_TOKEN"))
 
 def insert_to_db(prompt, response, temperature, top_p):
     try:
@@ -96,40 +92,33 @@ with gr.Blocks() as demo:
     gr.Markdown(DESCRIPTION)
 
     system_prompt = gr.Textbox(label='System prompt',
-       value=DEFAULT_SYSTEM_PROMPT,
-       lines=1)
-
-    with gr.Accordion(label='Advanced options', open=False):
-            
-            max_new_tokens = gr.Slider(
-                label='Max new tokens',
-                minimum=32,
-                maximum=2048,
-                step=1,
-                value=1024,
-            )
-            temperature = gr.Slider(
-                label='Temperature',
-                minimum=0.01,
-                maximum=1.0,
-                step=0.01,
-                value=0.01,
-            )
-            top_p = gr.Slider(
-                label='Top-p (nucleus sampling)',
-                minimum=0.01,
-                maximum=0.99,
-                step=0.01,
-                value=0.01,
-            )
-            repetition_penalty = gr.Slider(
-                label='Repetition Penalty',
-                minimum=0.1,
-                maximum=2,
-                step=0.01,
-                value=1.1,
-            )
+                               value=DEFAULT_SYSTEM_PROMPT,
+                               lines=1)
     
+    with gr.Accordion(label='Advanced options', open=False):
+
+        max_new_tokens = gr.Slider(
+            label='Max new tokens',
+            minimum=32,
+            maximum=2048,
+            step=1,
+            value=1024,
+        )
+        temperature = gr.Slider(
+            label='Temperature',
+            minimum=0.01,
+            maximum=0.5,
+            step=0.01,
+            value=0.01,
+        )
+        top_p = gr.Slider(
+            label='Top-p (nucleus sampling)',
+            minimum=0.01,
+            maximum=0.99,
+            step=0.01,
+            value=0.01,
+        )
+
     chatbot = gr.Chatbot()
     with gr.Row():
         msg = gr.Textbox(
@@ -151,6 +140,9 @@ with gr.Blocks() as demo:
 
     saved_input = gr.State()
 
+    
+
+
     def user(user_message, history):
         return "", history + [[user_message, None]]
 
@@ -188,7 +180,7 @@ with gr.Blocks() as demo:
             # start_time = time.time()
 
     
-    def bot(history, max_new_tokens, temperature, top_p, system_prompt, repetition_penalty):
+    def bot(history, max_new_tokens, temperature, top_p, system_prompt):
         chat_data = []
         system_prompt = system_prompt.strip()
         if system_prompt:
@@ -210,19 +202,13 @@ with gr.Blocks() as demo:
             yield history
         else:
             data = {
-                "model_type": "breeze-7b-instruct-v10",
+                "model_type": "breexe-8x7b-instruct-v01",
                 "prompt": str(message),
                 "parameters": {
                     "temperature": float(temperature),
                     "top_p": float(top_p),
                     "max_new_tokens": int(max_new_tokens),
-                    "repetition_penalty": float(repetition_penalty),
-
-                    "num_beams":1,    # w/o beam search
-                    "typical_p":0.99, 
-                    "top_k":61952,        # w/o top_k
-                    "do_sample": True,
-                    "min_length":1,
+                    "repetition_penalty": 1.1
                 }
             }
             
@@ -241,14 +227,13 @@ with gr.Blocks() as demo:
                 response = history[-1][1]
     
                 if refusal_condition(history[-1][1]):
-                    history[-1][1] = history[-1][1] + '\n\n**[免責聲明: Breeze-7B-Instruct 和 Breeze-7B-Instruct-64k 並未針對問答進行安全保護，因此語言模型的任何回應不代表 MediaTek Research 立場。]**'
+                    history[-1][1] = history[-1][1] + '\n\n**[免責聲明: Breexe-8x7B-Instruct 和 Breexe-8x7B-Instruct-64k 並未針對問答進行安全保護，因此語言模型的任何回應不代表 MediaTek Research 立場。]**'
                     yield history
             else:
                 del history[-1]
                 yield history
 
         print('== Record ==\nQuery: {query}\nResponse: {response}'.format(query=repr(message), response=repr(history[-1][1])))
-        insert_to_db(message, response, float(temperature), float(top_p))
         
     msg.submit(user, [msg, chatbot], [msg, chatbot], queue=False).then(
         fn=bot,
@@ -258,7 +243,6 @@ with gr.Blocks() as demo:
             temperature,
             top_p,
             system_prompt,
-            repetition_penalty,
         ],
         outputs=chatbot
     )
@@ -272,7 +256,6 @@ with gr.Blocks() as demo:
             temperature,
             top_p,
             system_prompt,
-            repetition_penalty,
         ],
         outputs=chatbot
     )
@@ -312,7 +295,6 @@ with gr.Blocks() as demo:
             temperature,
             top_p,
             system_prompt,
-            repetition_penalty,
         ],
         outputs=chatbot,
     )
@@ -335,5 +317,5 @@ with gr.Blocks() as demo:
 
     gr.Markdown(LICENSE)
 
-demo.queue(concurrency_count=2, max_size=128)
-demo.launch()
+demo.queue(concurrency_count=4, max_size=128)
+demo.launch()