Update app.py

app.py

@@ -1,6 +1,7 @@
 import os
 import requests
 import json
+import time
 
 import gradio as gr
 from transformers import AutoTokenizer
@@ -44,6 +45,8 @@ HEADERS = {
 PRESENCE_PENALTY=0
 FREQUENCY_PENALTY=1.1
 
+MAX_SEC = 30
+
 tokenizer = AutoTokenizer.from_pretrained("MediaTek-Research/Breeze-7B-Instruct-v0_1")
 
 def refusal_condition(query):
@@ -153,7 +156,8 @@ with gr.Blocks() as demo:
                 "frequency_penalty": FREQUENCY_PENALTY,
                 "stream": True,
             }
-     
+
+            start_time = time.time()
             with requests.post(API_URL, headers=HEADERS, data=json.dumps(data), stream=True) as r:
                 for response in r.iter_lines():
                     if len(response) > 0:
@@ -169,6 +173,8 @@ with gr.Blocks() as demo:
                                 history[-1][1] += delta
 
                             yield history
+                    if time.time() - start_time > MAX_SEC:
+                        break
                     
             if history[-1][1].endswith('</s>'):
                 history[-1][1] = history[-1][1][:-4]
@@ -262,5 +268,5 @@ with gr.Blocks() as demo:
 
     gr.Markdown(LICENSE)
 
-demo.queue(concurrency_count=1, max_size=32)
+demo.queue(concurrency_count=1, max_size=128)
 demo.launch()