import openai import gradio as gr import os os.system("sh run.sh") client = openai.OpenAI( api_key="EMPTY", base_url="http://localhost:7777/v1" ) def chatbot_mistral(messages:list,systemt_prompt=""): if messages[0]['role'] != "system": sys = {"role":"system","content":systemt_prompt} messages.insert(0,sys) else: messages[0]['content'] = systemt_prompt # model_name = "dewu-chat" call_args = { 'temperature': 0.7, 'top_p': 0.9, 'top_k': 40, 'max_tokens': 2048, # output-len 'presence_penalty': 1.0, 'frequency_penalty': 0.0, "repetition_penalty":1.0, "stop":[""], # "stop":["<|eot_id|>","<|end_of_text|>"], "stream":True } # create a chat completion for chunk in openai.ChatCompletion.create(model="dewu-chat",messages=messages,extra_body=call_args): if hasattr(chunk.choices[0].delta, "content"): response = chunk.choices[0].delta.content yield response def respond( message, history: list[tuple[str, str]], system_message, max_tokens, temperature, top_p, ): messages = [{"role": "system", "content": system_message}] for val in history: if val[0]: messages.append({"role": "user", "content": val[0]}) if val[1]: messages.append({"role": "assistant", "content": val[1]}) messages.append({"role": "user", "content": message}) response = "" for i in chatbot_mistral(messages): yield i """ For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface """ demo = gr.ChatInterface( respond, additional_inputs=[ gr.Textbox(value="You are a helpful, respectful and honest assistant.Help humman as much as you can.", label="System message"), gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"), gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"), gr.Slider( minimum=0.1, maximum=1.0, value=0.95, step=0.05, label="Top-p (nucleus sampling)", ), ], ) if __name__ == "__main__": demo.launch()