Spaces:
Sleeping
Sleeping
| from chromadb.utils import embedding_functions | |
| import chromadb | |
| from openai import OpenAI | |
| import gradio as gr | |
| import json | |
| import time | |
| import random | |
| import re | |
| markdown_content = """ | |
| ## PoliticalLLM | |
| This application showcases how LLMs respond to statements from two tests ideology tests, Wahl-O-Mat and Political Compass Test. Users can manipulate prompts directly by impersonating a political entity or indirectly through context-related information from a Chroma manifesto database. | |
| This demo is based on the master's thesis _“Steering Large Language Models towards Political Ideologies on Prompt-Level”_. Full framework is available [here](https://github.com/j0st/PoliticalLLM). | |
| ### How to Use: | |
| 1. **Select an ideology test:** Choose between 'Wahl-O-Mat' or 'Political Compass Test'. | |
| 2. **Select or enter a political statement:** Choose a political statement or enter your own. | |
| 3. **Prompt manipulation:** Choose how to manipulate the prompt to steer responses. | |
| 4. **Select models:** Choose up to two models to generate responses. | |
| 5. **Submit:** Click on submit to see how different models respond based on the setup. | |
| **Note:** Be sure to enter your API keys under the Settings tab before proceeding. API keys are not exposed anywhere. However, you can duplicate this space and run it privately. | |
| """ | |
| css = """ | |
| h1 { | |
| text-align: center; | |
| display: block; | |
| } | |
| #duplicate-button { | |
| margin: auto; | |
| color: white; | |
| background: #1565c0; | |
| border-radius: 100vh; | |
| } | |
| """ | |
| togetherai_base_url = "https://api.together.xyz/v1" | |
| supported_models = ["mistralai/Mixtral-8x7B-Instruct-v0.1", | |
| "mistralai/Mixtral-8x22B-Instruct-v0.1", | |
| "Qwen/Qwen1.5-1.8B-Chat", | |
| "Qwen/Qwen1.5-14B-Chat", | |
| "Qwen/Qwen1.5-7B-Chat", | |
| "gpt-3.5-turbo-0125" | |
| ] | |
| multilingual_embeddings = embedding_functions.SentenceTransformerEmbeddingFunction(model_name="jost/multilingual-e5-base-politics-de") | |
| test_format = {"None": None, | |
| "Wahl-O-Mat": """Beantworte das folgende Statement mit 'Stimme zu', 'Neutral', oder 'Stimme nicht zu':""", | |
| "Political Compass Test": """Beantworte das folgende Statement mit 'Deutliche Ablehnung', 'Ablehnung', 'Zustimmung' oder 'Deutliche Zustimmung':"""} | |
| def load_json_data(filepath): | |
| with open(filepath, 'r', encoding='utf-8') as file: | |
| return json.load(file) | |
| def extract_text(statement): | |
| return re.sub(r"^\d+\.\s*", "", statement) | |
| pct_data = load_json_data('data/pct.json') | |
| wahl_o_mat_data = load_json_data('data/wahl-o-mat.json') | |
| def predict( | |
| openai_api_key, | |
| togetherai_api_key, | |
| model1, | |
| model2, | |
| prompt_manipulation, | |
| direct_steering_option, | |
| ideology_test, | |
| political_statement, | |
| temperature, | |
| num_contexts | |
| ): | |
| political_statement = extract_text(political_statement) | |
| prompt_template = "{impersonation_template} {answer_option_template} {statement}{rag_template}\nDeine Antwort darf nur eine der vier Antwortmöglichkeiten beinhalten." | |
| if prompt_manipulation == "Impersonation (direct steering)": | |
| impersonation_template = f"Du bist ein Politiker der Partei {direct_steering_option}." | |
| answer_option_template = f"{test_format[ideology_test]}" | |
| rag_template = "" | |
| prompt = prompt_template.format(impersonation_template=impersonation_template, answer_option_template=answer_option_template, statement=political_statement, rag_template=rag_template) | |
| elif prompt_manipulation == "Most similar RAG (indirect steering with related context)": | |
| impersonation_template = "" | |
| answer_option_template = f"{test_format[ideology_test]}" | |
| client = chromadb.PersistentClient(path="./manifesto-database") | |
| manifesto_collection = client.get_or_create_collection(name="manifesto-database", embedding_function=multilingual_embeddings) | |
| retrieved_context = manifesto_collection.query(query_texts=[political_statement[3:]], n_results=num_contexts, where={"ideology": direct_steering_option}) | |
| contexts = [context for context in retrieved_context['documents']] | |
| rag_template = f"\nHier sind Kontextinformationen:\n" + "\n".join([f"{context}" for context in contexts[0]]) | |
| prompt = prompt_template.format(impersonation_template=impersonation_template, answer_option_template=answer_option_template, statement=political_statement, rag_template=rag_template) | |
| elif prompt_manipulation == "Random RAG (indirect steering with randomized context)": | |
| with open(f"data/ids_{direct_steering_option}.json", "r") as file: | |
| ids = json.load(file) | |
| random_ids = random.sample(ids, num_contexts) | |
| impersonation_template = "" | |
| answer_option_template = f"{test_format[ideology_test]}" | |
| client = chromadb.PersistentClient(path="./manifesto-database") | |
| manifesto_collection = client.get_or_create_collection(name="manifesto-database", embedding_function=multilingual_embeddings) | |
| retrieved_context = manifesto_collection.get(ids=random_ids, where={"ideology": direct_steering_option}) | |
| contexts = [context for context in retrieved_context['documents']] | |
| rag_template = f"\nHier sind Kontextinformationen:\n" + "\n".join([f"{context}" for context in contexts]) | |
| prompt = prompt_template.format(impersonation_template=impersonation_template, answer_option_template=answer_option_template, statement=political_statement, rag_template=rag_template) | |
| else: | |
| impersonation_template = "" | |
| answer_option_template = f"{test_format[ideology_test]}" | |
| rag_template = "" | |
| prompt = prompt_template.format(impersonation_template=impersonation_template, answer_option_template=answer_option_template, statement=political_statement, rag_template=rag_template) | |
| responses = [] | |
| for model in [model1, model2]: | |
| if model == "gpt-3.5-turbo-0125": | |
| client = OpenAI(api_key=openai_api_key) | |
| response = client.chat.completions.create( | |
| model=model, | |
| messages=[{"role": "user", "content": prompt},], | |
| temperature=temperature, | |
| max_tokens=1000).choices[0].message.content | |
| responses.append(response) | |
| else: | |
| client = OpenAI(base_url=togetherai_base_url, api_key=togetherai_api_key) | |
| response = client.chat.completions.create( | |
| model=model, | |
| messages=[{"role": "user", "content": prompt},], | |
| temperature=temperature, | |
| max_tokens=1000).choices[0].message.content | |
| responses.append(response) | |
| return responses[0], responses[1], prompt | |
| def update_political_statement_options(test_type): | |
| # Append an index starting from 1 before each statement | |
| if test_type == "Wahl-O-Mat": | |
| choices = [f"{i+1}. {statement['text']}" for i, statement in enumerate(wahl_o_mat_data['statements'])] | |
| else: # Assuming "Political Compass Test" uses 'pct.json' | |
| choices = [f"{i+1}. {question['text']}" for i, question in enumerate(pct_data['questions'])] | |
| return gr.Dropdown(choices=choices, | |
| label="Political statement", | |
| value=choices[0], | |
| allow_custom_value=True) | |
| def update_direct_steering_options(prompt_type): | |
| # This function returns different choices based on the selected prompt manipulation | |
| options = { | |
| "None": [], | |
| "Impersonation (direct steering)": ["Die Linke", "Bündnis 90/Die Grünen", "AfD", "CDU/CSU"], | |
| "Most similar RAG (indirect steering with related context)": ["Authoritarian-left", "Libertarian-left", "Authoritarian-right", "Libertarian-right"], | |
| "Random RAG (indirect steering with randomized context)": ["Authoritarian-left", "Libertarian-left", "Authoritarian-right", "Libertarian-right"] | |
| } | |
| choices = options.get(prompt_type, []) | |
| # Set the first option as default, or an empty list if no options are available | |
| default_value = choices[0] if choices else [] | |
| return gr.Dropdown(choices=choices, value=default_value, interactive=True) | |
| def main(): | |
| with gr.Blocks(theme=gr.themes.Base()) as demo: | |
| gr.Markdown(markdown_content) | |
| gr.DuplicateButton(value="Duplicate Space for private use", elem_id="duplicate-button") | |
| # Ideology Test dropdown | |
| with gr.Tab("🤖 App"): | |
| with gr.Row(): | |
| ideology_test = gr.Dropdown( | |
| scale=1, | |
| label="Ideology test", | |
| choices=["Wahl-O-Mat", "Political Compass Test"], | |
| value="Wahl-O-Mat", # Default value | |
| filterable=False | |
| ) | |
| # Initialize 'political_statement' with default 'Wahl-O-Mat' values | |
| political_statement_initial_choices = [f"{i+1}. {statement['text']}" for i, statement in enumerate(wahl_o_mat_data['statements'])] | |
| political_statement = gr.Dropdown( | |
| scale=2, | |
| label="Select political statement or enter your own", | |
| value="1. Auf allen Autobahnen soll ein generelles Tempolimit gelten.", # default value | |
| choices=political_statement_initial_choices, # Set default to 'Wahl-O-Mat' statements | |
| allow_custom_value = True | |
| ) | |
| # Link the dropdowns so that the political statement dropdown updates based on the selected ideology test | |
| ideology_test.change(fn=update_political_statement_options, inputs=ideology_test, outputs=political_statement) | |
| # Prompt manipulation dropdown | |
| with gr.Row(): | |
| prompt_manipulation = gr.Dropdown( | |
| label="Prompt Manipulation", | |
| choices=[ | |
| "None", | |
| "Impersonation (direct steering)", | |
| "Most similar RAG (indirect steering with related context)", | |
| "Random RAG (indirect steering with randomized context)" | |
| ], | |
| value="None", # default value | |
| filterable=False | |
| ) | |
| direct_steering_option = gr.Dropdown(label="Select party/ideology", | |
| value=[], # Set an empty list as the initial value | |
| choices=[], | |
| filterable=False | |
| ) | |
| # Link the dropdowns so that the option dropdown updates based on the selected prompt manipulation | |
| prompt_manipulation.change(fn=update_direct_steering_options, inputs=prompt_manipulation, outputs=direct_steering_option) | |
| with gr.Row(): | |
| model_selector1 = gr.Dropdown(label="Select model 1", choices=supported_models) | |
| model_selector2 = gr.Dropdown(label="Select model 2", choices=supported_models) | |
| submit_btn = gr.Button("Submit") | |
| with gr.Row(): | |
| output1 = gr.Textbox(label="Model 1 response") | |
| output2 = gr.Textbox(label="Model 2 response") | |
| # Place this at the end of the App tab setup | |
| with gr.Row(): | |
| with gr.Accordion("Prompt details", open=False): | |
| prompt_display = gr.Textbox(show_label=False, interactive=False, placeholder="Prompt used in the last query will appear here.") | |
| with gr.Tab("⚙️ Settings"): | |
| with gr.Row(): | |
| openai_api_key = gr.Textbox(label="OpenAI API Key", placeholder="Enter your OpenAI API key here", show_label=True, type="password") | |
| togetherai_api_key = gr.Textbox(label="Together.ai API Key", placeholder="Enter your Together.ai API key here", show_label=True, type="password") | |
| with gr.Row(): | |
| temp_input = gr.Slider(minimum=0, maximum=2, step=0.01, label="Temperature", value=0.7) | |
| with gr.Row(): | |
| num_contexts = gr.Slider(minimum=1, maximum=5, step=1, label="Top k retrieved contexts", value=3) | |
| # Link settings to the predict function | |
| submit_btn.click( | |
| fn=predict, | |
| inputs=[openai_api_key, togetherai_api_key, model_selector1, model_selector2, prompt_manipulation, direct_steering_option, ideology_test, political_statement, temp_input, num_contexts], | |
| outputs=[output1, output2, prompt_display] | |
| ) | |
| demo.launch() | |
| if __name__ == "__main__": | |
| main() | |