Spaces:
Sleeping
Sleeping
File size: 9,008 Bytes
4ea53b9 0711ef5 4ea53b9 2a7e3b8 4ea53b9 2a7e3b8 4ea53b9 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 |
import pandas as pd
import gradio as gr
import hashlib, base64
import openai
from tqdm import tqdm
tqdm().pandas()
# querying OpenAI for generation
import openAI_manager as oai_mgr
#import initOpenAI, examples_to_prompt, genChatGPT, generateTestSentences
# bias testing manager
import mgr_bias_scoring as bt_mgr
import mgr_sentences as smgr
# error messages
from error_messages import *
G_CORE_BIAS_NAME = None
# hashing
def getHashForString(text):
d=hashlib.md5(bytes(text, encoding='utf-8')).digest()
d=base64.urlsafe_b64encode(d)
return d.decode('utf-8')
def getBiasName(gr1_lst, gr2_lst, att1_lst, att2_lst):
global G_CORE_BIAS_NAME
bias_name = G_CORE_BIAS_NAME
if bias_name == None:
full_spec = ''.join(gr1_lst)+''.join(gr2_lst)+''.join(att1_lst)+''.join(att2_lst)
hash = getHashForString(full_spec)
bias_name = f"{gr1_lst[0].replace(' ','-')}_{gr2_lst[0].replace(' ','-')}__{att1_lst[0].replace(' ','-')}_{att2_lst[0].replace(' ','-')}_{hash}"
return bias_name
def _generateOnline(bias_spec, progress, key, num2gen, isSaving=False):
test_sentences = []
gen_err_msg = None
genAttrCounts = {}
print(f"Bias spec dict: {bias_spec}")
g1, g2, a1, a2 = bt_mgr.get_words(bias_spec)
print(f"A1: {a1}")
print(f"A2: {a2}")
if "custom_counts" in bias_spec:
print("Bias spec is custom !!")
genAttrCounts = bias_spec['custom_counts'][0]
for a,c in bias_spec['custom_counts'][1].items():
genAttrCounts[a] = c
else:
print("Bias spec is standard !!")
genAttrCounts = {a:num2gen for a in a1+a2}
# Initiate with key
try:
models = oai_mgr.initOpenAI(key)
model_names = [m['id'] for m in models['data']]
print(f"Model names: {model_names}")
except openai.error.AuthenticationError as err:
#raise gr.Error(OPENAI_INIT_ERROR.replace("<ERR>", str(err)))
gen_err_msg = OPENAI_INIT_ERROR.replace("<ERR>", str(err))
if gen_err_msg != None:
return [], gen_err_msg
else:
if "gpt-3.5-turbo" in model_names:
print("Access to ChatGPT")
if "gpt-4" in model_names:
print("Access to GPT-4")
model_name = "gpt-3.5-turbo" #"gpt-4"
# Generate one example
#gen = genChatGPT(model_name, ["man","math"], 2, 5,
# [{"Keywords": ["sky","blue"], "Sentence": "the sky is blue"}
# ],
# temperature=0.8)
#print(f"Test gen: {gen}")
# Generate all test sentences
#gens = oai_mgr.generateTestSentences(model_name, g1+g2, a1+a2, num2gen, progress)
gens = oai_mgr.generateTestSentencesCustom(model_name, g1, g2, a1+a2, genAttrCounts, bias_spec, progress)
print("--GENS--")
print(gens)
if len(gens) == 0:
print("No sentences generated, returning")
return [], gen_err_msg
for org_gt, at, s, gt1, gt2 in gens:
test_sentences.append([s,org_gt,at,gt1,gt2])
# save the generations immediately
print("Making save dataframe...")
save_df = pd.DataFrame(test_sentences, columns=["Sentence",'org_grp_term',
"Attribute term", "Group term 1",
"Group term 2"])
## make the templates to save
# 1. bias specification
print(f"Bias spec dict: {bias_spec}")
# generate laternative sentence
print(f"Columns before alternative sentence: {list(save_df.columns)}")
save_df['Alternative Sentence'] = save_df.progress_apply(oai_mgr.chatgpt_sentence_alternative, axis=1, model_name=model_name)
print(f"Columns after alternative sentence: {list(save_df.columns)}")
# 2. convert to templates
save_df['Template'] = save_df.progress_apply(bt_mgr.sentence_to_template_df, axis=1)
print("Convert generated sentences to templates...")
save_df[['Alternative Template','grp_refs']] = save_df.progress_apply(bt_mgr.ref_terms_sentence_to_template, axis=1)
print(f"Columns with templates: {list(save_df.columns)}")
# 3. convert to pairs
print("Convert generated sentences to ordered pairs...")
test_pairs_df = bt_mgr.convert2pairsFromDF(bias_spec, save_df)
print(f"Test pairs cols: {list(test_pairs_df.columns)}")
bias_name = getBiasName(g1, g2, a1, a2)
save_df = save_df.rename(columns={"Sentence":'sentence',
"Alternative Sentence":"alt_sentence",
"Attribute term": 'att_term',
"Template":"template",
"Alternative Template": "alt_template",
"Group term 1": "grp_term1",
"Group term 2": "grp_term2"})
save_df['label_1'] = test_pairs_df['label_1']
save_df['label_2'] = test_pairs_df['label_2']
save_df['bias_spec'] = bias_name
save_df['type'] = 'tool'
save_df['gen_model'] = model_name
col_order = ["sentence", "alt_sentence", "org_grp_term", "att_term", "template",
"alt_template", "grp_term1", "grp_term2", "grp_refs", "label_1", "label_2",
"bias_spec", "type", "gen_model"]
save_df = save_df[col_order]
print(f"Save cols prep: {list(save_df.columns)}")
if isSaving == True:
print(f"Saving: {save_df.head(1)}")
smgr.saveSentences(save_df) #[["Group term","Attribute term","Test sentence"]])
num_sentences = len(test_sentences)
print(f"Returned num sentences: {num_sentences}")
# list for Gradio dataframe
ret_df = [list(r.values) for i, r in save_df[['sentence', 'alt_sentence', 'grp_term1', 'grp_term2', "att_term"]].iterrows()]
print(ret_df)
return ret_df, gen_err_msg
def _getSavedSentences(bias_spec, progress, use_paper_sentences):
test_sentences = []
print(f"Bias spec dict: {bias_spec}")
g1, g2, a1, a2 = bt_mgr.get_words(bias_spec)
for gi, g_term in enumerate(g1+g2):
att_list = a1+a2
grp_list = g1+g2
# match "-" and no space
att_list_dash = [t.replace(' ','-') for t in att_list]
att_list.extend(att_list_dash)
att_list_nospace = [t.replace(' ','') for t in att_list]
att_list.extend(att_list_nospace)
att_list = list(set(att_list))
#progress(gi/len(g1+g2), desc=f"{g_term}")
_, sentence_df, _ = smgr.getSavedSentences(g_term)#, gi, len(g1+g2), progress)
if sentence_df.shape[0] > 0:
progress(gi/len(g1+g2), desc=f"{sentence_df['sentence'].tolist()[0]}")
else:
progress(gi/len(g1+g2), desc=f"{g_term}")
# only take from paper & gpt3.5
flt_gen_models = ["gpt-3.5","gpt-3.5-turbo","gpt-4"]
print(f"Before filter: {sentence_df.shape[0]}")
if use_paper_sentences == True:
if 'type' in list(sentence_df.columns):
sentence_df = sentence_df.query("type=='paper' and gen_model in @flt_gen_models")
print(f"After filter: {sentence_df.shape[0]}")
else:
if 'type' in list(sentence_df.columns):
# only use GPT-3.5 generations for now - todo: add settings option for this
sentence_df = sentence_df.query("gen_model in @flt_gen_models")
print(f"After filter: {sentence_df.shape[0]}")
if sentence_df.shape[0] > 0:
sentence_df = sentence_df[['grp_term1','grp_term2','att_term','sentence','alt_sentence']]
sentence_df = sentence_df.rename(columns={'grp_term1': "Group term 1",
'grp_term2': "Group term 2",
"att_term": "Attribute term",
"sentence": "Sentence",
"alt_sentence": "Alt Sentence"})
sel = sentence_df[(sentence_df['Attribute term'].isin(att_list)) & \
((sentence_df['Group term 1'].isin(grp_list)) & (sentence_df['Group term 2'].isin(grp_list))) ].values
if len(sel) > 0:
for gt1,gt2,at,s,a_s in sel:
#if at == "speech-language-pathologist":
# print(f"Special case: {at}")
# at == "speech-language pathologist" # legacy, special case
#else:
#at = at #.replace("-"," ")
#gt = gt #.replace("-"," ")
test_sentences.append([s,a_s,gt1,gt2,at])
else:
print("Test sentences empty!")
#raise gr.Error(NO_SENTENCES_ERROR)
return test_sentences |