Update app.py

app.py

@@ -1,53 +1,13 @@
-import re
-import numpy as np
-import tensorflow_hub as hub
-import openai
+from langchain.embeddings import OpenAIEmbeddings
+from langchain.vectorstores import Chroma
+from langchain.llms import OpenAI
+from langchain.chains.question_answering import load_qa_chain
 import os
-import tensorflow_text
-from sklearn.neighbors import NearestNeighbors
-import gradio as gr
-import requests
-import json
-import fitz
-
-#这里填写调用openai需要的密钥
-openai.api_key = '9481961416fa4c8e883047c5679cf971'
-openai.api_base = 'https://demopro-oai-we2.openai.azure.com/' 
-openai.api_type = 'azure'
-openai.api_version = '2022-12-01'
-
-#将嵌套的列表展平
-def flatten(_2d_list):
-    flat_list = []
-    for element in _2d_list:
-        if type(element) is list:
-            for item in element:
-                flat_list.append(item)
-        else:
-            flat_list.append(element)
-    return flat_list
-
 
 def preprocess(text):
-    text = text.replace('\n', ' ')
-    text = re.sub('\s+', ' ', text)
+    text = text.replace('\n', '')
     return text
 
-#将pdf文档按段落分
-# def pdf_to_text(path):
-#     doc = pdfplumber.open(path)
-#     pages = doc.pages
-#     text_list=[]
-#     for page,d in enumerate(pages):
-#         d=d.extract_text()
-#         d=preprocess(d)
-#         text_list.append(d)  
-#     doc.close()   
-  
-#     return text_list
-
-
-
 def pdf_to_text(path, start_page=1, end_page=None):
     doc = fitz.open(path)
     total_pages = doc.page_count
@@ -59,125 +19,53 @@ def pdf_to_text(path, start_page=1, end_page=None):
 
     for i in range(start_page - 1, end_page):
         text = doc.load_page(i).get_text("text")
-        text = preprocess(text)
         text_list.append(text)
 
     doc.close()
     return text_list
 
+def law_split(path,name):
+    text_list=pdf_to_text(path)
+    text= ''.join(text_list)
+    text_split=re.split(r'第.+条\s',text)[1:]
 
-def text_to_chunks(texts, word_length=150, start_page=1):
-    text_toks = [t.split(' ') for t in texts]
-    page_nums = []
-    chunks = []
-
-    for idx, words in enumerate(text_toks):
-        for i in range(0, len(words), word_length):
-            chunk = words[i : i + word_length]
-            if (
-                (i + word_length) > len(words)
-                and (len(chunk) < word_length)
-                and (len(text_toks) != (idx + 1))
-            ):
-                text_toks[idx + 1] = chunk + text_toks[idx + 1]
-                continue
-            chunk = ' '.join(chunk).strip()
-            chunk = f'[Page no. {idx+start_page}]' + ' ' + '"' + chunk + '"'
-            chunks.append(chunk)
-    return chunks
+    for index, text in enumerate(text_split):
+        text=preprocess(text)
+        text_split[index]=f'《中华人民共和国{name}》 第{index+1}条 '+text
     
-history=pdf_to_text('The Elements of Statisitcal Learning.pdf',start_page=20)
-history=text_to_chunks(history,start_page=1)
+    return text_split
 
 
-def encoder(text):
-    embed=openai.Embedding.create(input=text, engine="text-embedding-ada-002")
-    return embed.get('data')[0].get('embedding')
-    
-    
-#定义语义搜索类
-class SemanticSearch:
-    
-    def __init__(self):
-        #类初始化，使用google公司的多语言语句编码，第一次运行时需要十几分钟的时间下载
-        self.use =hub.load('https://tfhub.dev/google/universal-sentence-encoder-multilingual/3')
-        self.fitted = False
-    
-    def get_text_embedding(self, texts, batch=1000):
-        embeddings = []
-        for i in range(0, len(texts), batch):
-            text_batch = texts[i : (i + batch)]
-            emb_batch = self.use(text_batch)
-            embeddings.append(emb_batch)
-        embeddings = np.vstack(embeddings)
-        return embeddings
-
-    
-    
-    #K近邻算法，找到与问题最相似的 k 个段落,这里的 k 即n_neighbors=10
-    def fit(self, data, batch=1000, n_neighbors=5):
-        self.data = data
-        self.embeddings = self.get_text_embedding(data, batch=batch)
-        n_neighbors = min(n_neighbors, len(self.embeddings))
-        self.nn = NearestNeighbors(n_neighbors=n_neighbors)
-        self.nn.fit(self.embeddings)
-        self.fitted = True
-    
-    #定义了该方法后，实例就可以被当作函数调用,text参数即用户提出的问题，inp_emb为其转化成的向量
-    def __call__(self, text, return_data=True):
-        inp_emb = self.use([text])
+def folder_read(path):
+    text_list=[]
+    paths=os.listdir(path)
+    for file in paths:
+        name=file.split('.')[0]
+        suffix=file.split('.')[-1]
+        if suffix=='pdf':
+            text_list+=law_split(f'{path}/{file}',name)
         
         
-        neighbors = self.nn.kneighbors(inp_emb, return_distance=False)[0]
-        
-        if return_data:
-            return [self.data[i] for i in neighbors]
-        else:
-            return neighbors
-    
-    
-#openai的api接口，engine参数为我们选择的大语言模型，prompt即提示词
-def generate_text(prompt, engine="text-davinci-003"):
-    completions = openai.Completion.create(
-        engine=engine,
-        prompt=prompt,
-        max_tokens=512,
-        n=1,
-        stop=None,
-        temperature=0.7,
-    )
-    message = completions.choices[0].text
-    return message
+    return text_list
+
+
+text_list=folder_read('laws')
+embeddings = OpenAIEmbeddings()
+vectordb = Chroma.from_texts(texts=text_list, embedding=embeddings)
+llm = OpenAI(temperature=0.5,max_tokens=1024)
 
 
 def generate_answer(question):
-    #匹配与问题最相近的n个段落，前面定义了n=10
-    topn_chunks = recommender(question)
-    prompt = ""
-    prompt += 'search results:\n\n'
-    
-    #把匹配到的段落加进提示词
-    for c in topn_chunks:
-        prompt += c + '\n\n'
-    
-    #提示词
-    prompt += '''
-    Instructions: 如果搜索结果中找不到相关信息，只需要回答'未在该文档中找到相关信息'。
-    如果找到了相关信息，请使用中文回答，回答尽量精确简洁。并在句子的末尾使用[页码]符号引用每个参考文献（每个结果的开头都有这个编号)
-    如果不确定答案是否正确，就仅给出相似段落的来源，不要回复错误的答案。
-    \n\nQuery: {question}\nAnswer: 
+    prompt='''
+    请根据给出的法律条文回答问题，给出适当的法律建议。回答时要说出你引用的法律条文是第几条，并说出引用的每一条是哪部法律中的。
+    引用的法律条文不要超过两条，回答尽量简明扼要
+    如果问题与搜索结果无关，就仅回答"该问题与青少年法律无关"即可。
     '''
-    
-    prompt += f"Query: {question}\nAnswer:"
-    answer = generate_text(prompt,"text-davinci-003")
+    most_relevant_texts = vectordb.max_marginal_relevance_search(question, k=5)
+    chain = load_qa_chain(llm)
+    answer = chain.run(input_documents=most_relevant_texts, question=question+prompt)
     return answer
-
-
-recommender = SemanticSearch()
-recommender.fit(history)
-
-
-#以下为web客户端搭建，运行后产生客户端界面
+    
 def ask_api(question):
     
     if question.strip() == '':
@@ -185,29 +73,24 @@ def ask_api(question):
 
     return generate_answer(question)
 
-title = 'Chat With Statistical Learning'
-description = """ 该机器人将以Trevor Hastie等人所著的The Elements of Statistical Learning Data Mining, Inference, and Prediction
-(即我们上课所用的课本)为主题回答你的问题，如果所问问题与书的内容无关，将会返回"未在该文档中找到相关信息"
-"""
 
-with gr.Blocks() as demo:
-    gr.Markdown(f'<center><h1>{title}</h1></center>')
-    gr.Markdown(description)
 
-    with gr.Row():
-        with gr.Group():
-            question = gr.Textbox(label='请输入你的问题')
-            btn = gr.Button(value='提交')
-            btn.style(full_width=True)
+title = '青少年法律科普问答'
+description = """ 本bot旨在根据中华人民共和国的法律回答有关青少年的问题，目前囊括的法律有\n
+《未成年人保护法》\n
+《义务教育法》\n
+《预防未成年人犯罪法》\n
+《妇女儿童权益保护法》
+"""
+
 
-        with gr.Group():
-            answer = gr.Textbox(label='回答:')
+demo = gr.Interface(
+    title=title,
+    description=description,
+    fn=ask_api, 
+    inputs=gr.Textbox(label="请输入与青少年法律相关的问题",lines=2), 
+    outputs=gr.outputs.Textbox(label="参考回答"), 
+    examples=[["未成年遭受网络欺凌该怎么办？"],['年满多少岁的儿童应当接受义务教育？'],['若发现离家出走的未成年人，应如何处理？']])
 
-        btn.click(
-            ask_api,
-            inputs=[question],
-            outputs=[answer]
-        )
 
-#参数share=True会产生一个公开网页，别人可以通过访问该网页使用你的模型，前提是你需要正在运行这段代码（将自己的电脑当作服务器）
 demo.launch()