import streamlit as st from langchain.document_loaders import PyPDFLoader from langchain.text_splitter import RecursiveCharacterTextSplitter from langchain.embeddings import SentenceTransformerEmbeddings from langchain.vectorstores import Chroma from constants import CHROMA_SETTINGS persist_directory = "db" def main(): st.title("PDF Processor") uploaded_file = st.file_uploader("Upload a PDF file") if uploaded_file is not None: st.write("Processing PDF...") loader = PyPDFLoader(uploaded_file.read()) documents = loader.load() st.write("Splitting into chunks") text_splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=100) texts = text_splitter.split_documents(documents) st.write("Loading sentence transformers model") embeddings = SentenceTransformerEmbeddings(model_name="all-MiniLM-L6-v2") st.write("Creating embeddings. This may take some time...") db = Chroma.from_documents(texts, embeddings, persist_directory=persist_directory, client_settings=CHROMA_SETTINGS) db.persist() db = None st.success("Ingestion complete! You can now run privateGPT.py to query your documents") if __name__ == "__main__": main()