From e1b7852dfc4e4f304024ac4fcc7115f0f5a5d955 Mon Sep 17 00:00:00 2001 From: Edward Johnson Date: Wed, 2 Aug 2023 18:11:13 -0400 Subject: [PATCH 1/6] Create doc_qa.py --- generative_ai/langchain/doc_qa.py | 76 +++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 generative_ai/langchain/doc_qa.py diff --git a/generative_ai/langchain/doc_qa.py b/generative_ai/langchain/doc_qa.py new file mode 100644 index 0000000..50cc5c6 --- /dev/null +++ b/generative_ai/langchain/doc_qa.py @@ -0,0 +1,76 @@ +# https://python.langchain.com/docs/use_cases/question_answering.html +from langchain.document_loaders import UnstructuredPDFLoader# PDFMinerLoader#UnstructuredPDFLoader # PyPDFLoader +from langchain.text_splitter import RecursiveCharacterTextSplitter +from langchain.embeddings import OpenAIEmbeddings +from langchain.vectorstores import Chroma +from langchain.chat_models import ChatOpenAI +# from langchain.chains import RetrievalQA +from langchain.chains import RetrievalQAWithSourcesChain +from langchain.prompts import PromptTemplate +import os + + +# Step 1. Load +loader = UnstructuredPDFLoader("./data/example.pdf") +data = loader.load() +# pages = loader.load_and_split() + +# Step 2. Split +text_splitter = RecursiveCharacterTextSplitter(chunk_size = 500, chunk_overlap = 0) +doc_splits = text_splitter.split_documents(data) + +# Step 3. Store +vectorstore = Chroma.from_documents( + documents=doc_splits, embedding=OpenAIEmbeddings(), + persist_directory="./data/chroma_db", + collection_name="hr_collection" +) +# ---- DB interaction: https://docs.trychroma.com/api-reference +# import chromadb +# client = chromadb.PersistentClient(path="src/data/chroma_db") +# client.list_collections() +# client.delete_collection("") +#--------------------------- + +# Step 4. Retrieve +question = "summarize the document in 3 bulletpoints?" +docs = vectorstore.similarity_search(question) +len(docs) + +# Step 5. Generate + +template = """Use the following pieces of context to answer the question at the end. +If you don't know the answer, just say that you don't know, don't try to make up an answer. +Use five sentences maximum and keep the answer as concise as possible. +{context} +Question: {question} +Helpful Answer:""" +# `from_template` method will automatically infer the input_variables based on the template passed. +prompt = PromptTemplate.from_template(template) + +llm = ChatOpenAI(model_name="gpt-3.5-turbo", temperature=0) +qa_chain = RetrievalQAWithSourcesChain.from_chain_type( + llm, + retriever=vectorstore.as_retriever(), + chain_type="stuff", # "stuffs" all retrieved documents into the prompt. + # chain_type_kwargs={"prompt": prompt} + verbose=True +) +result = qa_chain({"question": question}) +result + +# Step 6. Converse (Extension) +from langchain.memory import ConversationBufferMemory + +buffer_memory = ConversationBufferMemory(memory_key="chat_history", return_messages=True, k=2) + +qa_chain = RetrievalQAWithSourcesChain.from_chain_type( + llm, + retriever=vectorstore.as_retriever(), + chain_type="stuff", # "stuffs" all retrieved documents into the prompt. + memory=buffer_memory, + chain_type_kwargs={"prompt": QA_CHAIN_PROMPT}, + verbose=True +) +result = qa_chain({"query": question}) +result From a6a9518c80f8e35cd77a0d74253319aedc7139a6 Mon Sep 17 00:00:00 2001 From: Edward Johnson Date: Wed, 2 Aug 2023 18:14:08 -0400 Subject: [PATCH 2/6] Create requirements.txt --- generative_ai/langchain/requirements.txt | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 generative_ai/langchain/requirements.txt diff --git a/generative_ai/langchain/requirements.txt b/generative_ai/langchain/requirements.txt new file mode 100644 index 0000000..3fae819 --- /dev/null +++ b/generative_ai/langchain/requirements.txt @@ -0,0 +1,14 @@ +chromadb +langchain +openai +pandas +pypdf +pdf2image +pdfminer.six +pyodbc +pytest-cov +pytesseract +tabulate +tiktoken +unstructured +xlrd From b234f9a7e8d3aa79c3b0938c00a5720149894599 Mon Sep 17 00:00:00 2001 From: Edward Johnson Date: Tue, 8 Aug 2023 10:08:33 -0400 Subject: [PATCH 3/6] Update doc_qa.py --- generative_ai/langchain/doc_qa.py | 92 +++++++++++++------------------ 1 file changed, 37 insertions(+), 55 deletions(-) diff --git a/generative_ai/langchain/doc_qa.py b/generative_ai/langchain/doc_qa.py index 50cc5c6..83e7c23 100644 --- a/generative_ai/langchain/doc_qa.py +++ b/generative_ai/langchain/doc_qa.py @@ -1,5 +1,10 @@ # https://python.langchain.com/docs/use_cases/question_answering.html -from langchain.document_loaders import UnstructuredPDFLoader# PDFMinerLoader#UnstructuredPDFLoader # PyPDFLoader +from langchain.llms import OpenAI +from langchain.chains import ( + StuffDocumentsChain, LLMChain, ConversationalRetrievalChain +) +# PDFMinerLoader#UnstructuredPDFLoader # PyPDFLoader +from langchain.document_loaders import UnstructuredPDFLoader from langchain.text_splitter import RecursiveCharacterTextSplitter from langchain.embeddings import OpenAIEmbeddings from langchain.vectorstores import Chroma @@ -7,70 +12,47 @@ # from langchain.chains import RetrievalQA from langchain.chains import RetrievalQAWithSourcesChain from langchain.prompts import PromptTemplate +from langchain.chains.qa_with_sources import load_qa_with_sources_chain +import pprint import os - -# Step 1. Load -loader = UnstructuredPDFLoader("./data/example.pdf") -data = loader.load() -# pages = loader.load_and_split() - -# Step 2. Split -text_splitter = RecursiveCharacterTextSplitter(chunk_size = 500, chunk_overlap = 0) -doc_splits = text_splitter.split_documents(data) - -# Step 3. Store -vectorstore = Chroma.from_documents( - documents=doc_splits, embedding=OpenAIEmbeddings(), +# Step 1. Load vectorstore +vectorstore = Chroma( persist_directory="./data/chroma_db", - collection_name="hr_collection" + collection_name="test_collection", + embedding_function=OpenAIEmbeddings() ) -# ---- DB interaction: https://docs.trychroma.com/api-reference -# import chromadb -# client = chromadb.PersistentClient(path="src/data/chroma_db") -# client.list_collections() -# client.delete_collection("") -#--------------------------- -# Step 4. Retrieve -question = "summarize the document in 3 bulletpoints?" -docs = vectorstore.similarity_search(question) -len(docs) +# Step 2. Generate +# streaming_llm = OpenAI(streaming=True, callbacks=[StreamingStdOutCallbackHandler()], temperature=0) -# Step 5. Generate +# libs/langchain/langchain/chains/conversational_retrieval/prompts.py +template = """You are a friendly assistant. Given the following conversation and a follow up question, rephrase the follow up question to be a standalone question, in its original language. -template = """Use the following pieces of context to answer the question at the end. -If you don't know the answer, just say that you don't know, don't try to make up an answer. -Use five sentences maximum and keep the answer as concise as possible. -{context} -Question: {question} -Helpful Answer:""" -# `from_template` method will automatically infer the input_variables based on the template passed. -prompt = PromptTemplate.from_template(template) +Chat History: +{chat_history} +Follow Up Input: {question} +Standalone question:""" +prompt = PromptTemplate.from_template(template) llm = ChatOpenAI(model_name="gpt-3.5-turbo", temperature=0) -qa_chain = RetrievalQAWithSourcesChain.from_chain_type( - llm, - retriever=vectorstore.as_retriever(), - chain_type="stuff", # "stuffs" all retrieved documents into the prompt. - # chain_type_kwargs={"prompt": prompt} - verbose=True +from langchain.chains.conversational_retrieval.prompts import CONDENSE_QUESTION_PROMPT +question_generator_chain = LLMChain(llm=llm, prompt=prompt) # prompt=prompt; CONDENSE_QUESTION_PROMPT +doc_chain = load_qa_with_sources_chain(llm, chain_type="stuff") + +chain = ConversationalRetrievalChain( + retriever=vectorstore.as_retriever(search_type="similarity", search_kwargs={"k": 4}), + question_generator=question_generator_chain, + combine_docs_chain=doc_chain, ) -result = qa_chain({"question": question}) -result -# Step 6. Converse (Extension) -from langchain.memory import ConversationBufferMemory +chat_history = [] +question = "what question should i ask you?" +result = chain({"question": question, "chat_history": chat_history}) +pprint.pprint(result) -buffer_memory = ConversationBufferMemory(memory_key="chat_history", return_messages=True, k=2) +chat_history = [(question, result['answer'])] +question = "What other questions should i ask you?" +result = chain({"question": question, "chat_history": chat_history}) +pprint.pprint(result) -qa_chain = RetrievalQAWithSourcesChain.from_chain_type( - llm, - retriever=vectorstore.as_retriever(), - chain_type="stuff", # "stuffs" all retrieved documents into the prompt. - memory=buffer_memory, - chain_type_kwargs={"prompt": QA_CHAIN_PROMPT}, - verbose=True -) -result = qa_chain({"query": question}) -result From 46b487109e3d2bff97dae16e393924eedb9d3b5e Mon Sep 17 00:00:00 2001 From: Edward Johnson Date: Tue, 8 Aug 2023 10:11:58 -0400 Subject: [PATCH 4/6] Create load_vectordb.py --- generative_ai/langchain/load_vectordb.py | 40 ++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 generative_ai/langchain/load_vectordb.py diff --git a/generative_ai/langchain/load_vectordb.py b/generative_ai/langchain/load_vectordb.py new file mode 100644 index 0000000..d341e85 --- /dev/null +++ b/generative_ai/langchain/load_vectordb.py @@ -0,0 +1,40 @@ +# https://python.langchain.com/docs/use_cases/question_answering.html + +from langchain.document_loaders import UnstructuredPDFLoader # PyPDFLoader +from langchain.text_splitter import RecursiveCharacterTextSplitter +from langchain.embeddings import OpenAIEmbeddings +from langchain.vectorstores import Chroma + +# Step 1. Load +loader = UnstructuredPDFLoader("./data/test.pdf") # PyPDFLoader +data = loader.load() + +# Step 2. Split +text_splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=0) +doc_splits = text_splitter.split_documents(data) + +# Step 3. Store +vectorstore = Chroma.from_documents( + documents=doc_splits, embedding=OpenAIEmbeddings(), + persist_directory="./data/chroma_db", + collection_name="test_collection" +) + +# Step 4. Retrieve from vectorDB +question = "" +docs = vectorstore.similarity_search(question, collection_name="test_collection",) +len(docs) + +# optional load from disk: +vectorstore = Chroma( + persist_directory="./data/chroma_db", + collection_name="test_collection", + embedding_function=OpenAIEmbeddings() +) + +# ---- DB interaction: https://docs.trychroma.com/api-reference +# import chromadb +# client = chromadb.PersistentClient(path="./data/chroma_db") +# client.list_collections() +# client.delete_collection("langchain") +# --------------------------- From bc07dbf03584ecd2d6167f2bfbf0b2c571cfd459 Mon Sep 17 00:00:00 2001 From: Edward Johnson Date: Tue, 8 Aug 2023 19:27:33 -0400 Subject: [PATCH 5/6] Update load_vectordb.py --- generative_ai/langchain/load_vectordb.py | 152 ++++++++++++++++++----- 1 file changed, 123 insertions(+), 29 deletions(-) diff --git a/generative_ai/langchain/load_vectordb.py b/generative_ai/langchain/load_vectordb.py index d341e85..5f65ee7 100644 --- a/generative_ai/langchain/load_vectordb.py +++ b/generative_ai/langchain/load_vectordb.py @@ -1,36 +1,130 @@ -# https://python.langchain.com/docs/use_cases/question_answering.html - -from langchain.document_loaders import UnstructuredPDFLoader # PyPDFLoader +from langchain.document_loaders import UnstructuredPDFLoader from langchain.text_splitter import RecursiveCharacterTextSplitter from langchain.embeddings import OpenAIEmbeddings from langchain.vectorstores import Chroma -# Step 1. Load -loader = UnstructuredPDFLoader("./data/test.pdf") # PyPDFLoader -data = loader.load() - -# Step 2. Split -text_splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=0) -doc_splits = text_splitter.split_documents(data) - -# Step 3. Store -vectorstore = Chroma.from_documents( - documents=doc_splits, embedding=OpenAIEmbeddings(), - persist_directory="./data/chroma_db", - collection_name="test_collection" -) - -# Step 4. Retrieve from vectorDB -question = "" -docs = vectorstore.similarity_search(question, collection_name="test_collection",) -len(docs) - -# optional load from disk: -vectorstore = Chroma( - persist_directory="./data/chroma_db", - collection_name="test_collection", - embedding_function=OpenAIEmbeddings() -) +class DocumentProcessor: + """Handles loading and splitting of documents.""" + + def __init__(self, pdf_path, chunk_size=500, chunk_overlap=0): + """ + Initialize the DocumentProcessor. + + Args: + pdf_path (str): Path to the PDF document. + chunk_size (int, optional): Size of text chunks. Defaults to 500. + chunk_overlap (int, optional): Overlap between text chunks. Defaults to 0. + """ + self.pdf_path = pdf_path + self.chunk_size = chunk_size + self.chunk_overlap = chunk_overlap + + def load_document(self): + """ + Load the document using UnstructuredPDFLoader. + + Returns: + str: Loaded document data. + """ + loader = UnstructuredPDFLoader(self.pdf_path) + return loader.load() + + def split_document(self, data): + """ + Split the document into chunks using RecursiveCharacterTextSplitter. + + Args: + data (str): Document data. + + Returns: + list: List of document splits. + """ + text_splitter = RecursiveCharacterTextSplitter(chunk_size=self.chunk_size, chunk_overlap=self.chunk_overlap) + return text_splitter.split_documents(data) + +class VectorStoreManager: + """Handles vector store creation and similarity searches.""" + + def __init__(self, persist_directory, collection_name, embedding_function): + """ + Initialize the VectorStoreManager. + + Args: + persist_directory (str): Directory for vector store persistence. + collection_name (str): Name of the collection. + embedding_function (callable): Embedding function for creating vectors. + """ + self.persist_directory = persist_directory + self.collection_name = collection_name + self.embedding_function = embedding_function + + def create_vector_store(self, documents): + """ + Create a vector store using Chroma.from_documents. + + Args: + documents (list): List of document splits. + + Returns: + Chroma: Created vector store. + """ + vectorstore = Chroma.from_documents( + documents=documents, embedding=self.embedding_function, + persist_directory=self.persist_directory, + collection_name=self.collection_name + ) + return vectorstore + + def similarity_search(self, question): + """ + Perform a similarity search in the vector store. + + Args: + question (str): Question for similarity search. + + Returns: + list: List of similar documents. + """ + vectorstore = Chroma( + persist_directory=self.persist_directory, + collection_name=self.collection_name, + embedding_function=self.embedding_function + ) + return vectorstore.similarity_search(question, collection_name=self.collection_name) + +def main(): + """Main function to orchestrate the document processing and similarity search.""" + pdf_path = "./data/test.pdf" + persist_directory = "./data/chroma_db" + collection_name = "test_collection" + + processor = DocumentProcessor(pdf_path) + data = processor.load_document() + doc_splits = processor.split_document(data) + + embeddings = OpenAIEmbeddings() + + vectorstore_manager = VectorStoreManager(persist_directory, collection_name, embeddings) + vectorstore = vectorstore_manager.create_vector_store(doc_splits) + + question = "" + similar_docs = vectorstore_manager.similarity_search(question) + num_similar_docs = len(similar_docs) + + print(f"Number of similar documents: {num_similar_docs}") + print(f"Retrieved similar documents: {similar_docs}") + +if __name__ == "__main__": + main() + + +#---- optional load from disk: +# vectorstore = Chroma( +# persist_directory="./data/chroma_db", +# collection_name="test_collection", +# embedding_function=OpenAIEmbeddings() +# ) +# --------------------------- # ---- DB interaction: https://docs.trychroma.com/api-reference # import chromadb From f16d32f8b2d74f04d2f0b56a9147257d332a3d0b Mon Sep 17 00:00:00 2001 From: Edward Johnson Date: Tue, 8 Aug 2023 19:54:17 -0400 Subject: [PATCH 6/6] Update doc_qa.py --- generative_ai/langchain/doc_qa.py | 104 +++++++++++++++++++----------- 1 file changed, 68 insertions(+), 36 deletions(-) diff --git a/generative_ai/langchain/doc_qa.py b/generative_ai/langchain/doc_qa.py index 83e7c23..a56db15 100644 --- a/generative_ai/langchain/doc_qa.py +++ b/generative_ai/langchain/doc_qa.py @@ -1,58 +1,90 @@ -# https://python.langchain.com/docs/use_cases/question_answering.html from langchain.llms import OpenAI from langchain.chains import ( StuffDocumentsChain, LLMChain, ConversationalRetrievalChain ) -# PDFMinerLoader#UnstructuredPDFLoader # PyPDFLoader from langchain.document_loaders import UnstructuredPDFLoader -from langchain.text_splitter import RecursiveCharacterTextSplitter from langchain.embeddings import OpenAIEmbeddings from langchain.vectorstores import Chroma from langchain.chat_models import ChatOpenAI -# from langchain.chains import RetrievalQA -from langchain.chains import RetrievalQAWithSourcesChain -from langchain.prompts import PromptTemplate from langchain.chains.qa_with_sources import load_qa_with_sources_chain +from langchain.prompts import PromptTemplate import pprint -import os -# Step 1. Load vectorstore -vectorstore = Chroma( - persist_directory="./data/chroma_db", - collection_name="test_collection", - embedding_function=OpenAIEmbeddings() -) +def load_vectorstore(persist_directory, collection_name, embedding_function): + """ + Load a vector store. + + Args: + persist_directory (str): Directory for vector store persistence. + collection_name (str): Name of the collection. + embedding_function (callable): Embedding function for creating vectors. + + Returns: + Chroma: Loaded vector store. + """ + vectorstore = Chroma( + persist_directory=persist_directory, + collection_name=collection_name, + embedding_function=embedding_function + ) + return vectorstore + +def chain_executor(question, chat_history, chat_model, question_generator_chain, document_chain): + """ + Execute the conversation chain to generate a rephrased question based on conversation history. -# Step 2. Generate -# streaming_llm = OpenAI(streaming=True, callbacks=[StreamingStdOutCallbackHandler()], temperature=0) + Args: + question (str): Input question. + chat_history (list): List of (question, answer) tuples representing chat history. + chat_model (ChatOpenAI): Chat model for question generation. + question_generator_chain: LLMChain for generating questions. + document_chain: QAWithSourcesChain for document-based question answering. -# libs/langchain/langchain/chains/conversational_retrieval/prompts.py -template = """You are a friendly assistant. Given the following conversation and a follow up question, rephrase the follow up question to be a standalone question, in its original language. + Returns: + dict: Result containing the answer and rephrased question. + """ + retriever = chat_model({"question": question, "chat_history": chat_history}) + condensed_question = question_generator_chain({"question": retriever['answer'], "chat_history": chat_history}) + answer_with_sources = document_chain({"question": condensed_question, "retriever": retriever}) + return {"answer": retriever['answer'], "rephrased_question": condensed_question, "sources": answer_with_sources} + +def main(): + """ + Main function to orchestrate the conversation and question generation. + """ + persist_directory = "./data/chroma_db" + collection_name = "test_collection" + + embeddings = OpenAIEmbeddings() + vectorstore = load_vectorstore(persist_directory, collection_name, embeddings) + + llm = ChatOpenAI(model_name="gpt-3.5-turbo", temperature=0) + template = """You are a friendly assistant. Given the following conversation and a follow up question, rephrase the follow up question to be a standalone question, in its original language. Chat History: {chat_history} Follow Up Input: {question} Standalone question:""" + prompt = PromptTemplate.from_template(template) + question_generator_chain = LLMChain(llm=llm, prompt=prompt) + doc_chain = load_qa_with_sources_chain(llm, chain_type="stuff") -prompt = PromptTemplate.from_template(template) -llm = ChatOpenAI(model_name="gpt-3.5-turbo", temperature=0) -from langchain.chains.conversational_retrieval.prompts import CONDENSE_QUESTION_PROMPT -question_generator_chain = LLMChain(llm=llm, prompt=prompt) # prompt=prompt; CONDENSE_QUESTION_PROMPT -doc_chain = load_qa_with_sources_chain(llm, chain_type="stuff") - -chain = ConversationalRetrievalChain( - retriever=vectorstore.as_retriever(search_type="similarity", search_kwargs={"k": 4}), - question_generator=question_generator_chain, - combine_docs_chain=doc_chain, -) + retriever = vectorstore.as_retriever(search_type="similarity", search_kwargs={"k": 4}) + chain = ConversationalRetrievalChain( + retriever=retriever, + question_generator=question_generator_chain, + combine_docs_chain=doc_chain, + ) -chat_history = [] -question = "what question should i ask you?" -result = chain({"question": question, "chat_history": chat_history}) -pprint.pprint(result) + chat_history = [] + question = "what question should i ask you?" + result = chain_executor(question, chat_history, llm, question_generator_chain, doc_chain) + pprint.pprint(result) -chat_history = [(question, result['answer'])] -question = "What other questions should i ask you?" -result = chain({"question": question, "chat_history": chat_history}) -pprint.pprint(result) + chat_history.append((question, result['answer'])) + question = "What other questions should i ask you?" + result = chain_executor(question, chat_history, llm, question_generator_chain, doc_chain) + pprint.pprint(result) +if __name__ == "__main__": + main()