-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathmain.py
More file actions
115 lines (89 loc) · 2.91 KB
/
Copy pathmain.py
File metadata and controls
115 lines (89 loc) · 2.91 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
from fastapi import FastAPI, UploadFile, File, HTTPException
from dotenv import load_dotenv
from sentence_transformers import SentenceTransformer
import os
from uuid import uuid4
import io
from PyPDF2 import PdfReader
import docx
from pptx import Presentation
from fastapi import Query
from pydantic import BaseModel
from langchain.text_splitter import RecursiveCharacterTextSplitter
from pinecone import Pinecone, PodSpec
load_dotenv()
app = FastAPI()
model = SentenceTransformer('all-MiniLM-L6-v2')
pc = Pinecone(api_key=os.getenv("PINECONE_API_KEY"))
index_name = os.getenv("PINECONE_INDEX_NAME")
index = pc.Index(index_name)
def extract_text_from_pptx(file_bytes: io.BytesIO) -> str:
from pptx import Presentation
prs = Presentation(file_bytes)
text = ""
for slide in prs.slides:
for shape in slide.shapes:
if hasattr(shape, "text"):
text += shape.text + "\n"
return text.strip()
def extract_text(file: UploadFile) -> str:
ext = os.path.splitext(file.filename)[1].lower()
file_bytes = io.BytesIO(file.file.read())
if ext == ".pdf":
reader = PdfReader(file_bytes)
return "\n".join([page.extract_text() or "" for page in reader.pages])
elif ext == ".docx":
doc = docx.Document(file_bytes)
return "\n".join([p.text for p in doc.paragraphs])
elif ext == ".pptx":
return extract_text_from_pptx(file_bytes)
elif ext == ".txt":
return file_bytes.read().decode("utf-8")
else:
raise HTTPException(status_code=400, detail="Unsupported file format")
def chunk_text(text: str):
splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=50)
return splitter.split_text(text)
def embed_chunks(chunks):
return model.encode(chunks).tolist()
@app.post("/embed")
async def embed_file(file: UploadFile = File(...)):
try:
text = extract_text(file)
chunks = chunk_text(text)
embeddings = embed_chunks(chunks)
vectors = [{
"id": str(uuid4()),
"values": emb,
"metadata": {"text": chunk}
} for chunk, emb in zip(chunks, embeddings)]
index.upsert(vectors=vectors)
return {
"success": True,
"chunks_indexed": len(chunks)
}
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
class QueryRequest(BaseModel):
query: str
@app.post("/query")
async def query_docs(request: QueryRequest):
query_embedding = model.encode(request.query).tolist()
# Search Pinecone
response = index.query(
vector=query_embedding,
top_k=5,
include_metadata=True
)
matches = response.get("matches", [])
results = [
{
"score": match["score"],
"text": match["metadata"]["text"]
}
for match in matches
]
return {
"success": True,
"results": results
}