Update app.py
Browse files
app.py
CHANGED
@@ -56,34 +56,46 @@ def generate_chunk_id(pdf_file, chunk_text, chunk_idx):
|
|
56 |
|
57 |
# Function to process PDFs and upsert embeddings to Pinecone
|
58 |
def process_pdfs(pdf_files):
|
|
|
59 |
for pdf_file in pdf_files:
|
|
|
|
|
|
|
60 |
reader = PdfReader(pdf_file.name)
|
61 |
pdf_text = "".join(page.extract_text() for page in reader.pages if page.extract_text())
|
|
|
|
|
62 |
|
63 |
# Split text into smaller chunks
|
64 |
chunks = [pdf_text[i:i+500] for i in range(0, len(pdf_text), 500)]
|
65 |
|
|
|
|
|
66 |
# Generate embeddings in batches
|
67 |
embeddings = encode_chunks_batch(chunks, batch_size=8)
|
68 |
|
|
|
|
|
69 |
# Prepare data for Pinecone with unique IDs
|
70 |
vectors = [
|
71 |
(generate_chunk_id(pdf_file, chunk, idx), embedding.tolist(), {"text": chunk})
|
72 |
for idx, (embedding, chunk) in enumerate(zip(embeddings, chunks))
|
73 |
]
|
74 |
|
|
|
|
|
75 |
# Upsert embeddings into Pinecone
|
76 |
index.upsert(vectors)
|
77 |
|
78 |
# Fetch index stats
|
79 |
stats = index.describe_index_stats()
|
80 |
|
81 |
-
|
82 |
|
83 |
# Gradio Interface
|
84 |
demo = gr.Interface(
|
85 |
fn=process_pdfs,
|
86 |
-
inputs=gr.Files(label="Upload
|
87 |
outputs="text",
|
88 |
title="NASA Bi-encoder PDF Embedding & Pinecone Storage",
|
89 |
description="Upload PDF files to generate embeddings with NASA Bi-encoder and store in Pinecone."
|
|
|
56 |
|
57 |
# Function to process PDFs and upsert embeddings to Pinecone
|
58 |
def process_pdfs(pdf_files):
|
59 |
+
|
60 |
for pdf_file in pdf_files:
|
61 |
+
|
62 |
+
yield "Reading PDF..."
|
63 |
+
|
64 |
reader = PdfReader(pdf_file.name)
|
65 |
pdf_text = "".join(page.extract_text() for page in reader.pages if page.extract_text())
|
66 |
+
|
67 |
+
yield "Processing PDF..."
|
68 |
|
69 |
# Split text into smaller chunks
|
70 |
chunks = [pdf_text[i:i+500] for i in range(0, len(pdf_text), 500)]
|
71 |
|
72 |
+
yield "Generating Embeddings..."
|
73 |
+
|
74 |
# Generate embeddings in batches
|
75 |
embeddings = encode_chunks_batch(chunks, batch_size=8)
|
76 |
|
77 |
+
yield "Embeddings generated successfully...Preparing..."
|
78 |
+
|
79 |
# Prepare data for Pinecone with unique IDs
|
80 |
vectors = [
|
81 |
(generate_chunk_id(pdf_file, chunk, idx), embedding.tolist(), {"text": chunk})
|
82 |
for idx, (embedding, chunk) in enumerate(zip(embeddings, chunks))
|
83 |
]
|
84 |
|
85 |
+
yield "Pushing to Pinecone...Please wait"
|
86 |
+
|
87 |
# Upsert embeddings into Pinecone
|
88 |
index.upsert(vectors)
|
89 |
|
90 |
# Fetch index stats
|
91 |
stats = index.describe_index_stats()
|
92 |
|
93 |
+
yield f"Processed {len(pdf_files)} PDF(s) successfully and embeddings stored in Pinecone. Current Index Stats: {stats}"
|
94 |
|
95 |
# Gradio Interface
|
96 |
demo = gr.Interface(
|
97 |
fn=process_pdfs,
|
98 |
+
inputs=gr.Files(label="Upload PDF", file_types=[".pdf"]),
|
99 |
outputs="text",
|
100 |
title="NASA Bi-encoder PDF Embedding & Pinecone Storage",
|
101 |
description="Upload PDF files to generate embeddings with NASA Bi-encoder and store in Pinecone."
|