seraj-b2 commited on
Commit
02cda1b
·
1 Parent(s): 5df365d

pdf upload route added

Browse files
README.md CHANGED
@@ -51,6 +51,42 @@ It is fully optimized to be deployed to **Hugging Face Spaces** via the Docker S
51
  "chunks_created": 1
52
  }
53
  ```
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
54
 
55
  ### 2. Retrieve Context
56
  * **Route**: `POST /retrieve`
 
51
  "chunks_created": 1
52
  }
53
  ```
54
+ * **Response**:
55
+ ```json
56
+ {
57
+ "status": "success",
58
+ "message": "Successfully indexed 1 documents into 1 chunks.",
59
+ "documents_processed": 1,
60
+ "chunks_created": 1
61
+ }
62
+ ```
63
+
64
+ ### 1b. Upload Document File
65
+ * **Route**: `POST /documents/upload`
66
+ * **Content-Type**: `multipart/form-data`
67
+ * **Description**: Accepts an uploaded file (`.pdf` or `.txt`), automatically extracts text, chunks, embeds, and stores it in Qdrant.
68
+ * **Form Fields**:
69
+ - `file`: The document file to upload (PDF or TXT).
70
+ - `chunk_size` (Optional): Maximum character length of each chunk (Default: `500`, Range: `50`-`5000`).
71
+ - `chunk_overlap` (Optional): Character overlap between adjacent chunks (Default: `50`, Range: `0`-`1000`).
72
+ - `metadata` (Optional): A JSON-formatted string representing custom key-value metadata to attach to the ingested chunks (e.g. `{"category": "physics"}`).
73
+ * **Example curl command**:
74
+ ```bash
75
+ curl -X POST "http://localhost:7860/documents/upload" \
76
+ -F "file=@/path/to/document.pdf" \
77
+ -F "chunk_size=500" \
78
+ -F "chunk_overlap=50" \
79
+ -F "metadata={\"category\": \"research\", \"author\": \"John Doe\"}"
80
+ ```
81
+ * **Response**:
82
+ ```json
83
+ {
84
+ "status": "success",
85
+ "message": "Successfully indexed document 'document.pdf' into 4 chunks.",
86
+ "documents_processed": 1,
87
+ "chunks_created": 4
88
+ }
89
+ ```
90
 
91
  ### 2. Retrieve Context
92
  * **Route**: `POST /retrieve`
__pycache__/main.cpython-314.pyc CHANGED
Binary files a/__pycache__/main.cpython-314.pyc and b/__pycache__/main.cpython-314.pyc differ
 
__pycache__/test_api.cpython-314-pytest-9.0.3.pyc CHANGED
Binary files a/__pycache__/test_api.cpython-314-pytest-9.0.3.pyc and b/__pycache__/test_api.cpython-314-pytest-9.0.3.pyc differ
 
main.py CHANGED
@@ -1,10 +1,13 @@
 
 
1
  import logging
2
  from typing import List, Optional
3
- from fastapi import FastAPI, HTTPException, status
4
  from fastapi.middleware.cors import CORSMiddleware
5
  from pydantic import BaseModel, Field
6
  from sentence_transformers import SentenceTransformer
7
  from langchain_text_splitters import RecursiveCharacterTextSplitter
 
8
 
9
  from vector_store import QdrantVectorStore
10
 
@@ -160,6 +163,156 @@ def ingest_documents(payload: IngestionRequest):
160
  detail=f"An error occurred while indexing documents: {str(e)}"
161
  )
162
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
163
  @app.post("/retrieve", response_model=RetrievalResponse, status_code=status.HTTP_200_OK)
164
  def retrieve_context(payload: RetrievalRequest):
165
  """
 
1
+ import io
2
+ import json
3
  import logging
4
  from typing import List, Optional
5
+ from fastapi import FastAPI, HTTPException, status, File, UploadFile, Form
6
  from fastapi.middleware.cors import CORSMiddleware
7
  from pydantic import BaseModel, Field
8
  from sentence_transformers import SentenceTransformer
9
  from langchain_text_splitters import RecursiveCharacterTextSplitter
10
+ from pypdf import PdfReader
11
 
12
  from vector_store import QdrantVectorStore
13
 
 
163
  detail=f"An error occurred while indexing documents: {str(e)}"
164
  )
165
 
166
+ @app.post("/documents/upload", response_model=IngestionResponse, status_code=status.HTTP_201_CREATED)
167
+ async def upload_document(
168
+ file: UploadFile = File(...),
169
+ chunk_size: int = Form(500),
170
+ chunk_overlap: int = Form(50),
171
+ metadata: Optional[str] = Form(None)
172
+ ):
173
+ """
174
+ Ingests an uploaded document (.pdf or .txt), chunks, embeds, and stores it in the vector database.
175
+ """
176
+ # Validate form parameters
177
+ if chunk_size < 50 or chunk_size > 5000:
178
+ raise HTTPException(
179
+ status_code=status.HTTP_400_BAD_REQUEST,
180
+ detail="chunk_size must be between 50 and 5000."
181
+ )
182
+ if chunk_overlap < 0 or chunk_overlap > 1000:
183
+ raise HTTPException(
184
+ status_code=status.HTTP_400_BAD_REQUEST,
185
+ detail="chunk_overlap must be between 0 and 1000."
186
+ )
187
+
188
+ # Parse optional metadata JSON string
189
+ parsed_metadata = {}
190
+ if metadata:
191
+ try:
192
+ parsed_metadata = json.loads(metadata)
193
+ if not isinstance(parsed_metadata, dict):
194
+ raise HTTPException(
195
+ status_code=status.HTTP_400_BAD_REQUEST,
196
+ detail="Metadata must be a JSON object (dictionary)."
197
+ )
198
+ except json.JSONDecodeError:
199
+ raise HTTPException(
200
+ status_code=status.HTTP_400_BAD_REQUEST,
201
+ detail="Metadata must be a valid JSON-formatted string."
202
+ )
203
+
204
+ filename = file.filename or ""
205
+ content_type = file.content_type or ""
206
+
207
+ logger.info(f"Received file upload: {filename} (Content-Type: {content_type})")
208
+
209
+ try:
210
+ file_bytes = await file.read()
211
+ except Exception as e:
212
+ logger.error(f"Failed to read uploaded file: {e}")
213
+ raise HTTPException(
214
+ status_code=status.HTTP_400_BAD_REQUEST,
215
+ detail=f"Failed to read uploaded file: {str(e)}"
216
+ )
217
+
218
+ text_content = ""
219
+ is_pdf = filename.lower().endswith(".pdf") or content_type == "application/pdf"
220
+ is_txt = filename.lower().endswith(".txt") or content_type.startswith("text/")
221
+
222
+ if is_pdf:
223
+ try:
224
+ pdf_file = io.BytesIO(file_bytes)
225
+ reader = PdfReader(pdf_file)
226
+ extracted_pages = []
227
+ for page in reader.pages:
228
+ page_text = page.extract_text()
229
+ if page_text:
230
+ extracted_pages.append(page_text)
231
+ text_content = "\n".join(extracted_pages)
232
+
233
+ if not text_content.strip():
234
+ raise HTTPException(
235
+ status_code=status.HTTP_400_BAD_REQUEST,
236
+ detail="The uploaded PDF file does not contain any extractable text."
237
+ )
238
+ except HTTPException:
239
+ raise
240
+ except Exception as e:
241
+ logger.error(f"Error parsing PDF file: {e}")
242
+ raise HTTPException(
243
+ status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
244
+ detail=f"Error parsing PDF file: {str(e)}"
245
+ )
246
+ elif is_txt or content_type == "":
247
+ try:
248
+ text_content = file_bytes.decode("utf-8")
249
+ except UnicodeDecodeError:
250
+ try:
251
+ text_content = file_bytes.decode("latin-1")
252
+ except Exception as e:
253
+ logger.error(f"Failed to decode text file: {e}")
254
+ raise HTTPException(
255
+ status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
256
+ detail=f"Failed to decode text file as UTF-8 or Latin-1: {str(e)}"
257
+ )
258
+ else:
259
+ raise HTTPException(
260
+ status_code=status.HTTP_400_BAD_REQUEST,
261
+ detail="Unsupported file format. Only PDF (.pdf) and raw text (.txt) files are supported."
262
+ )
263
+
264
+ if not text_content.strip():
265
+ raise HTTPException(
266
+ status_code=status.HTTP_400_BAD_REQUEST,
267
+ detail="The uploaded file contains no text content to index."
268
+ )
269
+
270
+ text_splitter = RecursiveCharacterTextSplitter(
271
+ chunk_size=chunk_size,
272
+ chunk_overlap=chunk_overlap,
273
+ length_function=len
274
+ )
275
+
276
+ try:
277
+ chunks = text_splitter.split_text(text_content)
278
+ if not chunks:
279
+ return IngestionResponse(
280
+ status="success",
281
+ message="No chunks were created from the uploaded file.",
282
+ documents_processed=1,
283
+ chunks_created=0
284
+ )
285
+
286
+ all_metadatas = []
287
+ for chunk in chunks:
288
+ meta = parsed_metadata.copy()
289
+ meta["source"] = filename
290
+ all_metadatas.append(meta)
291
+
292
+ logger.info(f"Generating embeddings for {len(chunks)} chunks from uploaded file...")
293
+ embeddings = embedding_model.encode(chunks, show_progress_bar=False).tolist()
294
+
295
+ logger.info("Upserting vectors into Qdrant...")
296
+ chunks_inserted = vector_store.add_documents(
297
+ texts=chunks,
298
+ embeddings=embeddings,
299
+ metadatas=all_metadatas
300
+ )
301
+
302
+ return IngestionResponse(
303
+ status="success",
304
+ message=f"Successfully indexed document '{filename}' into {chunks_inserted} chunks.",
305
+ documents_processed=1,
306
+ chunks_created=chunks_inserted
307
+ )
308
+
309
+ except Exception as e:
310
+ logger.error(f"Error during document file ingestion: {e}")
311
+ raise HTTPException(
312
+ status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
313
+ detail=f"An error occurred while indexing the uploaded file: {str(e)}"
314
+ )
315
+
316
  @app.post("/retrieve", response_model=RetrievalResponse, status_code=status.HTTP_200_OK)
317
  def retrieve_context(payload: RetrievalRequest):
318
  """
requirements.txt CHANGED
@@ -8,4 +8,5 @@ numpy>=1.20.0
8
  gunicorn>=20.1.0
9
  httpx>=0.24.0
10
  pytest>=7.0.0
11
-
 
 
8
  gunicorn>=20.1.0
9
  httpx>=0.24.0
10
  pytest>=7.0.0
11
+ pypdf>=4.0.0
12
+ python-multipart>=0.0.9
test_api.py CHANGED
@@ -102,7 +102,107 @@ def test_rag_pipeline_flow():
102
  status_response = client.get("/documents")
103
  assert status_response.json()["vectors_count"] == 0
104
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
105
  if __name__ == "__main__":
106
  import sys
107
  import pytest
108
  sys.exit(pytest.main([__file__]))
 
 
102
  status_response = client.get("/documents")
103
  assert status_response.json()["vectors_count"] == 0
104
 
105
+ def test_upload_text_document():
106
+ # 1. Clear database before starting
107
+ client.delete("/documents")
108
+
109
+ # 2. Upload a txt file
110
+ dummy_text_data = "This is raw text content about photosynthesis from an uploaded text file."
111
+
112
+ response = client.post(
113
+ "/documents/upload",
114
+ files={"file": ("test.txt", dummy_text_data.encode("utf-8"), "text/plain")},
115
+ data={"chunk_size": 100, "chunk_overlap": 10, "metadata": '{"category": "test_txt"}'}
116
+ )
117
+
118
+ assert response.status_code == 201
119
+ data = response.json()
120
+ assert data["status"] == "success"
121
+ assert data["documents_processed"] == 1
122
+ assert data["chunks_created"] > 0
123
+
124
+ # Verify retrieval
125
+ retrieve_response = client.post("/retrieve", json={"query": "photosynthesis", "top_k": 1})
126
+ assert retrieve_response.status_code == 200
127
+ retrieve_data = retrieve_response.json()
128
+ assert len(retrieve_data["results"]) > 0
129
+ assert "photosynthesis" in retrieve_data["results"][0]["text"]
130
+ assert retrieve_data["results"][0]["metadata"]["source"] == "test.txt"
131
+ assert retrieve_data["results"][0]["metadata"]["category"] == "test_txt"
132
+
133
+ from unittest.mock import MagicMock, patch
134
+
135
+ def test_upload_pdf_document():
136
+ # 1. Clear database before starting
137
+ client.delete("/documents")
138
+
139
+ # 2. Mock PdfReader to return a dummy page with text
140
+ mock_page = MagicMock()
141
+ mock_page.extract_text.return_value = "This is text extracted from a mock PDF file about gravity."
142
+
143
+ mock_reader = MagicMock()
144
+ mock_reader.pages = [mock_page]
145
+
146
+ with patch("main.PdfReader", return_value=mock_reader) as mock_pdf_reader:
147
+ # Create a dummy PDF byte stream
148
+ dummy_pdf_data = b"%PDF-1.4 dummy content"
149
+
150
+ response = client.post(
151
+ "/documents/upload",
152
+ files={"file": ("test.pdf", dummy_pdf_data, "application/pdf")},
153
+ data={"chunk_size": 100, "chunk_overlap": 10, "metadata": '{"category": "test_pdf"}'}
154
+ )
155
+
156
+ assert response.status_code == 201
157
+ data = response.json()
158
+ assert data["status"] == "success"
159
+ assert data["documents_processed"] == 1
160
+ assert data["chunks_created"] > 0
161
+
162
+ # Verify PdfReader was initialized with the file bytes
163
+ mock_pdf_reader.assert_called_once()
164
+ called_args, _ = mock_pdf_reader.call_args
165
+ assert called_args[0].getvalue() == dummy_pdf_data
166
+
167
+ # Verify retrieval
168
+ retrieve_response = client.post("/retrieve", json={"query": "gravity", "top_k": 1})
169
+ assert retrieve_response.status_code == 200
170
+ retrieve_data = retrieve_response.json()
171
+ assert len(retrieve_data["results"]) > 0
172
+ assert "gravity" in retrieve_data["results"][0]["text"]
173
+ assert retrieve_data["results"][0]["metadata"]["source"] == "test.pdf"
174
+ assert retrieve_data["results"][0]["metadata"]["category"] == "test_pdf"
175
+
176
+ def test_upload_invalid_file_type():
177
+ response = client.post(
178
+ "/documents/upload",
179
+ files={"file": ("test.png", b"fake png bytes", "image/png")},
180
+ data={"chunk_size": 100, "chunk_overlap": 10}
181
+ )
182
+ assert response.status_code == 400
183
+ assert "Unsupported file format" in response.json()["detail"]
184
+
185
+ def test_upload_invalid_metadata():
186
+ response = client.post(
187
+ "/documents/upload",
188
+ files={"file": ("test.txt", b"some text", "text/plain")},
189
+ data={"metadata": "{invalid json}"}
190
+ )
191
+ assert response.status_code == 400
192
+ assert "valid JSON" in response.json()["detail"]
193
+
194
+ def test_upload_invalid_parameters():
195
+ # Invalid chunk_size
196
+ response = client.post(
197
+ "/documents/upload",
198
+ files={"file": ("test.txt", b"some text", "text/plain")},
199
+ data={"chunk_size": 10} # Below 50 limit
200
+ )
201
+ assert response.status_code == 400
202
+ assert "chunk_size must be between" in response.json()["detail"]
203
+
204
  if __name__ == "__main__":
205
  import sys
206
  import pytest
207
  sys.exit(pytest.main([__file__]))
208
+