Varun2007 commited on
Commit
b0716ab
·
1 Parent(s): df1c03e

Fix Gradio FileData path extraction and remove ZeroGPU decorator from file ingestion

Browse files
Files changed (2) hide show
  1. app.py +32 -12
  2. ingest.py +37 -16
app.py CHANGED
@@ -104,7 +104,6 @@ def get_status_markdown() -> str:
104
  """
105
 
106
 
107
- @gpu_decorator
108
  def process_uploaded_files(files) -> str:
109
  """Processes newly uploaded files and adds them to the index."""
110
  global indexed_filenames
@@ -112,26 +111,47 @@ def process_uploaded_files(files) -> str:
112
  return get_status_markdown()
113
 
114
  new_parsed = []
 
 
115
  for file_obj in files:
116
- file_path = file_obj.name if hasattr(file_obj, "name") else str(file_obj)
117
- orig_name = os.path.basename(file_path)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
118
  try:
119
- parsed = pipeline.ingest_file(file_path)
120
- parsed.file_name = orig_name
121
- for c in parsed.chunks:
122
- c.file_name = orig_name
123
- new_parsed.append(parsed)
124
- if orig_name not in indexed_filenames:
125
- indexed_filenames.append(orig_name)
126
  except Exception as e:
127
- print(f"Error parsing {orig_name}: {e}")
128
 
129
  if new_parsed:
130
  existing = list(retriever.chunks)
131
  combined = existing + [c for p in new_parsed for c in p.chunks]
132
  retriever.index_documents(combined)
133
 
134
- return get_status_markdown()
 
 
 
135
 
136
 
137
  def reload_samples_action() -> str:
 
104
  """
105
 
106
 
 
107
  def process_uploaded_files(files) -> str:
108
  """Processes newly uploaded files and adds them to the index."""
109
  global indexed_filenames
 
111
  return get_status_markdown()
112
 
113
  new_parsed = []
114
+ error_messages = []
115
+
116
  for file_obj in files:
117
+ # Robust path and filename extraction across Gradio FileData / wrappers
118
+ if hasattr(file_obj, "path"):
119
+ file_path = file_obj.path
120
+ orig_name = getattr(file_obj, "orig_name", None) or os.path.basename(file_path)
121
+ elif hasattr(file_obj, "name"):
122
+ file_path = file_obj.name
123
+ orig_name = getattr(file_obj, "orig_name", None) or os.path.basename(file_path)
124
+ elif isinstance(file_obj, dict):
125
+ file_path = file_obj.get("path") or file_obj.get("name")
126
+ orig_name = file_obj.get("orig_name") or os.path.basename(file_path)
127
+ else:
128
+ file_path = str(file_obj)
129
+ orig_name = os.path.basename(file_path)
130
+
131
+ if not file_path or not os.path.exists(file_path):
132
+ error_messages.append(f"File not found: {orig_name}")
133
+ continue
134
+
135
  try:
136
+ parsed = pipeline.ingest_file(file_path, original_filename=orig_name)
137
+ if parsed.chunks:
138
+ new_parsed.append(parsed)
139
+ if orig_name not in indexed_filenames:
140
+ indexed_filenames.append(orig_name)
141
+ else:
142
+ error_messages.append(f"No extractable text found in: {orig_name}")
143
  except Exception as e:
144
+ error_messages.append(f"Error parsing {orig_name}: {str(e)}")
145
 
146
  if new_parsed:
147
  existing = list(retriever.chunks)
148
  combined = existing + [c for p in new_parsed for c in p.chunks]
149
  retriever.index_documents(combined)
150
 
151
+ status_md = get_status_markdown()
152
+ if error_messages:
153
+ status_md += "\n\n**⚠️ Upload Warnings:**\n" + "\n".join([f"- {msg}" for msg in error_messages])
154
+ return status_md
155
 
156
 
157
  def reload_samples_action() -> str:
ingest.py CHANGED
@@ -138,30 +138,51 @@ class DocumentIngestionPipeline:
138
  self.ocr_fallback = ocr_fallback
139
  self.ocr_min_chars = ocr_min_chars
140
 
141
- def ingest_file(self, file_path: str, doc_id: Optional[str] = None) -> ParsedDocument:
 
 
 
 
 
142
  """Main entry point to parse and chunk a document."""
143
  if not os.path.exists(file_path):
144
  raise FileNotFoundError(f"File not found: {file_path}")
145
 
146
- file_name = os.path.basename(file_path)
147
  ext = os.path.splitext(file_name)[1].lower()
148
  if not doc_id:
149
  doc_id = str(uuid.uuid4())[:8]
150
 
151
- if ext == ".pdf":
152
- raw_elements, total_pages = self._parse_pdf(file_path)
153
- file_type = "pdf"
154
- elif ext in [".docx", ".doc"]:
155
- raw_elements, total_pages = self._parse_docx(file_path)
156
- file_type = "docx"
157
- elif ext in [".txt", ".md"]:
158
- raw_elements, total_pages = self._parse_txt(file_path)
159
- file_type = "txt"
160
- else:
161
- raise ValueError(f"Unsupported file type: {ext}. Supported: .pdf, .docx, .txt, .md")
162
-
163
- # Chunk parsed elements with structure awareness
164
- chunks = self._chunk_elements(raw_elements, doc_id, file_name)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
165
 
166
  return ParsedDocument(
167
  doc_id=doc_id,
 
138
  self.ocr_fallback = ocr_fallback
139
  self.ocr_min_chars = ocr_min_chars
140
 
141
+ def ingest_file(
142
+ self,
143
+ file_path: str,
144
+ doc_id: Optional[str] = None,
145
+ original_filename: Optional[str] = None
146
+ ) -> ParsedDocument:
147
  """Main entry point to parse and chunk a document."""
148
  if not os.path.exists(file_path):
149
  raise FileNotFoundError(f"File not found: {file_path}")
150
 
151
+ file_name = original_filename or os.path.basename(file_path)
152
  ext = os.path.splitext(file_name)[1].lower()
153
  if not doc_id:
154
  doc_id = str(uuid.uuid4())[:8]
155
 
156
+ # Ensure working path has proper extension for third-party parsers
157
+ actual_path = file_path
158
+ temp_target = None
159
+ if not file_path.lower().endswith(ext) and ext:
160
+ import shutil
161
+ temp_target = f"{file_path}_{uuid.uuid4().hex[:6]}{ext}"
162
+ shutil.copyfile(file_path, temp_target)
163
+ actual_path = temp_target
164
+
165
+ try:
166
+ if ext == ".pdf":
167
+ raw_elements, total_pages = self._parse_pdf(actual_path)
168
+ file_type = "pdf"
169
+ elif ext in [".docx", ".doc"]:
170
+ raw_elements, total_pages = self._parse_docx(actual_path)
171
+ file_type = "docx"
172
+ elif ext in [".txt", ".md"]:
173
+ raw_elements, total_pages = self._parse_txt(actual_path)
174
+ file_type = "txt"
175
+ else:
176
+ raise ValueError(f"Unsupported file type: {ext}. Supported: .pdf, .docx, .txt, .md")
177
+
178
+ # Chunk parsed elements with structure awareness
179
+ chunks = self._chunk_elements(raw_elements, doc_id, file_name)
180
+ finally:
181
+ if temp_target and os.path.exists(temp_target):
182
+ try:
183
+ os.remove(temp_target)
184
+ except Exception:
185
+ pass
186
 
187
  return ParsedDocument(
188
  doc_id=doc_id,