eranmi commited on
Commit
a452dd5
Β·
2 Parent(s): 75c75ced3925e1

Merge branch 'dev' of https://github.com/AnnaErsh/RAGnTeX into dev

Browse files
src/database/db_manipulation.py CHANGED
@@ -12,7 +12,7 @@ LOGGER = Logger.get_logger()
12
 
13
 
14
  @observe(name="𝌊 ingest_files_to_db")
15
- def ingest_files_to_db(pdf_files) -> None:
16
  """Add new files to the database if they are not yet there.
17
  Args:
18
  pdf_files (list): List of PDF files to be ingested.
@@ -37,7 +37,7 @@ def ingest_files_to_db(pdf_files) -> None:
37
  return
38
 
39
  # Process new files only
40
- documents, metadatas = process_documents(new_pdfs)
41
 
42
  LOGGER.info("βž• Adding %d new PDFs to the database...", len(new_pdfs))
43
  if documents:
@@ -48,6 +48,8 @@ def ingest_files_to_db(pdf_files) -> None:
48
  )
49
  LOGGER.info("βœ… Documents ingested successfully.")
50
 
 
 
51
 
52
  Metadata = Mapping[str, Union[str, int, float, bool]]
53
  DocumentsBatch = List[List[str]]
 
12
 
13
 
14
  @observe(name="𝌊 ingest_files_to_db")
15
+ def ingest_files_to_db(pdf_files) -> list[str]:
16
  """Add new files to the database if they are not yet there.
17
  Args:
18
  pdf_files (list): List of PDF files to be ingested.
 
37
  return
38
 
39
  # Process new files only
40
+ documents, metadatas, failed = process_documents(new_pdfs)
41
 
42
  LOGGER.info("βž• Adding %d new PDFs to the database...", len(new_pdfs))
43
  if documents:
 
48
  )
49
  LOGGER.info("βœ… Documents ingested successfully.")
50
 
51
+ return failed
52
+
53
 
54
  Metadata = Mapping[str, Union[str, int, float, bool]]
55
  DocumentsBatch = List[List[str]]
src/interface/gradio_interface.py CHANGED
@@ -156,8 +156,8 @@ with gr.Blocks(theme=theme, js=JS_FUNC) as demo:
156
  )
157
  gr.HTML(
158
  f"""
159
- <div style="display: flex; flex-direction: row; align-items: center; justify-content: center; gap: 12px; margin-bottom: 12px;">
160
- <p style="font-size: 16px;">Please ⭐️ us on GitHub!</p>
161
  <a href="https://github.com/RAGnTeX/RAGnTeX" target="_blank" style="
162
  display: inline-flex;
163
  align-items: center;
@@ -166,13 +166,16 @@ with gr.Blocks(theme=theme, js=JS_FUNC) as demo:
166
  background-color: #4c1d95;
167
  color: white;
168
  font-weight: 600;
169
- border-radius: 8px;
 
 
170
  text-decoration: none;
171
  gap: 8px;
172
  transition: background-color 0.2s ease-in-out;
 
173
  "
174
- onmouseover="this.style.backgroundColor='#6d28d9';"
175
- onmouseout="this.style.backgroundColor='#4c1d95';"
176
  >
177
  <img src="{github_base64}" alt="GitHub" style="height: 20px;">
178
  RAG'n'TeX
 
156
  )
157
  gr.HTML(
158
  f"""
159
+ <div style="display: flex; flex-direction: row; align-items: center; justify-content: center; gap: 20px; margin-bottom: 12px;">
160
+ <p style="font-size: 16px; line-height: 1; margin: 0;">Please ⭐️ us on GitHub!</p>
161
  <a href="https://github.com/RAGnTeX/RAGnTeX" target="_blank" style="
162
  display: inline-flex;
163
  align-items: center;
 
166
  background-color: #4c1d95;
167
  color: white;
168
  font-weight: 600;
169
+ font-size: 16px;
170
+ line-height: 1;
171
+ border-radius: 0px;
172
  text-decoration: none;
173
  gap: 8px;
174
  transition: background-color 0.2s ease-in-out;
175
+ border: 2px solid transparent;
176
  "
177
+ onmouseover="this.style.backgroundColor='#6d28d9'; this.style.border='2px solid #686868';"
178
+ onmouseout="this.style.backgroundColor='#4c1d95'; this.style.border='2px solid transparent';"
179
  >
180
  <img src="{github_base64}" alt="GitHub" style="height: 20px;">
181
  RAG'n'TeX
src/interface/upload_files.py CHANGED
@@ -1,5 +1,6 @@
1
  """Module to handle file uploads via UI."""
2
 
 
3
  import shutil
4
  from pathlib import Path
5
 
@@ -31,7 +32,10 @@ def upload_files(files: list) -> tuple[str, list[str]]:
31
  for file in files:
32
  try:
33
  temp_path = Path(file.name)
34
- file_name = temp_path.name.replace(" ", "_")
 
 
 
35
  target_path = upload_dir / file_name
36
 
37
  if target_path.exists():
@@ -53,6 +57,16 @@ def upload_files(files: list) -> tuple[str, list[str]]:
53
  messages.append(f"❌ Error uploading {file_name}: {e}")
54
 
55
  if saved_paths:
56
- ingest_files_to_db(saved_paths)
 
 
 
 
 
 
 
 
 
 
57
 
58
  return "\n".join(messages), saved_paths
 
1
  """Module to handle file uploads via UI."""
2
 
3
+ import re
4
  import shutil
5
  from pathlib import Path
6
 
 
32
  for file in files:
33
  try:
34
  temp_path = Path(file.name)
35
+ # Prepare the safe file name
36
+ safe_name = re.sub(r"[^a-zA-Z0-9_]", "_", temp_path.stem)
37
+ safe_name = re.sub(r"_+", "_", safe_name)
38
+ file_name = f"{safe_name}{temp_path.suffix}"
39
  target_path = upload_dir / file_name
40
 
41
  if target_path.exists():
 
57
  messages.append(f"❌ Error uploading {file_name}: {e}")
58
 
59
  if saved_paths:
60
+ failed = ingest_files_to_db(saved_paths)
61
+ if failed:
62
+ messages.append(
63
+ f"⚠️ Failed to process files: {', '.join(Path(f).name for f in failed)}"
64
+ )
65
+ LOGGER.warning(
66
+ "⚠️ Failed to process files: %s", ", ".join(Path(f).name for f in failed)
67
+ )
68
+ else:
69
+ messages.append("πŸ› οΈ All files processed successfully!")
70
+ LOGGER.info("πŸ› οΈ All files processed successfully!")
71
 
72
  return "\n".join(messages), saved_paths
src/processing/document_processing.py CHANGED
@@ -47,7 +47,7 @@ def extract_pdf_content(pdf_path: str) -> tuple[str, list[dict], dict]:
47
 
48
 
49
  @observe(name="πŸ“Š process_documents")
50
- def process_documents(pdf_files) -> tuple[list[str], list[dict]]:
51
  """Process a list of PDF files to extract text and images metadata.
52
  Args:
53
  pdf_files (list): List of paths to PDF files.
@@ -58,9 +58,22 @@ def process_documents(pdf_files) -> tuple[list[str], list[dict]]:
58
  """
59
  documents = []
60
  metadatas = []
 
61
 
62
  for pdf_path in pdf_files:
63
- text, imgs, metas = extract_pdf_content(pdf_path)
 
 
 
 
 
 
 
 
 
 
 
 
64
 
65
  documents.append(text)
66
 
@@ -106,7 +119,7 @@ def process_documents(pdf_files) -> tuple[list[str], list[dict]]:
106
  )
107
  metadatas.append(fixed_metadata)
108
 
109
- return documents, metadatas
110
 
111
 
112
  @observe(name="πŸ—‘οΈ delete_uploaded_files")
 
47
 
48
 
49
  @observe(name="πŸ“Š process_documents")
50
+ def process_documents(pdf_files) -> tuple[list[str], list[dict], list[str]]:
51
  """Process a list of PDF files to extract text and images metadata.
52
  Args:
53
  pdf_files (list): List of paths to PDF files.
 
58
  """
59
  documents = []
60
  metadatas = []
61
+ failed = []
62
 
63
  for pdf_path in pdf_files:
64
+ try:
65
+ text, imgs, metas = extract_pdf_content(pdf_path)
66
+ except Exception as e:
67
+ LOGGER.error("❌ Error processing %s", pdf_path, exc_info=e)
68
+ langfuse_context.update_current_observation(
69
+ output={
70
+ "output.images_passage": None,
71
+ "output.num_images": None,
72
+ "output.pdf_path": pdf_path,
73
+ }
74
+ )
75
+ failed.append(pdf_path)
76
+ continue
77
 
78
  documents.append(text)
79
 
 
119
  )
120
  metadatas.append(fixed_metadata)
121
 
122
+ return documents, metadatas, failed
123
 
124
 
125
  @observe(name="πŸ—‘οΈ delete_uploaded_files")
src/processing/images_processing.py CHANGED
@@ -106,7 +106,11 @@ def extract_images(pdf, doc, page, page_num) -> list[dict]:
106
  # Get the image ratio
107
  width = image_bbox.width
108
  height = image_bbox.height
109
- ratio = width / height if height != 0 else 2
 
 
 
 
110
 
111
  # Classify the image based on the ratio
112
  if ratio >= 1.5:
@@ -279,7 +283,7 @@ def extract_vector(pdf, page, page_num) -> list[dict]:
279
  - "hash": The MD5 hash of the figure content.
280
  """
281
  # pylint: disable=invalid-name
282
- MAX_DRAWINGS = 1000
283
  MIN_SIZE = 0.05
284
  MAX_SIZE = 0.30
285
  THRESHOLD = 5
@@ -324,7 +328,11 @@ def extract_vector(pdf, page, page_num) -> list[dict]:
324
  )
325
 
326
  # Get the figure ratio
327
- ratio = width / height if height != 0 else 2
 
 
 
 
328
 
329
  # Classify the image based on the ratio
330
  if ratio >= 1.5:
@@ -503,7 +511,6 @@ def find_used_gfx(answer, work_dir: str, metadatas: list) -> None:
503
  "img": int(match.group("img")),
504
  "hash": match.group("hash"),
505
  }
506
- print("matches_img = ", match.group("doc"))
507
  req_imgs.append(req_img)
508
 
509
  # Find figures, which are used in the presentation
@@ -519,7 +526,6 @@ def find_used_gfx(answer, work_dir: str, metadatas: list) -> None:
519
  "fig": int(match.group("fig")),
520
  "hash": match.group("hash"),
521
  }
522
- print("matches_fig = ", match.group("doc"))
523
  req_figs.append(req_fig)
524
  # span.set_attribute("output.req_figs", json.dumps(req_figs))
525
  langfuse_context.update_current_observation(
 
106
  # Get the image ratio
107
  width = image_bbox.width
108
  height = image_bbox.height
109
+ ratio = width / height if height != 0 else 20
110
+
111
+ # Remove images with weird ratios
112
+ if ratio < 0.1 or ratio > 10:
113
+ continue
114
 
115
  # Classify the image based on the ratio
116
  if ratio >= 1.5:
 
283
  - "hash": The MD5 hash of the figure content.
284
  """
285
  # pylint: disable=invalid-name
286
+ MAX_DRAWINGS = 800
287
  MIN_SIZE = 0.05
288
  MAX_SIZE = 0.30
289
  THRESHOLD = 5
 
328
  )
329
 
330
  # Get the figure ratio
331
+ ratio = width / height if height != 0 else 20
332
+
333
+ # Remove images with weird ratios
334
+ if ratio < 0.1 or ratio > 10:
335
+ continue
336
 
337
  # Classify the image based on the ratio
338
  if ratio >= 1.5:
 
511
  "img": int(match.group("img")),
512
  "hash": match.group("hash"),
513
  }
 
514
  req_imgs.append(req_img)
515
 
516
  # Find figures, which are used in the presentation
 
526
  "fig": int(match.group("fig")),
527
  "hash": match.group("hash"),
528
  }
 
529
  req_figs.append(req_fig)
530
  # span.set_attribute("output.req_figs", json.dumps(req_figs))
531
  langfuse_context.update_current_observation(