Merge branch 'dev' of https://github.com/AnnaErsh/RAGnTeX into dev
Browse files
src/database/db_manipulation.py
CHANGED
|
@@ -12,7 +12,7 @@ LOGGER = Logger.get_logger()
|
|
| 12 |
|
| 13 |
|
| 14 |
@observe(name="π ingest_files_to_db")
|
| 15 |
-
def ingest_files_to_db(pdf_files) ->
|
| 16 |
"""Add new files to the database if they are not yet there.
|
| 17 |
Args:
|
| 18 |
pdf_files (list): List of PDF files to be ingested.
|
|
@@ -37,7 +37,7 @@ def ingest_files_to_db(pdf_files) -> None:
|
|
| 37 |
return
|
| 38 |
|
| 39 |
# Process new files only
|
| 40 |
-
documents, metadatas = process_documents(new_pdfs)
|
| 41 |
|
| 42 |
LOGGER.info("β Adding %d new PDFs to the database...", len(new_pdfs))
|
| 43 |
if documents:
|
|
@@ -48,6 +48,8 @@ def ingest_files_to_db(pdf_files) -> None:
|
|
| 48 |
)
|
| 49 |
LOGGER.info("β
Documents ingested successfully.")
|
| 50 |
|
|
|
|
|
|
|
| 51 |
|
| 52 |
Metadata = Mapping[str, Union[str, int, float, bool]]
|
| 53 |
DocumentsBatch = List[List[str]]
|
|
|
|
| 12 |
|
| 13 |
|
| 14 |
@observe(name="π ingest_files_to_db")
|
| 15 |
+
def ingest_files_to_db(pdf_files) -> list[str]:
|
| 16 |
"""Add new files to the database if they are not yet there.
|
| 17 |
Args:
|
| 18 |
pdf_files (list): List of PDF files to be ingested.
|
|
|
|
| 37 |
return
|
| 38 |
|
| 39 |
# Process new files only
|
| 40 |
+
documents, metadatas, failed = process_documents(new_pdfs)
|
| 41 |
|
| 42 |
LOGGER.info("β Adding %d new PDFs to the database...", len(new_pdfs))
|
| 43 |
if documents:
|
|
|
|
| 48 |
)
|
| 49 |
LOGGER.info("β
Documents ingested successfully.")
|
| 50 |
|
| 51 |
+
return failed
|
| 52 |
+
|
| 53 |
|
| 54 |
Metadata = Mapping[str, Union[str, int, float, bool]]
|
| 55 |
DocumentsBatch = List[List[str]]
|
src/interface/gradio_interface.py
CHANGED
|
@@ -156,8 +156,8 @@ with gr.Blocks(theme=theme, js=JS_FUNC) as demo:
|
|
| 156 |
)
|
| 157 |
gr.HTML(
|
| 158 |
f"""
|
| 159 |
-
<div style="display: flex; flex-direction: row; align-items: center; justify-content: center; gap:
|
| 160 |
-
<p style="font-size: 16px;">Please βοΈ us on GitHub!</p>
|
| 161 |
<a href="https://github.com/RAGnTeX/RAGnTeX" target="_blank" style="
|
| 162 |
display: inline-flex;
|
| 163 |
align-items: center;
|
|
@@ -166,13 +166,16 @@ with gr.Blocks(theme=theme, js=JS_FUNC) as demo:
|
|
| 166 |
background-color: #4c1d95;
|
| 167 |
color: white;
|
| 168 |
font-weight: 600;
|
| 169 |
-
|
|
|
|
|
|
|
| 170 |
text-decoration: none;
|
| 171 |
gap: 8px;
|
| 172 |
transition: background-color 0.2s ease-in-out;
|
|
|
|
| 173 |
"
|
| 174 |
-
onmouseover="this.style.backgroundColor='#6d28d9';"
|
| 175 |
-
onmouseout="this.style.backgroundColor='#4c1d95';"
|
| 176 |
>
|
| 177 |
<img src="{github_base64}" alt="GitHub" style="height: 20px;">
|
| 178 |
RAG'n'TeX
|
|
|
|
| 156 |
)
|
| 157 |
gr.HTML(
|
| 158 |
f"""
|
| 159 |
+
<div style="display: flex; flex-direction: row; align-items: center; justify-content: center; gap: 20px; margin-bottom: 12px;">
|
| 160 |
+
<p style="font-size: 16px; line-height: 1; margin: 0;">Please βοΈ us on GitHub!</p>
|
| 161 |
<a href="https://github.com/RAGnTeX/RAGnTeX" target="_blank" style="
|
| 162 |
display: inline-flex;
|
| 163 |
align-items: center;
|
|
|
|
| 166 |
background-color: #4c1d95;
|
| 167 |
color: white;
|
| 168 |
font-weight: 600;
|
| 169 |
+
font-size: 16px;
|
| 170 |
+
line-height: 1;
|
| 171 |
+
border-radius: 0px;
|
| 172 |
text-decoration: none;
|
| 173 |
gap: 8px;
|
| 174 |
transition: background-color 0.2s ease-in-out;
|
| 175 |
+
border: 2px solid transparent;
|
| 176 |
"
|
| 177 |
+
onmouseover="this.style.backgroundColor='#6d28d9'; this.style.border='2px solid #686868';"
|
| 178 |
+
onmouseout="this.style.backgroundColor='#4c1d95'; this.style.border='2px solid transparent';"
|
| 179 |
>
|
| 180 |
<img src="{github_base64}" alt="GitHub" style="height: 20px;">
|
| 181 |
RAG'n'TeX
|
src/interface/upload_files.py
CHANGED
|
@@ -1,5 +1,6 @@
|
|
| 1 |
"""Module to handle file uploads via UI."""
|
| 2 |
|
|
|
|
| 3 |
import shutil
|
| 4 |
from pathlib import Path
|
| 5 |
|
|
@@ -31,7 +32,10 @@ def upload_files(files: list) -> tuple[str, list[str]]:
|
|
| 31 |
for file in files:
|
| 32 |
try:
|
| 33 |
temp_path = Path(file.name)
|
| 34 |
-
|
|
|
|
|
|
|
|
|
|
| 35 |
target_path = upload_dir / file_name
|
| 36 |
|
| 37 |
if target_path.exists():
|
|
@@ -53,6 +57,16 @@ def upload_files(files: list) -> tuple[str, list[str]]:
|
|
| 53 |
messages.append(f"β Error uploading {file_name}: {e}")
|
| 54 |
|
| 55 |
if saved_paths:
|
| 56 |
-
ingest_files_to_db(saved_paths)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 57 |
|
| 58 |
return "\n".join(messages), saved_paths
|
|
|
|
| 1 |
"""Module to handle file uploads via UI."""
|
| 2 |
|
| 3 |
+
import re
|
| 4 |
import shutil
|
| 5 |
from pathlib import Path
|
| 6 |
|
|
|
|
| 32 |
for file in files:
|
| 33 |
try:
|
| 34 |
temp_path = Path(file.name)
|
| 35 |
+
# Prepare the safe file name
|
| 36 |
+
safe_name = re.sub(r"[^a-zA-Z0-9_]", "_", temp_path.stem)
|
| 37 |
+
safe_name = re.sub(r"_+", "_", safe_name)
|
| 38 |
+
file_name = f"{safe_name}{temp_path.suffix}"
|
| 39 |
target_path = upload_dir / file_name
|
| 40 |
|
| 41 |
if target_path.exists():
|
|
|
|
| 57 |
messages.append(f"β Error uploading {file_name}: {e}")
|
| 58 |
|
| 59 |
if saved_paths:
|
| 60 |
+
failed = ingest_files_to_db(saved_paths)
|
| 61 |
+
if failed:
|
| 62 |
+
messages.append(
|
| 63 |
+
f"β οΈ Failed to process files: {', '.join(Path(f).name for f in failed)}"
|
| 64 |
+
)
|
| 65 |
+
LOGGER.warning(
|
| 66 |
+
"β οΈ Failed to process files: %s", ", ".join(Path(f).name for f in failed)
|
| 67 |
+
)
|
| 68 |
+
else:
|
| 69 |
+
messages.append("π οΈ All files processed successfully!")
|
| 70 |
+
LOGGER.info("π οΈ All files processed successfully!")
|
| 71 |
|
| 72 |
return "\n".join(messages), saved_paths
|
src/processing/document_processing.py
CHANGED
|
@@ -47,7 +47,7 @@ def extract_pdf_content(pdf_path: str) -> tuple[str, list[dict], dict]:
|
|
| 47 |
|
| 48 |
|
| 49 |
@observe(name="π process_documents")
|
| 50 |
-
def process_documents(pdf_files) -> tuple[list[str], list[dict]]:
|
| 51 |
"""Process a list of PDF files to extract text and images metadata.
|
| 52 |
Args:
|
| 53 |
pdf_files (list): List of paths to PDF files.
|
|
@@ -58,9 +58,22 @@ def process_documents(pdf_files) -> tuple[list[str], list[dict]]:
|
|
| 58 |
"""
|
| 59 |
documents = []
|
| 60 |
metadatas = []
|
|
|
|
| 61 |
|
| 62 |
for pdf_path in pdf_files:
|
| 63 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 64 |
|
| 65 |
documents.append(text)
|
| 66 |
|
|
@@ -106,7 +119,7 @@ def process_documents(pdf_files) -> tuple[list[str], list[dict]]:
|
|
| 106 |
)
|
| 107 |
metadatas.append(fixed_metadata)
|
| 108 |
|
| 109 |
-
return documents, metadatas
|
| 110 |
|
| 111 |
|
| 112 |
@observe(name="ποΈ delete_uploaded_files")
|
|
|
|
| 47 |
|
| 48 |
|
| 49 |
@observe(name="π process_documents")
|
| 50 |
+
def process_documents(pdf_files) -> tuple[list[str], list[dict], list[str]]:
|
| 51 |
"""Process a list of PDF files to extract text and images metadata.
|
| 52 |
Args:
|
| 53 |
pdf_files (list): List of paths to PDF files.
|
|
|
|
| 58 |
"""
|
| 59 |
documents = []
|
| 60 |
metadatas = []
|
| 61 |
+
failed = []
|
| 62 |
|
| 63 |
for pdf_path in pdf_files:
|
| 64 |
+
try:
|
| 65 |
+
text, imgs, metas = extract_pdf_content(pdf_path)
|
| 66 |
+
except Exception as e:
|
| 67 |
+
LOGGER.error("β Error processing %s", pdf_path, exc_info=e)
|
| 68 |
+
langfuse_context.update_current_observation(
|
| 69 |
+
output={
|
| 70 |
+
"output.images_passage": None,
|
| 71 |
+
"output.num_images": None,
|
| 72 |
+
"output.pdf_path": pdf_path,
|
| 73 |
+
}
|
| 74 |
+
)
|
| 75 |
+
failed.append(pdf_path)
|
| 76 |
+
continue
|
| 77 |
|
| 78 |
documents.append(text)
|
| 79 |
|
|
|
|
| 119 |
)
|
| 120 |
metadatas.append(fixed_metadata)
|
| 121 |
|
| 122 |
+
return documents, metadatas, failed
|
| 123 |
|
| 124 |
|
| 125 |
@observe(name="ποΈ delete_uploaded_files")
|
src/processing/images_processing.py
CHANGED
|
@@ -106,7 +106,11 @@ def extract_images(pdf, doc, page, page_num) -> list[dict]:
|
|
| 106 |
# Get the image ratio
|
| 107 |
width = image_bbox.width
|
| 108 |
height = image_bbox.height
|
| 109 |
-
ratio = width / height if height != 0 else
|
|
|
|
|
|
|
|
|
|
|
|
|
| 110 |
|
| 111 |
# Classify the image based on the ratio
|
| 112 |
if ratio >= 1.5:
|
|
@@ -279,7 +283,7 @@ def extract_vector(pdf, page, page_num) -> list[dict]:
|
|
| 279 |
- "hash": The MD5 hash of the figure content.
|
| 280 |
"""
|
| 281 |
# pylint: disable=invalid-name
|
| 282 |
-
MAX_DRAWINGS =
|
| 283 |
MIN_SIZE = 0.05
|
| 284 |
MAX_SIZE = 0.30
|
| 285 |
THRESHOLD = 5
|
|
@@ -324,7 +328,11 @@ def extract_vector(pdf, page, page_num) -> list[dict]:
|
|
| 324 |
)
|
| 325 |
|
| 326 |
# Get the figure ratio
|
| 327 |
-
ratio = width / height if height != 0 else
|
|
|
|
|
|
|
|
|
|
|
|
|
| 328 |
|
| 329 |
# Classify the image based on the ratio
|
| 330 |
if ratio >= 1.5:
|
|
@@ -503,7 +511,6 @@ def find_used_gfx(answer, work_dir: str, metadatas: list) -> None:
|
|
| 503 |
"img": int(match.group("img")),
|
| 504 |
"hash": match.group("hash"),
|
| 505 |
}
|
| 506 |
-
print("matches_img = ", match.group("doc"))
|
| 507 |
req_imgs.append(req_img)
|
| 508 |
|
| 509 |
# Find figures, which are used in the presentation
|
|
@@ -519,7 +526,6 @@ def find_used_gfx(answer, work_dir: str, metadatas: list) -> None:
|
|
| 519 |
"fig": int(match.group("fig")),
|
| 520 |
"hash": match.group("hash"),
|
| 521 |
}
|
| 522 |
-
print("matches_fig = ", match.group("doc"))
|
| 523 |
req_figs.append(req_fig)
|
| 524 |
# span.set_attribute("output.req_figs", json.dumps(req_figs))
|
| 525 |
langfuse_context.update_current_observation(
|
|
|
|
| 106 |
# Get the image ratio
|
| 107 |
width = image_bbox.width
|
| 108 |
height = image_bbox.height
|
| 109 |
+
ratio = width / height if height != 0 else 20
|
| 110 |
+
|
| 111 |
+
# Remove images with weird ratios
|
| 112 |
+
if ratio < 0.1 or ratio > 10:
|
| 113 |
+
continue
|
| 114 |
|
| 115 |
# Classify the image based on the ratio
|
| 116 |
if ratio >= 1.5:
|
|
|
|
| 283 |
- "hash": The MD5 hash of the figure content.
|
| 284 |
"""
|
| 285 |
# pylint: disable=invalid-name
|
| 286 |
+
MAX_DRAWINGS = 800
|
| 287 |
MIN_SIZE = 0.05
|
| 288 |
MAX_SIZE = 0.30
|
| 289 |
THRESHOLD = 5
|
|
|
|
| 328 |
)
|
| 329 |
|
| 330 |
# Get the figure ratio
|
| 331 |
+
ratio = width / height if height != 0 else 20
|
| 332 |
+
|
| 333 |
+
# Remove images with weird ratios
|
| 334 |
+
if ratio < 0.1 or ratio > 10:
|
| 335 |
+
continue
|
| 336 |
|
| 337 |
# Classify the image based on the ratio
|
| 338 |
if ratio >= 1.5:
|
|
|
|
| 511 |
"img": int(match.group("img")),
|
| 512 |
"hash": match.group("hash"),
|
| 513 |
}
|
|
|
|
| 514 |
req_imgs.append(req_img)
|
| 515 |
|
| 516 |
# Find figures, which are used in the presentation
|
|
|
|
| 526 |
"fig": int(match.group("fig")),
|
| 527 |
"hash": match.group("hash"),
|
| 528 |
}
|
|
|
|
| 529 |
req_figs.append(req_fig)
|
| 530 |
# span.set_attribute("output.req_figs", json.dumps(req_figs))
|
| 531 |
langfuse_context.update_current_observation(
|