Merge branch 'dev' of https://github.com/AnnaErsh/RAGnTeX into dev
Browse files- src/__init__.py +2 -0
- src/database/ingest.py +4 -5
- src/{services/generator.py β generator.py} +9 -10
- src/interface/gradio_interface.py +1 -2
- src/interface/upload_files.py +11 -6
- src/services/__init__.py +1 -2
src/__init__.py
CHANGED
|
@@ -2,8 +2,10 @@
|
|
| 2 |
|
| 3 |
from .interface import demo
|
| 4 |
from .telemetry import init_telemetry
|
|
|
|
| 5 |
|
| 6 |
__all__ = [
|
| 7 |
"demo",
|
| 8 |
"init_telemetry",
|
|
|
|
| 9 |
]
|
|
|
|
| 2 |
|
| 3 |
from .interface import demo
|
| 4 |
from .telemetry import init_telemetry
|
| 5 |
+
from .generator import generate_presentation
|
| 6 |
|
| 7 |
__all__ = [
|
| 8 |
"demo",
|
| 9 |
"init_telemetry",
|
| 10 |
+
"generate_presentation",
|
| 11 |
]
|
src/database/ingest.py
CHANGED
|
@@ -19,13 +19,13 @@ def ingest_files_to_db(pdf_files) -> None:
|
|
| 19 |
# Get all the existing entries and extract the known filenames
|
| 20 |
all_entries = db.get(include=["metadatas"])
|
| 21 |
existing_pdfs = set(
|
| 22 |
-
meta.get("
|
| 23 |
for meta in all_entries["metadatas"]
|
| 24 |
-
if meta.get("
|
| 25 |
)
|
| 26 |
|
| 27 |
# Filter out already ingested PDFs
|
| 28 |
-
new_pdfs = [file for file in pdf_files if
|
| 29 |
|
| 30 |
if not new_pdfs:
|
| 31 |
LOGGER.info("π No new PDFs to ingest. Skipping ingestion.")
|
|
@@ -33,9 +33,8 @@ def ingest_files_to_db(pdf_files) -> None:
|
|
| 33 |
|
| 34 |
# Process new files only
|
| 35 |
documents, metadatas = process_documents(new_pdfs)
|
| 36 |
-
# print("ingest metadatas", metadatas)
|
| 37 |
|
| 38 |
-
LOGGER.info("β Adding %
|
| 39 |
if documents:
|
| 40 |
db.add(
|
| 41 |
documents=documents,
|
|
|
|
| 19 |
# Get all the existing entries and extract the known filenames
|
| 20 |
all_entries = db.get(include=["metadatas"])
|
| 21 |
existing_pdfs = set(
|
| 22 |
+
meta.get("pdf_path")
|
| 23 |
for meta in all_entries["metadatas"]
|
| 24 |
+
if meta.get("pdf_path")
|
| 25 |
)
|
| 26 |
|
| 27 |
# Filter out already ingested PDFs
|
| 28 |
+
new_pdfs = [file for file in pdf_files if file not in existing_pdfs]
|
| 29 |
|
| 30 |
if not new_pdfs:
|
| 31 |
LOGGER.info("π No new PDFs to ingest. Skipping ingestion.")
|
|
|
|
| 33 |
|
| 34 |
# Process new files only
|
| 35 |
documents, metadatas = process_documents(new_pdfs)
|
|
|
|
| 36 |
|
| 37 |
+
LOGGER.info("β Adding %d new PDFs to the database...", len(new_pdfs))
|
| 38 |
if documents:
|
| 39 |
db.add(
|
| 40 |
documents=documents,
|
src/{services/generator.py β generator.py}
RENAMED
|
@@ -3,26 +3,26 @@
|
|
| 3 |
from pathlib import Path
|
| 4 |
from langfuse.decorators import langfuse_context, observe
|
| 5 |
|
| 6 |
-
from
|
| 7 |
-
from .
|
| 8 |
-
from .
|
| 9 |
-
from . import
|
| 10 |
-
from .
|
| 11 |
-
from .
|
|
|
|
| 12 |
|
| 13 |
|
| 14 |
LOGGER = Logger.get_logger()
|
| 15 |
|
| 16 |
|
| 17 |
@observe(name="π§βπ¨ generate_presentation")
|
| 18 |
-
def generate_presentation(theme, color, topic
|
| 19 |
"""Main function to generate a presentation based on the provided theme, color, and topic.
|
| 20 |
Args:
|
| 21 |
theme (str): The theme of the presentation, chosen by user via UI, defaults to "default".
|
| 22 |
color (str): The color scheme for the presentation, chosen by user via UI,
|
| 23 |
defaults to "default".
|
| 24 |
topic (str): The topic of the presentation, set by user via UI, defaults to "default".
|
| 25 |
-
uploaded_files (list): List of file paths to be ingested, provided by user via UI.
|
| 26 |
Returns:
|
| 27 |
tuple: A 3-element tuple:
|
| 28 |
- str: Success message.
|
|
@@ -31,7 +31,6 @@ def generate_presentation(theme, color, topic, uploaded_files) -> tuple[str, str
|
|
| 31 |
"""
|
| 32 |
|
| 33 |
trace_id = langfuse_context.get_current_trace_id()
|
| 34 |
-
ingest_files_to_db(uploaded_files)
|
| 35 |
|
| 36 |
[documents], [metadatas] = retrive_files_from_db(topic)
|
| 37 |
prompt = get_prompt(theme, color)
|
|
@@ -60,6 +59,6 @@ def generate_presentation(theme, color, topic, uploaded_files) -> tuple[str, str
|
|
| 60 |
find_used_gfx(answer, work_dir, metadatas)
|
| 61 |
compile_presentation(answer.text, work_dir)
|
| 62 |
|
| 63 |
-
delete_uploaded_files(uploaded_files)
|
| 64 |
|
| 65 |
return "Presentation generated successfully!", trace_id, work_dir
|
|
|
|
| 3 |
from pathlib import Path
|
| 4 |
from langfuse.decorators import langfuse_context, observe
|
| 5 |
|
| 6 |
+
# from datetime import datetime
|
| 7 |
+
from src.database import retrive_files_from_db
|
| 8 |
+
from src.telemetry import Logger
|
| 9 |
+
from src.processing import get_prompt, create_output_folder
|
| 10 |
+
from src.services import client
|
| 11 |
+
from src.compilation import compile_presentation
|
| 12 |
+
from src.processing import delete_uploaded_files, find_used_gfx
|
| 13 |
|
| 14 |
|
| 15 |
LOGGER = Logger.get_logger()
|
| 16 |
|
| 17 |
|
| 18 |
@observe(name="π§βπ¨ generate_presentation")
|
| 19 |
+
def generate_presentation(theme, color, topic) -> tuple[str, str, str]:
|
| 20 |
"""Main function to generate a presentation based on the provided theme, color, and topic.
|
| 21 |
Args:
|
| 22 |
theme (str): The theme of the presentation, chosen by user via UI, defaults to "default".
|
| 23 |
color (str): The color scheme for the presentation, chosen by user via UI,
|
| 24 |
defaults to "default".
|
| 25 |
topic (str): The topic of the presentation, set by user via UI, defaults to "default".
|
|
|
|
| 26 |
Returns:
|
| 27 |
tuple: A 3-element tuple:
|
| 28 |
- str: Success message.
|
|
|
|
| 31 |
"""
|
| 32 |
|
| 33 |
trace_id = langfuse_context.get_current_trace_id()
|
|
|
|
| 34 |
|
| 35 |
[documents], [metadatas] = retrive_files_from_db(topic)
|
| 36 |
prompt = get_prompt(theme, color)
|
|
|
|
| 59 |
find_used_gfx(answer, work_dir, metadatas)
|
| 60 |
compile_presentation(answer.text, work_dir)
|
| 61 |
|
| 62 |
+
# delete_uploaded_files(uploaded_files)
|
| 63 |
|
| 64 |
return "Presentation generated successfully!", trace_id, work_dir
|
src/interface/gradio_interface.py
CHANGED
|
@@ -3,7 +3,7 @@
|
|
| 3 |
import gradio as gr
|
| 4 |
from .upload_files import upload_files
|
| 5 |
from .download_files import download_files
|
| 6 |
-
from ..
|
| 7 |
from ..telemetry import submit_feedback
|
| 8 |
import base64
|
| 9 |
|
|
@@ -321,7 +321,6 @@ with gr.Blocks(theme=theme, js=js_func) as demo:
|
|
| 321 |
presentation_theme_state,
|
| 322 |
color_theme_state,
|
| 323 |
topic_input,
|
| 324 |
-
uploaded_files_state,
|
| 325 |
],
|
| 326 |
outputs=[compilation_status, trace_id_state, presentation_folder_state],
|
| 327 |
)
|
|
|
|
| 3 |
import gradio as gr
|
| 4 |
from .upload_files import upload_files
|
| 5 |
from .download_files import download_files
|
| 6 |
+
from ..generator import generate_presentation
|
| 7 |
from ..telemetry import submit_feedback
|
| 8 |
import base64
|
| 9 |
|
|
|
|
| 321 |
presentation_theme_state,
|
| 322 |
color_theme_state,
|
| 323 |
topic_input,
|
|
|
|
| 324 |
],
|
| 325 |
outputs=[compilation_status, trace_id_state, presentation_folder_state],
|
| 326 |
)
|
src/interface/upload_files.py
CHANGED
|
@@ -3,6 +3,7 @@
|
|
| 3 |
from pathlib import Path
|
| 4 |
import shutil
|
| 5 |
from ..telemetry.logging_utils import Logger
|
|
|
|
| 6 |
|
| 7 |
LOGGER = Logger.get_logger()
|
| 8 |
|
|
@@ -16,7 +17,7 @@ def upload_files(files: list) -> tuple[str, list[str]]:
|
|
| 16 |
list[str]: List of saved file paths.
|
| 17 |
"""
|
| 18 |
if not files:
|
| 19 |
-
return "No files selected
|
| 20 |
|
| 21 |
current_file = Path(__file__).resolve()
|
| 22 |
project_root = current_file.parents[2] # utils β src β PROJECT ROOT
|
|
@@ -32,15 +33,16 @@ def upload_files(files: list) -> tuple[str, list[str]]:
|
|
| 32 |
file_name = temp_path.name.replace(" ", "_")
|
| 33 |
target_path = UPLOAD_DIR / file_name
|
| 34 |
|
| 35 |
-
if
|
| 36 |
-
messages.append(f"
|
| 37 |
continue
|
| 38 |
|
| 39 |
-
if
|
| 40 |
-
messages.append(f"
|
|
|
|
| 41 |
else:
|
| 42 |
shutil.move(str(temp_path), str(target_path))
|
| 43 |
-
messages.append(f"β
Uploaded: {file_name}")
|
| 44 |
LOGGER.info("π€ Uploaded file %s to %s", file_name, UPLOAD_DIR)
|
| 45 |
|
| 46 |
saved_paths.append(str(target_path))
|
|
@@ -49,4 +51,7 @@ def upload_files(files: list) -> tuple[str, list[str]]:
|
|
| 49 |
LOGGER.error("β Error uploading %s", file_name, exc_info=e)
|
| 50 |
messages.append(f"β Error uploading {file_name}: {e}")
|
| 51 |
|
|
|
|
|
|
|
|
|
|
| 52 |
return "\n".join(messages), saved_paths
|
|
|
|
| 3 |
from pathlib import Path
|
| 4 |
import shutil
|
| 5 |
from ..telemetry.logging_utils import Logger
|
| 6 |
+
from ..database import ingest_files_to_db
|
| 7 |
|
| 8 |
LOGGER = Logger.get_logger()
|
| 9 |
|
|
|
|
| 17 |
list[str]: List of saved file paths.
|
| 18 |
"""
|
| 19 |
if not files:
|
| 20 |
+
return "β No new files selected", []
|
| 21 |
|
| 22 |
current_file = Path(__file__).resolve()
|
| 23 |
project_root = current_file.parents[2] # utils β src β PROJECT ROOT
|
|
|
|
| 33 |
file_name = temp_path.name.replace(" ", "_")
|
| 34 |
target_path = UPLOAD_DIR / file_name
|
| 35 |
|
| 36 |
+
if target_path.exists():
|
| 37 |
+
messages.append(f"β οΈ File already exists: {file_name}")
|
| 38 |
continue
|
| 39 |
|
| 40 |
+
if not temp_path.exists():
|
| 41 |
+
messages.append(f"β File not found: {file_name}")
|
| 42 |
+
continue
|
| 43 |
else:
|
| 44 |
shutil.move(str(temp_path), str(target_path))
|
| 45 |
+
messages.append(f"β
Uploaded new file: {file_name}")
|
| 46 |
LOGGER.info("π€ Uploaded file %s to %s", file_name, UPLOAD_DIR)
|
| 47 |
|
| 48 |
saved_paths.append(str(target_path))
|
|
|
|
| 51 |
LOGGER.error("β Error uploading %s", file_name, exc_info=e)
|
| 52 |
messages.append(f"β Error uploading {file_name}: {e}")
|
| 53 |
|
| 54 |
+
if saved_paths:
|
| 55 |
+
ingest_files_to_db(saved_paths)
|
| 56 |
+
|
| 57 |
return "\n".join(messages), saved_paths
|
src/services/__init__.py
CHANGED
|
@@ -1,6 +1,5 @@
|
|
| 1 |
"""This module initializes the services package."""
|
| 2 |
|
| 3 |
from .google_client import client
|
| 4 |
-
from .generator import generate_presentation
|
| 5 |
|
| 6 |
-
__all__ = ["client"
|
|
|
|
| 1 |
"""This module initializes the services package."""
|
| 2 |
|
| 3 |
from .google_client import client
|
|
|
|
| 4 |
|
| 5 |
+
__all__ = ["client"]
|