eranmi commited on
Commit
35caa3e
Β·
2 Parent(s): 9459739f402198

Merge branch 'dev' of https://github.com/AnnaErsh/RAGnTeX into dev

Browse files
src/__init__.py CHANGED
@@ -2,8 +2,10 @@
2
 
3
  from .interface import demo
4
  from .telemetry import init_telemetry
 
5
 
6
  __all__ = [
7
  "demo",
8
  "init_telemetry",
 
9
  ]
 
2
 
3
  from .interface import demo
4
  from .telemetry import init_telemetry
5
+ from .generator import generate_presentation
6
 
7
  __all__ = [
8
  "demo",
9
  "init_telemetry",
10
+ "generate_presentation",
11
  ]
src/database/ingest.py CHANGED
@@ -19,13 +19,13 @@ def ingest_files_to_db(pdf_files) -> None:
19
  # Get all the existing entries and extract the known filenames
20
  all_entries = db.get(include=["metadatas"])
21
  existing_pdfs = set(
22
- meta.get("source_pdf")
23
  for meta in all_entries["metadatas"]
24
- if meta.get("source_pdf")
25
  )
26
 
27
  # Filter out already ingested PDFs
28
- new_pdfs = [file for file in pdf_files if Path(file).name not in existing_pdfs]
29
 
30
  if not new_pdfs:
31
  LOGGER.info("πŸ“‚ No new PDFs to ingest. Skipping ingestion.")
@@ -33,9 +33,8 @@ def ingest_files_to_db(pdf_files) -> None:
33
 
34
  # Process new files only
35
  documents, metadatas = process_documents(new_pdfs)
36
- # print("ingest metadatas", metadatas)
37
 
38
- LOGGER.info("βž• Adding %s new PDFs to the database...", len(new_pdfs))
39
  if documents:
40
  db.add(
41
  documents=documents,
 
19
  # Get all the existing entries and extract the known filenames
20
  all_entries = db.get(include=["metadatas"])
21
  existing_pdfs = set(
22
+ meta.get("pdf_path")
23
  for meta in all_entries["metadatas"]
24
+ if meta.get("pdf_path")
25
  )
26
 
27
  # Filter out already ingested PDFs
28
+ new_pdfs = [file for file in pdf_files if file not in existing_pdfs]
29
 
30
  if not new_pdfs:
31
  LOGGER.info("πŸ“‚ No new PDFs to ingest. Skipping ingestion.")
 
33
 
34
  # Process new files only
35
  documents, metadatas = process_documents(new_pdfs)
 
36
 
37
+ LOGGER.info("βž• Adding %d new PDFs to the database...", len(new_pdfs))
38
  if documents:
39
  db.add(
40
  documents=documents,
src/{services/generator.py β†’ generator.py} RENAMED
@@ -3,26 +3,26 @@
3
  from pathlib import Path
4
  from langfuse.decorators import langfuse_context, observe
5
 
6
- from ..database import ingest_files_to_db, retrive_files_from_db
7
- from ..telemetry import Logger
8
- from ..processing import get_prompt, create_output_folder
9
- from . import client
10
- from ..compilation import compile_presentation
11
- from ..processing import delete_uploaded_files, find_used_gfx
 
12
 
13
 
14
  LOGGER = Logger.get_logger()
15
 
16
 
17
  @observe(name="πŸ§‘β€πŸŽ¨ generate_presentation")
18
- def generate_presentation(theme, color, topic, uploaded_files) -> tuple[str, str, str]:
19
  """Main function to generate a presentation based on the provided theme, color, and topic.
20
  Args:
21
  theme (str): The theme of the presentation, chosen by user via UI, defaults to "default".
22
  color (str): The color scheme for the presentation, chosen by user via UI,
23
  defaults to "default".
24
  topic (str): The topic of the presentation, set by user via UI, defaults to "default".
25
- uploaded_files (list): List of file paths to be ingested, provided by user via UI.
26
  Returns:
27
  tuple: A 3-element tuple:
28
  - str: Success message.
@@ -31,7 +31,6 @@ def generate_presentation(theme, color, topic, uploaded_files) -> tuple[str, str
31
  """
32
 
33
  trace_id = langfuse_context.get_current_trace_id()
34
- ingest_files_to_db(uploaded_files)
35
 
36
  [documents], [metadatas] = retrive_files_from_db(topic)
37
  prompt = get_prompt(theme, color)
@@ -60,6 +59,6 @@ def generate_presentation(theme, color, topic, uploaded_files) -> tuple[str, str
60
  find_used_gfx(answer, work_dir, metadatas)
61
  compile_presentation(answer.text, work_dir)
62
 
63
- delete_uploaded_files(uploaded_files)
64
 
65
  return "Presentation generated successfully!", trace_id, work_dir
 
3
  from pathlib import Path
4
  from langfuse.decorators import langfuse_context, observe
5
 
6
+ # from datetime import datetime
7
+ from src.database import retrive_files_from_db
8
+ from src.telemetry import Logger
9
+ from src.processing import get_prompt, create_output_folder
10
+ from src.services import client
11
+ from src.compilation import compile_presentation
12
+ from src.processing import delete_uploaded_files, find_used_gfx
13
 
14
 
15
  LOGGER = Logger.get_logger()
16
 
17
 
18
  @observe(name="πŸ§‘β€πŸŽ¨ generate_presentation")
19
+ def generate_presentation(theme, color, topic) -> tuple[str, str, str]:
20
  """Main function to generate a presentation based on the provided theme, color, and topic.
21
  Args:
22
  theme (str): The theme of the presentation, chosen by user via UI, defaults to "default".
23
  color (str): The color scheme for the presentation, chosen by user via UI,
24
  defaults to "default".
25
  topic (str): The topic of the presentation, set by user via UI, defaults to "default".
 
26
  Returns:
27
  tuple: A 3-element tuple:
28
  - str: Success message.
 
31
  """
32
 
33
  trace_id = langfuse_context.get_current_trace_id()
 
34
 
35
  [documents], [metadatas] = retrive_files_from_db(topic)
36
  prompt = get_prompt(theme, color)
 
59
  find_used_gfx(answer, work_dir, metadatas)
60
  compile_presentation(answer.text, work_dir)
61
 
62
+ # delete_uploaded_files(uploaded_files)
63
 
64
  return "Presentation generated successfully!", trace_id, work_dir
src/interface/gradio_interface.py CHANGED
@@ -3,7 +3,7 @@
3
  import gradio as gr
4
  from .upload_files import upload_files
5
  from .download_files import download_files
6
- from ..services import generate_presentation
7
  from ..telemetry import submit_feedback
8
  import base64
9
 
@@ -321,7 +321,6 @@ with gr.Blocks(theme=theme, js=js_func) as demo:
321
  presentation_theme_state,
322
  color_theme_state,
323
  topic_input,
324
- uploaded_files_state,
325
  ],
326
  outputs=[compilation_status, trace_id_state, presentation_folder_state],
327
  )
 
3
  import gradio as gr
4
  from .upload_files import upload_files
5
  from .download_files import download_files
6
+ from ..generator import generate_presentation
7
  from ..telemetry import submit_feedback
8
  import base64
9
 
 
321
  presentation_theme_state,
322
  color_theme_state,
323
  topic_input,
 
324
  ],
325
  outputs=[compilation_status, trace_id_state, presentation_folder_state],
326
  )
src/interface/upload_files.py CHANGED
@@ -3,6 +3,7 @@
3
  from pathlib import Path
4
  import shutil
5
  from ..telemetry.logging_utils import Logger
 
6
 
7
  LOGGER = Logger.get_logger()
8
 
@@ -16,7 +17,7 @@ def upload_files(files: list) -> tuple[str, list[str]]:
16
  list[str]: List of saved file paths.
17
  """
18
  if not files:
19
- return "No files selected."
20
 
21
  current_file = Path(__file__).resolve()
22
  project_root = current_file.parents[2] # utils β†’ src β†’ PROJECT ROOT
@@ -32,15 +33,16 @@ def upload_files(files: list) -> tuple[str, list[str]]:
32
  file_name = temp_path.name.replace(" ", "_")
33
  target_path = UPLOAD_DIR / file_name
34
 
35
- if not temp_path.exists():
36
- messages.append(f"❌ File not found: {file_name} (already uploaded?)")
37
  continue
38
 
39
- if target_path.exists():
40
- messages.append(f"⚠️ File already exists: {file_name} β€” skipped.")
 
41
  else:
42
  shutil.move(str(temp_path), str(target_path))
43
- messages.append(f"βœ… Uploaded: {file_name}")
44
  LOGGER.info("πŸ“€ Uploaded file %s to %s", file_name, UPLOAD_DIR)
45
 
46
  saved_paths.append(str(target_path))
@@ -49,4 +51,7 @@ def upload_files(files: list) -> tuple[str, list[str]]:
49
  LOGGER.error("❌ Error uploading %s", file_name, exc_info=e)
50
  messages.append(f"❌ Error uploading {file_name}: {e}")
51
 
 
 
 
52
  return "\n".join(messages), saved_paths
 
3
  from pathlib import Path
4
  import shutil
5
  from ..telemetry.logging_utils import Logger
6
+ from ..database import ingest_files_to_db
7
 
8
  LOGGER = Logger.get_logger()
9
 
 
17
  list[str]: List of saved file paths.
18
  """
19
  if not files:
20
+ return "❌ No new files selected", []
21
 
22
  current_file = Path(__file__).resolve()
23
  project_root = current_file.parents[2] # utils β†’ src β†’ PROJECT ROOT
 
33
  file_name = temp_path.name.replace(" ", "_")
34
  target_path = UPLOAD_DIR / file_name
35
 
36
+ if target_path.exists():
37
+ messages.append(f"⚠️ File already exists: {file_name}")
38
  continue
39
 
40
+ if not temp_path.exists():
41
+ messages.append(f"❌ File not found: {file_name}")
42
+ continue
43
  else:
44
  shutil.move(str(temp_path), str(target_path))
45
+ messages.append(f"βœ… Uploaded new file: {file_name}")
46
  LOGGER.info("πŸ“€ Uploaded file %s to %s", file_name, UPLOAD_DIR)
47
 
48
  saved_paths.append(str(target_path))
 
51
  LOGGER.error("❌ Error uploading %s", file_name, exc_info=e)
52
  messages.append(f"❌ Error uploading {file_name}: {e}")
53
 
54
+ if saved_paths:
55
+ ingest_files_to_db(saved_paths)
56
+
57
  return "\n".join(messages), saved_paths
src/services/__init__.py CHANGED
@@ -1,6 +1,5 @@
1
  """This module initializes the services package."""
2
 
3
  from .google_client import client
4
- from .generator import generate_presentation
5
 
6
- __all__ = ["client", "generate_presentation"]
 
1
  """This module initializes the services package."""
2
 
3
  from .google_client import client
 
4
 
5
+ __all__ = ["client"]