telemetry using native @observe decorators
Browse files- src/compilation/latex_compilation.py +35 -30
- src/database/ingest.py +3 -0
- src/database/retrieve.py +17 -10
- src/processing/__init__.py +2 -2
- src/processing/document_processing.py +66 -60
- src/processing/images_processing.py +48 -44
- src/services/generator.py +31 -32
- src/telemetry/__init__.py +0 -4
- src/telemetry/telemetry.py +4 -54
src/compilation/latex_compilation.py
CHANGED
|
@@ -2,11 +2,14 @@
|
|
| 2 |
|
| 3 |
import os
|
| 4 |
import subprocess
|
| 5 |
-
from .
|
|
|
|
|
|
|
| 6 |
|
| 7 |
LOGGER = Logger.get_logger()
|
| 8 |
|
| 9 |
|
|
|
|
| 10 |
def compile_presentation(latex_code, work_dir) -> None:
|
| 11 |
"""Compile LaTeX code into a PDF presentation.
|
| 12 |
Args:
|
|
@@ -34,35 +37,37 @@ def compile_presentation(latex_code, work_dir) -> None:
|
|
| 34 |
# Compile with pdflatex (using subprocess instead of `!`)
|
| 35 |
os.chdir(work_dir) # Change working directory
|
| 36 |
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
)
|
| 52 |
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 60 |
)
|
|
|
|
| 61 |
else:
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
span.set_attribute("pdf.success", False)
|
| 65 |
-
LOGGER.error("❌ PDF generation failed. No PDF file found.")
|
| 66 |
-
else:
|
| 67 |
-
span.set_attribute("pdf.success", True)
|
| 68 |
-
LOGGER.info("💾 PDF generated successfully in: %s", work_dir)
|
|
|
|
| 2 |
|
| 3 |
import os
|
| 4 |
import subprocess
|
| 5 |
+
from langfuse.decorators import langfuse_context, observe
|
| 6 |
+
|
| 7 |
+
from ..telemetry import Logger
|
| 8 |
|
| 9 |
LOGGER = Logger.get_logger()
|
| 10 |
|
| 11 |
|
| 12 |
+
@observe(name="🧱 compile_presentation")
|
| 13 |
def compile_presentation(latex_code, work_dir) -> None:
|
| 14 |
"""Compile LaTeX code into a PDF presentation.
|
| 15 |
Args:
|
|
|
|
| 37 |
# Compile with pdflatex (using subprocess instead of `!`)
|
| 38 |
os.chdir(work_dir) # Change working directory
|
| 39 |
|
| 40 |
+
try:
|
| 41 |
+
# Run pdflatex twice to ensure proper slide enumeration
|
| 42 |
+
subprocess.run(
|
| 43 |
+
["pdflatex", "-interaction=nonstopmode", "presentation.tex"],
|
| 44 |
+
check=True,
|
| 45 |
+
stdout=subprocess.PIPE,
|
| 46 |
+
stderr=subprocess.PIPE,
|
| 47 |
+
)
|
| 48 |
+
subprocess.run(
|
| 49 |
+
["pdflatex", "-interaction=nonstopmode", "presentation.tex"],
|
| 50 |
+
check=True,
|
| 51 |
+
stdout=subprocess.PIPE,
|
| 52 |
+
stderr=subprocess.PIPE,
|
| 53 |
+
)
|
|
|
|
| 54 |
|
| 55 |
+
except subprocess.CalledProcessError as e:
|
| 56 |
+
# Handle error in case of failed LaTeX compilation
|
| 57 |
+
langfuse_context.update_current_observation(
|
| 58 |
+
output={"pdf.success": False, "pdf.error_log": e.stderr.decode()}
|
| 59 |
+
)
|
| 60 |
+
LOGGER.error("❌ PDF generation failed. Here's the log: %s", e.stderr.decode())
|
| 61 |
+
else:
|
| 62 |
+
# Check for PDF output
|
| 63 |
+
if not os.path.exists("presentation.pdf"):
|
| 64 |
+
langfuse_context.update_current_observation(
|
| 65 |
+
output={
|
| 66 |
+
"pdf.success": False,
|
| 67 |
+
"pdf.error_log": "❌ PDF generation failed. No PDF file found.",
|
| 68 |
+
}
|
| 69 |
)
|
| 70 |
+
LOGGER.error("❌ PDF generation failed. No PDF file found.")
|
| 71 |
else:
|
| 72 |
+
langfuse_context.update_current_observation(output={"pdf.success": True})
|
| 73 |
+
LOGGER.info("💾 PDF generated successfully in: %s", work_dir)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
src/database/ingest.py
CHANGED
|
@@ -1,6 +1,8 @@
|
|
| 1 |
"""Ingest module for adding new PDF files to the database."""
|
| 2 |
|
| 3 |
from pathlib import Path
|
|
|
|
|
|
|
| 4 |
from .database import db
|
| 5 |
from ..telemetry import Logger
|
| 6 |
from ..processing import process_documents
|
|
@@ -8,6 +10,7 @@ from ..processing import process_documents
|
|
| 8 |
LOGGER = Logger.get_logger()
|
| 9 |
|
| 10 |
|
|
|
|
| 11 |
def ingest_files_to_db(pdf_files) -> None:
|
| 12 |
"""Add new files to the database if they are not yet there.
|
| 13 |
Args:
|
|
|
|
| 1 |
"""Ingest module for adding new PDF files to the database."""
|
| 2 |
|
| 3 |
from pathlib import Path
|
| 4 |
+
from langfuse.decorators import langfuse_context, observe
|
| 5 |
+
|
| 6 |
from .database import db
|
| 7 |
from ..telemetry import Logger
|
| 8 |
from ..processing import process_documents
|
|
|
|
| 10 |
LOGGER = Logger.get_logger()
|
| 11 |
|
| 12 |
|
| 13 |
+
@observe(name="𝌊 ingest_files_to_db")
|
| 14 |
def ingest_files_to_db(pdf_files) -> None:
|
| 15 |
"""Add new files to the database if they are not yet there.
|
| 16 |
Args:
|
src/database/retrieve.py
CHANGED
|
@@ -1,10 +1,13 @@
|
|
| 1 |
import json
|
|
|
|
|
|
|
| 2 |
from .database import db, embed_fn
|
| 3 |
-
from ..telemetry import Logger
|
| 4 |
|
| 5 |
LOGGER = Logger.get_logger()
|
| 6 |
|
| 7 |
|
|
|
|
| 8 |
def retrive_files_from_db(topic: str) -> tuple[list[str], list[dict]]:
|
| 9 |
"""Retrieve relevant documents from the database based on the provided topic.
|
| 10 |
Args:
|
|
@@ -17,13 +20,17 @@ def retrive_files_from_db(topic: str) -> tuple[list[str], list[dict]]:
|
|
| 17 |
LOGGER.info("🔍 Retrieving relevant documents...")
|
| 18 |
embed_fn.document_mode = False
|
| 19 |
query_oneline = topic.replace("\n", " ")
|
| 20 |
-
with traced_block("🔎 retrieve_documents") as span:
|
| 21 |
-
span.set_attribute("input.query_oneline", query_oneline)
|
| 22 |
-
n_yelded_docs = 2
|
| 23 |
-
result = db.query(query_texts=[query_oneline], n_results=n_yelded_docs)
|
| 24 |
-
span.set_attribute("output.n_results", n_yelded_docs)
|
| 25 |
-
span.set_attribute("output.metadatas", json.dumps(result["metadatas"]))
|
| 26 |
-
[documents] = result["documents"]
|
| 27 |
-
[metadatas] = result["metadatas"]
|
| 28 |
|
| 29 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
import json
|
| 2 |
+
from langfuse.decorators import langfuse_context, observe
|
| 3 |
+
|
| 4 |
from .database import db, embed_fn
|
| 5 |
+
from ..telemetry import Logger
|
| 6 |
|
| 7 |
LOGGER = Logger.get_logger()
|
| 8 |
|
| 9 |
|
| 10 |
+
@observe(name="🔍 retrieve_files_from_db")
|
| 11 |
def retrive_files_from_db(topic: str) -> tuple[list[str], list[dict]]:
|
| 12 |
"""Retrieve relevant documents from the database based on the provided topic.
|
| 13 |
Args:
|
|
|
|
| 20 |
LOGGER.info("🔍 Retrieving relevant documents...")
|
| 21 |
embed_fn.document_mode = False
|
| 22 |
query_oneline = topic.replace("\n", " ")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 23 |
|
| 24 |
+
n_yelded_docs = 2
|
| 25 |
+
result = db.query(query_texts=[query_oneline], n_results=n_yelded_docs)
|
| 26 |
+
langfuse_context.update_current_observation(
|
| 27 |
+
output={
|
| 28 |
+
"input.query_oneline": query_oneline,
|
| 29 |
+
"output.n_results": n_yelded_docs,
|
| 30 |
+
"output.metadatas": result["metadatas"],
|
| 31 |
+
}
|
| 32 |
+
)
|
| 33 |
+
[documents] = result["documents"]
|
| 34 |
+
[metadatas] = result["metadatas"]
|
| 35 |
+
|
| 36 |
+
return [documents], [metadatas]
|
src/processing/__init__.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
| 1 |
"""This module initializes the document processing package."""
|
| 2 |
|
| 3 |
from .document_processing import process_documents, delete_uploaded_files
|
| 4 |
-
from .images_processing import save_pdf_images, save_pdf_figures,
|
| 5 |
from .output_folder import create_output_folder
|
| 6 |
from .prompt import get_prompt
|
| 7 |
|
|
@@ -10,7 +10,7 @@ __all__ = [
|
|
| 10 |
"delete_uploaded_files",
|
| 11 |
"save_pdf_images",
|
| 12 |
"save_pdf_figures",
|
| 13 |
-
"
|
| 14 |
"create_output_folder",
|
| 15 |
"get_prompt",
|
| 16 |
]
|
|
|
|
| 1 |
"""This module initializes the document processing package."""
|
| 2 |
|
| 3 |
from .document_processing import process_documents, delete_uploaded_files
|
| 4 |
+
from .images_processing import save_pdf_images, save_pdf_figures, find_used_gfx
|
| 5 |
from .output_folder import create_output_folder
|
| 6 |
from .prompt import get_prompt
|
| 7 |
|
|
|
|
| 10 |
"delete_uploaded_files",
|
| 11 |
"save_pdf_images",
|
| 12 |
"save_pdf_figures",
|
| 13 |
+
"find_used_gfx",
|
| 14 |
"create_output_folder",
|
| 15 |
"get_prompt",
|
| 16 |
]
|
src/processing/document_processing.py
CHANGED
|
@@ -5,13 +5,15 @@ import os
|
|
| 5 |
import json
|
| 6 |
from pathlib import Path
|
| 7 |
import fitz
|
|
|
|
| 8 |
|
| 9 |
from .images_processing import extract_images, extract_vector
|
| 10 |
-
from ..telemetry import Logger
|
| 11 |
|
| 12 |
LOGGER = Logger.get_logger()
|
| 13 |
|
| 14 |
|
|
|
|
| 15 |
def extract_pdf_content(pdf_path: str):
|
| 16 |
"""Extract text and images from a PDF file.
|
| 17 |
Args:
|
|
@@ -22,75 +24,79 @@ def extract_pdf_content(pdf_path: str):
|
|
| 22 |
- list: List of dictionaries containing image metadata.
|
| 23 |
- dict: Metadata about the PDF, including the number of images and the PDF path.
|
| 24 |
"""
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
pdf = os.path.splitext(os.path.basename(pdf_path))[0]
|
| 28 |
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
|
| 34 |
|
| 35 |
-
|
| 36 |
-
|
| 37 |
|
| 38 |
-
|
| 39 |
-
|
| 40 |
|
| 41 |
-
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
span.set_attribute("output.figs", json.dumps(figs))
|
| 45 |
|
| 46 |
-
|
| 47 |
|
| 48 |
|
|
|
|
| 49 |
def process_documents(pdf_files):
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 88 |
}
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
metadatas.append(fixed_metadata)
|
| 92 |
|
| 93 |
-
|
| 94 |
|
| 95 |
|
| 96 |
def delete_uploaded_files(uploaded_files):
|
|
|
|
| 5 |
import json
|
| 6 |
from pathlib import Path
|
| 7 |
import fitz
|
| 8 |
+
from langfuse.decorators import langfuse_context, observe
|
| 9 |
|
| 10 |
from .images_processing import extract_images, extract_vector
|
| 11 |
+
from ..telemetry import Logger
|
| 12 |
|
| 13 |
LOGGER = Logger.get_logger()
|
| 14 |
|
| 15 |
|
| 16 |
+
@observe(name="⚙️ extract_pdf_content")
|
| 17 |
def extract_pdf_content(pdf_path: str):
|
| 18 |
"""Extract text and images from a PDF file.
|
| 19 |
Args:
|
|
|
|
| 24 |
- list: List of dictionaries containing image metadata.
|
| 25 |
- dict: Metadata about the PDF, including the number of images and the PDF path.
|
| 26 |
"""
|
| 27 |
+
doc = fitz.open(pdf_path)
|
| 28 |
+
pdf = os.path.splitext(os.path.basename(pdf_path))[0]
|
|
|
|
| 29 |
|
| 30 |
+
text = ""
|
| 31 |
+
figs = []
|
| 32 |
+
for page_num, page in enumerate(doc):
|
| 33 |
+
# Parse the text
|
| 34 |
+
text = " ".join([text, page.get_text().strip()])
|
| 35 |
|
| 36 |
+
# Extract images
|
| 37 |
+
figs += extract_images(pdf, doc, page, page_num)
|
| 38 |
|
| 39 |
+
# Extract vector graphics
|
| 40 |
+
figs += extract_vector(pdf, doc, page, page_num)
|
| 41 |
|
| 42 |
+
# Format the metadata
|
| 43 |
+
metas = {"num_images": len(figs), "pdf_path": pdf_path}
|
| 44 |
+
langfuse_context.update_current_observation(output={"figs": figs, "metas": metas})
|
|
|
|
| 45 |
|
| 46 |
+
return text, figs, metas
|
| 47 |
|
| 48 |
|
| 49 |
+
@observe(name="📊 process_documents")
|
| 50 |
def process_documents(pdf_files):
|
| 51 |
+
documents = []
|
| 52 |
+
metadatas = []
|
| 53 |
+
|
| 54 |
+
for pdf_path in pdf_files:
|
| 55 |
+
text, imgs, metas = extract_pdf_content(pdf_path)
|
| 56 |
+
|
| 57 |
+
documents.append(text)
|
| 58 |
+
|
| 59 |
+
# Format images
|
| 60 |
+
images_info = []
|
| 61 |
+
for i, img in enumerate(imgs, start=1):
|
| 62 |
+
caption = img.get("caption")
|
| 63 |
+
caption_str = str(caption) if caption is not None else ""
|
| 64 |
+
img_name = img["name"]
|
| 65 |
+
img_ratio = img["ratio"]
|
| 66 |
+
full_path = f"gfx/{img_name}"
|
| 67 |
+
|
| 68 |
+
cleaned_caption = re.sub(
|
| 69 |
+
r"^(fig(?:ure)?\.?\s*\d+\.\s*)",
|
| 70 |
+
"",
|
| 71 |
+
caption_str,
|
| 72 |
+
flags=re.IGNORECASE,
|
| 73 |
+
).strip()
|
| 74 |
+
caption = cleaned_caption if cleaned_caption else "None"
|
| 75 |
+
|
| 76 |
+
images_info.append(
|
| 77 |
+
f'{{"path": "{full_path}", "caption": "{caption}", "orientation": "{img_ratio}"}}'
|
| 78 |
+
)
|
| 79 |
+
|
| 80 |
+
images_passage = "\n".join(images_info)
|
| 81 |
+
# span.set_attribute("output.images_passage", images_passage)
|
| 82 |
+
|
| 83 |
+
# Format metadata
|
| 84 |
+
fixed_metadata = {
|
| 85 |
+
"num_images": metas.get("num_images"),
|
| 86 |
+
"pdf_path": metas.get("pdf_path"),
|
| 87 |
+
"images_passage": images_passage,
|
| 88 |
+
}
|
| 89 |
+
|
| 90 |
+
langfuse_context.update_current_observation(
|
| 91 |
+
output={
|
| 92 |
+
"output.images_passage": images_passage,
|
| 93 |
+
"output.num_images": metas.get("num_images"),
|
| 94 |
+
"output.pdf_path": metas.get("pdf_path"),
|
| 95 |
}
|
| 96 |
+
)
|
| 97 |
+
metadatas.append(fixed_metadata)
|
|
|
|
| 98 |
|
| 99 |
+
return documents, metadatas
|
| 100 |
|
| 101 |
|
| 102 |
def delete_uploaded_files(uploaded_files):
|
src/processing/images_processing.py
CHANGED
|
@@ -7,8 +7,9 @@ import json
|
|
| 7 |
from collections import defaultdict
|
| 8 |
import fitz
|
| 9 |
from rtree import index
|
|
|
|
| 10 |
|
| 11 |
-
from ..telemetry import Logger
|
| 12 |
|
| 13 |
LOGGER = Logger.get_logger()
|
| 14 |
|
|
@@ -379,7 +380,8 @@ def save_pdf_figures(pdf_path: str, req_figs: list, figures_dir: str):
|
|
| 379 |
return True
|
| 380 |
|
| 381 |
|
| 382 |
-
|
|
|
|
| 383 |
"""Finds and saves the figures used in the LLM output to the gfx directory
|
| 384 |
where the presentation will be compiled.
|
| 385 |
Args:
|
|
@@ -387,45 +389,47 @@ def find_used_fgx(answer, work_dir: str, metadatas: list):
|
|
| 387 |
work_dir (str): The working directory where the presentation will be compiled.
|
| 388 |
metadatas (list): List of metadata dictionaries containing PDF paths.
|
| 389 |
"""
|
| 390 |
-
|
| 391 |
-
|
| 392 |
-
|
| 393 |
-
|
| 394 |
-
|
| 395 |
-
|
| 396 |
-
|
| 397 |
-
|
| 398 |
-
|
| 399 |
-
|
| 400 |
-
|
| 401 |
-
|
| 402 |
-
|
| 403 |
-
|
| 404 |
-
|
| 405 |
-
|
| 406 |
-
|
| 407 |
-
|
| 408 |
-
|
| 409 |
-
|
| 410 |
-
|
| 411 |
-
|
| 412 |
-
|
| 413 |
-
|
| 414 |
-
|
| 415 |
-
|
| 416 |
-
|
| 417 |
-
|
| 418 |
-
|
| 419 |
-
|
| 420 |
-
|
| 421 |
-
|
| 422 |
-
|
| 423 |
-
|
| 424 |
-
|
| 425 |
-
|
| 426 |
-
|
| 427 |
-
|
| 428 |
-
|
| 429 |
-
|
| 430 |
-
|
| 431 |
-
|
|
|
|
|
|
|
|
|
| 7 |
from collections import defaultdict
|
| 8 |
import fitz
|
| 9 |
from rtree import index
|
| 10 |
+
from langfuse.decorators import langfuse_context, observe
|
| 11 |
|
| 12 |
+
from ..telemetry import Logger
|
| 13 |
|
| 14 |
LOGGER = Logger.get_logger()
|
| 15 |
|
|
|
|
| 380 |
return True
|
| 381 |
|
| 382 |
|
| 383 |
+
@observe(name="🔍 find_used_gfx")
|
| 384 |
+
def find_used_gfx(answer, work_dir: str, metadatas: list):
|
| 385 |
"""Finds and saves the figures used in the LLM output to the gfx directory
|
| 386 |
where the presentation will be compiled.
|
| 387 |
Args:
|
|
|
|
| 389 |
work_dir (str): The working directory where the presentation will be compiled.
|
| 390 |
metadatas (list): List of metadata dictionaries containing PDF paths.
|
| 391 |
"""
|
| 392 |
+
graphics_dir = os.path.join(work_dir, "gfx")
|
| 393 |
+
os.makedirs(graphics_dir, exist_ok=True)
|
| 394 |
+
LOGGER.info("🌄 Images will be saved to: %s", graphics_dir)
|
| 395 |
+
|
| 396 |
+
# Find images, which are used in the presentation
|
| 397 |
+
pattern_img = re.compile(
|
| 398 |
+
r"doc(?P<doc>[a-zA-Z0-9_]+)_page(?P<page>\d+)_img(?P<img>\d+)_hash(?P<hash>[a-fA-F0-9]{8})\.png"
|
| 399 |
+
)
|
| 400 |
+
matches_img = pattern_img.finditer(answer.text)
|
| 401 |
+
|
| 402 |
+
req_imgs = []
|
| 403 |
+
for match in matches_img:
|
| 404 |
+
req_img = {
|
| 405 |
+
"doc": match.group("doc"),
|
| 406 |
+
"page": int(match.group("page")),
|
| 407 |
+
"img": int(match.group("img")),
|
| 408 |
+
"hash": match.group("hash"),
|
| 409 |
+
}
|
| 410 |
+
print("matches_img = ", match.group("doc"))
|
| 411 |
+
req_imgs.append(req_img)
|
| 412 |
+
|
| 413 |
+
# Find figures, which are used in the presentation
|
| 414 |
+
pattern_fig = re.compile(
|
| 415 |
+
r"doc(?P<doc>[a-zA-Z0-9_]+)_page(?P<page>\d+)_fig(?P<fig>\d+)_hash(?P<hash>[a-fA-F0-9]{8})\.png"
|
| 416 |
+
)
|
| 417 |
+
matches_fig = pattern_fig.finditer(answer.text)
|
| 418 |
+
req_figs = []
|
| 419 |
+
for match in matches_fig:
|
| 420 |
+
req_fig = {
|
| 421 |
+
"doc": match.group("doc"),
|
| 422 |
+
"page": int(match.group("page")),
|
| 423 |
+
"fig": int(match.group("fig")),
|
| 424 |
+
"hash": match.group("hash"),
|
| 425 |
+
}
|
| 426 |
+
print("matches_fig = ", match.group("doc"))
|
| 427 |
+
req_figs.append(req_fig)
|
| 428 |
+
# span.set_attribute("output.req_figs", json.dumps(req_figs))
|
| 429 |
+
langfuse_context.update_current_observation(
|
| 430 |
+
output={"output.req_figs": json.dumps(req_figs)}
|
| 431 |
+
)
|
| 432 |
+
# Save the required graphics
|
| 433 |
+
for metadata in metadatas:
|
| 434 |
+
save_pdf_images(metadata["pdf_path"], req_imgs, graphics_dir)
|
| 435 |
+
save_pdf_figures(metadata["pdf_path"], req_figs, graphics_dir)
|
src/services/generator.py
CHANGED
|
@@ -1,19 +1,21 @@
|
|
| 1 |
"""Module to put it all together and manage the generation of a presentation."""
|
| 2 |
|
| 3 |
from pathlib import Path
|
|
|
|
| 4 |
|
| 5 |
# from datetime import datetime
|
| 6 |
from ..database import ingest_files_to_db, retrive_files_from_db
|
| 7 |
-
from ..telemetry import Logger
|
| 8 |
from ..processing import get_prompt, create_output_folder
|
| 9 |
from . import client
|
| 10 |
from ..compilation import compile_presentation
|
| 11 |
-
from ..processing import delete_uploaded_files,
|
| 12 |
|
| 13 |
|
| 14 |
LOGGER = Logger.get_logger()
|
| 15 |
|
| 16 |
|
|
|
|
| 17 |
def generate_presentation(theme, color, topic, uploaded_files) -> tuple[str, str, str]:
|
| 18 |
"""Main function to generate a presentation based on the provided theme, color, and topic.
|
| 19 |
Args:
|
|
@@ -28,40 +30,37 @@ def generate_presentation(theme, color, topic, uploaded_files) -> tuple[str, str
|
|
| 28 |
- str: Path to the generated presentation.
|
| 29 |
- str: Placeholder for the output viewer.
|
| 30 |
"""
|
| 31 |
-
with traced_block(
|
| 32 |
-
"📽️ generate_presentation",
|
| 33 |
-
input_theme=theme,
|
| 34 |
-
input_color=color,
|
| 35 |
-
input_topic=topic,
|
| 36 |
-
file_count=len(uploaded_files),
|
| 37 |
-
) as span:
|
| 38 |
-
trace_id = format(span.get_span_context().trace_id, "032x")
|
| 39 |
-
ingest_files_to_db(uploaded_files)
|
| 40 |
|
| 41 |
-
|
| 42 |
-
|
| 43 |
|
| 44 |
-
|
|
|
|
| 45 |
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
# print(f"passage_oneline = {passage_oneline}")
|
| 53 |
-
# print(f"images_passage = {images_passage}")
|
| 54 |
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 61 |
|
| 62 |
-
|
| 63 |
-
|
| 64 |
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
|
|
|
|
| 1 |
"""Module to put it all together and manage the generation of a presentation."""
|
| 2 |
|
| 3 |
from pathlib import Path
|
| 4 |
+
from langfuse.decorators import langfuse_context, observe
|
| 5 |
|
| 6 |
# from datetime import datetime
|
| 7 |
from ..database import ingest_files_to_db, retrive_files_from_db
|
| 8 |
+
from ..telemetry import Logger
|
| 9 |
from ..processing import get_prompt, create_output_folder
|
| 10 |
from . import client
|
| 11 |
from ..compilation import compile_presentation
|
| 12 |
+
from ..processing import delete_uploaded_files, find_used_gfx
|
| 13 |
|
| 14 |
|
| 15 |
LOGGER = Logger.get_logger()
|
| 16 |
|
| 17 |
|
| 18 |
+
@observe(name="🧑🎨 generate_presentation")
|
| 19 |
def generate_presentation(theme, color, topic, uploaded_files) -> tuple[str, str, str]:
|
| 20 |
"""Main function to generate a presentation based on the provided theme, color, and topic.
|
| 21 |
Args:
|
|
|
|
| 30 |
- str: Path to the generated presentation.
|
| 31 |
- str: Placeholder for the output viewer.
|
| 32 |
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 33 |
|
| 34 |
+
trace_id = langfuse_context.get_current_trace_id()
|
| 35 |
+
ingest_files_to_db(uploaded_files)
|
| 36 |
|
| 37 |
+
[documents], [metadatas] = retrive_files_from_db(topic)
|
| 38 |
+
prompt = get_prompt(theme, color)
|
| 39 |
|
| 40 |
+
for passage, metas in zip(documents, metadatas):
|
| 41 |
+
passage_oneline = passage.replace("\n", " ")
|
| 42 |
+
images_passage = metas["images_passage"]
|
| 43 |
|
| 44 |
+
prompt += f"PASSAGE: {passage_oneline}\n"
|
| 45 |
+
prompt += f"IMAGES: {images_passage}\n"
|
|
|
|
|
|
|
| 46 |
|
| 47 |
+
work_dir = create_output_folder()
|
| 48 |
+
# Generate the presentation code
|
| 49 |
+
model_name = "gemini-2.0-flash"
|
| 50 |
+
LOGGER.info("⭐️ Generating response...")
|
| 51 |
+
answer = client.models.generate_content(model=model_name, contents=prompt)
|
| 52 |
+
langfuse_context.update_current_observation(
|
| 53 |
+
output={
|
| 54 |
+
"model_name": model_name,
|
| 55 |
+
"prompt_length": len(prompt),
|
| 56 |
+
"output.response": answer.text,
|
| 57 |
+
}
|
| 58 |
+
)
|
| 59 |
+
LOGGER.info("🎲 Generated the response using: %s", model_name)
|
| 60 |
|
| 61 |
+
find_used_gfx(answer, work_dir, metadatas)
|
| 62 |
+
compile_presentation(answer.text, work_dir)
|
| 63 |
|
| 64 |
+
delete_uploaded_files(uploaded_files)
|
| 65 |
+
output_path = Path(work_dir + "/presentation.pdf")
|
| 66 |
+
return "Presentation generated successfully!", str(output_path), "", trace_id
|
src/telemetry/__init__.py
CHANGED
|
@@ -2,16 +2,12 @@
|
|
| 2 |
|
| 3 |
from .logging_utils import Logger
|
| 4 |
from .telemetry import (
|
| 5 |
-
generate_with_trace,
|
| 6 |
init_telemetry,
|
| 7 |
-
traced_block,
|
| 8 |
submit_feedback,
|
| 9 |
)
|
| 10 |
|
| 11 |
__all__ = [
|
| 12 |
-
"generate_with_trace",
|
| 13 |
"init_telemetry",
|
| 14 |
-
"traced_block",
|
| 15 |
"submit_feedback",
|
| 16 |
"Logger",
|
| 17 |
]
|
|
|
|
| 2 |
|
| 3 |
from .logging_utils import Logger
|
| 4 |
from .telemetry import (
|
|
|
|
| 5 |
init_telemetry,
|
|
|
|
| 6 |
submit_feedback,
|
| 7 |
)
|
| 8 |
|
| 9 |
__all__ = [
|
|
|
|
| 10 |
"init_telemetry",
|
|
|
|
| 11 |
"submit_feedback",
|
| 12 |
"Logger",
|
| 13 |
]
|
src/telemetry/telemetry.py
CHANGED
|
@@ -1,9 +1,6 @@
|
|
| 1 |
"""Telemetry utilities for Langfuse integration and OpenTelemetry tracing."""
|
| 2 |
|
| 3 |
import os
|
| 4 |
-
import hashlib
|
| 5 |
-
from typing import Union
|
| 6 |
-
from contextlib import contextmanager
|
| 7 |
|
| 8 |
import base64
|
| 9 |
import langfuse
|
|
@@ -41,56 +38,6 @@ def init_telemetry() -> None:
|
|
| 41 |
LOGGER.warning("Langfuse telemetry is disabled (USE_LANGFUSE != true)")
|
| 42 |
|
| 43 |
|
| 44 |
-
def hash_prompt(text: str) -> str:
|
| 45 |
-
"""Generate a SHA-256 hash of the prompt.
|
| 46 |
-
Args:
|
| 47 |
-
text (str): The input text to hash."""
|
| 48 |
-
return hashlib.sha256(text.encode()).hexdigest()
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
@contextmanager
|
| 52 |
-
def traced_block(name: str, **attrs):
|
| 53 |
-
"""Context manager to create a traced block.
|
| 54 |
-
Args:
|
| 55 |
-
name (str): The name of the span.
|
| 56 |
-
**attrs: Additional attributes to set on the span."""
|
| 57 |
-
with tracer.start_as_current_span(name) as span:
|
| 58 |
-
for key, value in attrs.items():
|
| 59 |
-
span.set_attribute(key, value)
|
| 60 |
-
try:
|
| 61 |
-
# trace_id = format(span.get_span_context().trace_id, "032x")
|
| 62 |
-
# yield span, trace_id
|
| 63 |
-
yield span
|
| 64 |
-
except Exception as e:
|
| 65 |
-
# Record exception and mark span status as error
|
| 66 |
-
span.record_exception(e)
|
| 67 |
-
span.set_status(trace.Status(trace.StatusCode.ERROR, str(e)))
|
| 68 |
-
LOGGER.error("Error in span %s while generating content: %s", name, e)
|
| 69 |
-
raise
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
def generate_with_trace(client, model_name: str, prompt: str) -> Union[object, None]:
|
| 73 |
-
"""Save traces from the generation.
|
| 74 |
-
Args:
|
| 75 |
-
client: google genai client instance.
|
| 76 |
-
model_name (str): The name of the model to use.
|
| 77 |
-
prompt (str): The input prompt for generation."""
|
| 78 |
-
with tracer.start_as_current_span("🤖 genai_generate") as span:
|
| 79 |
-
span.set_attribute("input.prompt_hash", hash_prompt(prompt))
|
| 80 |
-
span.set_attribute("input.prompt_length", len(prompt))
|
| 81 |
-
span.set_attribute("input.model_name", model_name)
|
| 82 |
-
try:
|
| 83 |
-
response = client.models.generate_content(model=model_name, contents=prompt)
|
| 84 |
-
span.set_attribute("output.response_length", len(str(response)))
|
| 85 |
-
span.set_attribute("output.success", True)
|
| 86 |
-
return response
|
| 87 |
-
except Exception as e:
|
| 88 |
-
span.set_attribute("output.success", False)
|
| 89 |
-
span.record_exception(e)
|
| 90 |
-
span.set_status(trace.Status(trace.StatusCode.ERROR, str(e)))
|
| 91 |
-
return LOGGER.error("Error in spans while generating content: %s", e)
|
| 92 |
-
|
| 93 |
-
|
| 94 |
def submit_feedback(rating: str, comment: str, trace_id: str) -> str:
|
| 95 |
if not rating:
|
| 96 |
return "⚠️ Please select a star rating before submitting."
|
|
@@ -102,7 +49,10 @@ def submit_feedback(rating: str, comment: str, trace_id: str) -> str:
|
|
| 102 |
if trace_id:
|
| 103 |
try:
|
| 104 |
langfuse.feedback(
|
| 105 |
-
trace_id=trace_id,
|
|
|
|
|
|
|
|
|
|
| 106 |
)
|
| 107 |
except Exception as e:
|
| 108 |
return f"✅ Feedback received ({stars_display}), but Langfuse logging failed: {e}"
|
|
|
|
| 1 |
"""Telemetry utilities for Langfuse integration and OpenTelemetry tracing."""
|
| 2 |
|
| 3 |
import os
|
|
|
|
|
|
|
|
|
|
| 4 |
|
| 5 |
import base64
|
| 6 |
import langfuse
|
|
|
|
| 38 |
LOGGER.warning("Langfuse telemetry is disabled (USE_LANGFUSE != true)")
|
| 39 |
|
| 40 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 41 |
def submit_feedback(rating: str, comment: str, trace_id: str) -> str:
|
| 42 |
if not rating:
|
| 43 |
return "⚠️ Please select a star rating before submitting."
|
|
|
|
| 49 |
if trace_id:
|
| 50 |
try:
|
| 51 |
langfuse.feedback(
|
| 52 |
+
trace_id=trace_id,
|
| 53 |
+
name="user_feedback",
|
| 54 |
+
comment=comment or None,
|
| 55 |
+
score=score,
|
| 56 |
)
|
| 57 |
except Exception as e:
|
| 58 |
return f"✅ Feedback received ({stars_display}), but Langfuse logging failed: {e}"
|