telemetry and refactoring
Browse files- poetry.lock +215 -53
- pyproject.toml +9 -1
- ragntex.py +6 -1
- src/__init__.py +3 -16
- src/compilation/__init__.py +4 -2
- src/compilation/latex_compilation.py +41 -27
- src/database/__init__.py +4 -1
- src/database/database.py +4 -1
- src/database/ingest.py +8 -3
- src/database/retrieve.py +28 -0
- src/interface/__init__.py +4 -1
- src/interface/gradio_interface.py +11 -4
- src/{utils/gradio_user_settings.py → interface/upload_files.py} +10 -3
- src/processing/__init__.py +14 -3
- src/processing/document_processing.py +22 -3
- src/processing/images_processing.py +60 -4
- src/processing/output_folder.py +25 -0
- src/processing/prompt.py +19 -7
- src/services/__init__.py +2 -0
- src/services/generator.py +57 -110
- src/services/google_client.py +2 -1
- src/telemetry/__init__.py +11 -0
- src/{utils → telemetry}/logging_utils.py +11 -1
- src/telemetry/telemetry.py +89 -0
- src/utils/__init__.py +0 -9
poetry.lock
CHANGED
|
@@ -60,6 +60,18 @@ files = [
|
|
| 60 |
[package.extras]
|
| 61 |
tests = ["mypy (>=0.800)", "pytest", "pytest-asyncio"]
|
| 62 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 63 |
[[package]]
|
| 64 |
name = "attrs"
|
| 65 |
version = "25.3.0"
|
|
@@ -483,12 +495,12 @@ version = "0.4.6"
|
|
| 483 |
description = "Cross-platform colored terminal text."
|
| 484 |
optional = false
|
| 485 |
python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7"
|
| 486 |
-
groups = ["main"]
|
| 487 |
-
markers = "os_name == \"nt\" or platform_system == \"Windows\" or sys_platform == \"win32\""
|
| 488 |
files = [
|
| 489 |
{file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"},
|
| 490 |
{file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"},
|
| 491 |
]
|
|
|
|
| 492 |
|
| 493 |
[[package]]
|
| 494 |
name = "coloredlogs"
|
|
@@ -526,6 +538,22 @@ wrapt = ">=1.10,<2"
|
|
| 526 |
[package.extras]
|
| 527 |
dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "setuptools ; python_version >= \"3.12\"", "tox"]
|
| 528 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 529 |
[[package]]
|
| 530 |
name = "distro"
|
| 531 |
version = "1.9.0"
|
|
@@ -1191,6 +1219,22 @@ enabler = ["pytest-enabler (>=2.2)"]
|
|
| 1191 |
test = ["jaraco.test (>=5.4)", "pytest (>=6,!=8.1.*)", "zipp (>=3.17)"]
|
| 1192 |
type = ["pytest-mypy"]
|
| 1193 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1194 |
[[package]]
|
| 1195 |
name = "jinja2"
|
| 1196 |
version = "3.1.6"
|
|
@@ -1274,6 +1318,33 @@ websocket-client = ">=0.32.0,<0.40.0 || >0.40.0,<0.41.dev0 || >=0.43.dev0"
|
|
| 1274 |
[package.extras]
|
| 1275 |
adal = ["adal (>=1.0.2)"]
|
| 1276 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1277 |
[[package]]
|
| 1278 |
name = "markdown-it-py"
|
| 1279 |
version = "3.0.0"
|
|
@@ -1370,6 +1441,18 @@ files = [
|
|
| 1370 |
{file = "markupsafe-3.0.2.tar.gz", hash = "sha256:ee55d3edf80167e48ea11a923c7386f4669df67d7994554387f84e7d8b0a2bf0"},
|
| 1371 |
]
|
| 1372 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1373 |
[[package]]
|
| 1374 |
name = "mdurl"
|
| 1375 |
version = "0.1.2"
|
|
@@ -1631,45 +1714,61 @@ sympy = "*"
|
|
| 1631 |
|
| 1632 |
[[package]]
|
| 1633 |
name = "opentelemetry-api"
|
| 1634 |
-
version = "1.
|
| 1635 |
description = "OpenTelemetry Python API"
|
| 1636 |
optional = false
|
| 1637 |
python-versions = ">=3.8"
|
| 1638 |
groups = ["main"]
|
| 1639 |
files = [
|
| 1640 |
-
{file = "opentelemetry_api-1.
|
| 1641 |
-
{file = "opentelemetry_api-1.
|
| 1642 |
]
|
| 1643 |
|
| 1644 |
[package.dependencies]
|
| 1645 |
deprecated = ">=1.2.6"
|
| 1646 |
importlib-metadata = ">=6.0,<8.7.0"
|
| 1647 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1648 |
[[package]]
|
| 1649 |
name = "opentelemetry-exporter-otlp-proto-common"
|
| 1650 |
-
version = "1.
|
| 1651 |
description = "OpenTelemetry Protobuf encoding"
|
| 1652 |
optional = false
|
| 1653 |
python-versions = ">=3.8"
|
| 1654 |
groups = ["main"]
|
| 1655 |
files = [
|
| 1656 |
-
{file = "opentelemetry_exporter_otlp_proto_common-1.
|
| 1657 |
-
{file = "opentelemetry_exporter_otlp_proto_common-1.
|
| 1658 |
]
|
| 1659 |
|
| 1660 |
[package.dependencies]
|
| 1661 |
-
opentelemetry-proto = "1.
|
| 1662 |
|
| 1663 |
[[package]]
|
| 1664 |
name = "opentelemetry-exporter-otlp-proto-grpc"
|
| 1665 |
-
version = "1.
|
| 1666 |
description = "OpenTelemetry Collector Protobuf over gRPC Exporter"
|
| 1667 |
optional = false
|
| 1668 |
python-versions = ">=3.8"
|
| 1669 |
groups = ["main"]
|
| 1670 |
files = [
|
| 1671 |
-
{file = "opentelemetry_exporter_otlp_proto_grpc-1.
|
| 1672 |
-
{file = "opentelemetry_exporter_otlp_proto_grpc-1.
|
| 1673 |
]
|
| 1674 |
|
| 1675 |
[package.dependencies]
|
|
@@ -1677,82 +1776,103 @@ deprecated = ">=1.2.6"
|
|
| 1677 |
googleapis-common-protos = ">=1.52,<2.0"
|
| 1678 |
grpcio = {version = ">=1.66.2,<2.0.0", markers = "python_version >= \"3.13\""}
|
| 1679 |
opentelemetry-api = ">=1.15,<2.0"
|
| 1680 |
-
opentelemetry-exporter-otlp-proto-common = "1.
|
| 1681 |
-
opentelemetry-proto = "1.
|
| 1682 |
-
opentelemetry-sdk = ">=1.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1683 |
|
| 1684 |
[[package]]
|
| 1685 |
name = "opentelemetry-instrumentation"
|
| 1686 |
-
version = "0.
|
| 1687 |
description = "Instrumentation Tools & Auto Instrumentation for OpenTelemetry Python"
|
| 1688 |
optional = false
|
| 1689 |
python-versions = ">=3.8"
|
| 1690 |
groups = ["main"]
|
| 1691 |
files = [
|
| 1692 |
-
{file = "opentelemetry_instrumentation-0.
|
| 1693 |
-
{file = "opentelemetry_instrumentation-0.
|
| 1694 |
]
|
| 1695 |
|
| 1696 |
[package.dependencies]
|
| 1697 |
opentelemetry-api = ">=1.4,<2.0"
|
| 1698 |
-
opentelemetry-semantic-conventions = "0.
|
| 1699 |
packaging = ">=18.0"
|
| 1700 |
wrapt = ">=1.0.0,<2.0.0"
|
| 1701 |
|
| 1702 |
[[package]]
|
| 1703 |
name = "opentelemetry-instrumentation-asgi"
|
| 1704 |
-
version = "0.
|
| 1705 |
description = "ASGI instrumentation for OpenTelemetry"
|
| 1706 |
optional = false
|
| 1707 |
python-versions = ">=3.8"
|
| 1708 |
groups = ["main"]
|
| 1709 |
files = [
|
| 1710 |
-
{file = "opentelemetry_instrumentation_asgi-0.
|
| 1711 |
-
{file = "opentelemetry_instrumentation_asgi-0.
|
| 1712 |
]
|
| 1713 |
|
| 1714 |
[package.dependencies]
|
| 1715 |
asgiref = ">=3.0,<4.0"
|
| 1716 |
opentelemetry-api = ">=1.12,<2.0"
|
| 1717 |
-
opentelemetry-instrumentation = "0.
|
| 1718 |
-
opentelemetry-semantic-conventions = "0.
|
| 1719 |
-
opentelemetry-util-http = "0.
|
| 1720 |
|
| 1721 |
[package.extras]
|
| 1722 |
instruments = ["asgiref (>=3.0,<4.0)"]
|
| 1723 |
|
| 1724 |
[[package]]
|
| 1725 |
name = "opentelemetry-instrumentation-fastapi"
|
| 1726 |
-
version = "0.
|
| 1727 |
description = "OpenTelemetry FastAPI Instrumentation"
|
| 1728 |
optional = false
|
| 1729 |
python-versions = ">=3.8"
|
| 1730 |
groups = ["main"]
|
| 1731 |
files = [
|
| 1732 |
-
{file = "opentelemetry_instrumentation_fastapi-0.
|
| 1733 |
-
{file = "opentelemetry_instrumentation_fastapi-0.
|
| 1734 |
]
|
| 1735 |
|
| 1736 |
[package.dependencies]
|
| 1737 |
opentelemetry-api = ">=1.12,<2.0"
|
| 1738 |
-
opentelemetry-instrumentation = "0.
|
| 1739 |
-
opentelemetry-instrumentation-asgi = "0.
|
| 1740 |
-
opentelemetry-semantic-conventions = "0.
|
| 1741 |
-
opentelemetry-util-http = "0.
|
| 1742 |
|
| 1743 |
[package.extras]
|
| 1744 |
instruments = ["fastapi (>=0.58,<1.0)"]
|
| 1745 |
|
| 1746 |
[[package]]
|
| 1747 |
name = "opentelemetry-proto"
|
| 1748 |
-
version = "1.
|
| 1749 |
description = "OpenTelemetry Python Proto"
|
| 1750 |
optional = false
|
| 1751 |
python-versions = ">=3.8"
|
| 1752 |
groups = ["main"]
|
| 1753 |
files = [
|
| 1754 |
-
{file = "opentelemetry_proto-1.
|
| 1755 |
-
{file = "opentelemetry_proto-1.
|
| 1756 |
]
|
| 1757 |
|
| 1758 |
[package.dependencies]
|
|
@@ -1760,47 +1880,47 @@ protobuf = ">=5.0,<6.0"
|
|
| 1760 |
|
| 1761 |
[[package]]
|
| 1762 |
name = "opentelemetry-sdk"
|
| 1763 |
-
version = "1.
|
| 1764 |
description = "OpenTelemetry Python SDK"
|
| 1765 |
optional = false
|
| 1766 |
python-versions = ">=3.8"
|
| 1767 |
groups = ["main"]
|
| 1768 |
files = [
|
| 1769 |
-
{file = "opentelemetry_sdk-1.
|
| 1770 |
-
{file = "opentelemetry_sdk-1.
|
| 1771 |
]
|
| 1772 |
|
| 1773 |
[package.dependencies]
|
| 1774 |
-
opentelemetry-api = "1.
|
| 1775 |
-
opentelemetry-semantic-conventions = "0.
|
| 1776 |
typing-extensions = ">=3.7.4"
|
| 1777 |
|
| 1778 |
[[package]]
|
| 1779 |
name = "opentelemetry-semantic-conventions"
|
| 1780 |
-
version = "0.
|
| 1781 |
description = "OpenTelemetry Semantic Conventions"
|
| 1782 |
optional = false
|
| 1783 |
python-versions = ">=3.8"
|
| 1784 |
groups = ["main"]
|
| 1785 |
files = [
|
| 1786 |
-
{file = "opentelemetry_semantic_conventions-0.
|
| 1787 |
-
{file = "opentelemetry_semantic_conventions-0.
|
| 1788 |
]
|
| 1789 |
|
| 1790 |
[package.dependencies]
|
| 1791 |
deprecated = ">=1.2.6"
|
| 1792 |
-
opentelemetry-api = "1.
|
| 1793 |
|
| 1794 |
[[package]]
|
| 1795 |
name = "opentelemetry-util-http"
|
| 1796 |
-
version = "0.
|
| 1797 |
description = "Web util for OpenTelemetry"
|
| 1798 |
optional = false
|
| 1799 |
python-versions = ">=3.8"
|
| 1800 |
groups = ["main"]
|
| 1801 |
files = [
|
| 1802 |
-
{file = "opentelemetry_util_http-0.
|
| 1803 |
-
{file = "opentelemetry_util_http-0.
|
| 1804 |
]
|
| 1805 |
|
| 1806 |
[[package]]
|
|
@@ -1895,14 +2015,14 @@ files = [
|
|
| 1895 |
|
| 1896 |
[[package]]
|
| 1897 |
name = "packaging"
|
| 1898 |
-
version = "
|
| 1899 |
description = "Core utilities for Python packages"
|
| 1900 |
optional = false
|
| 1901 |
python-versions = ">=3.8"
|
| 1902 |
groups = ["main"]
|
| 1903 |
files = [
|
| 1904 |
-
{file = "packaging-
|
| 1905 |
-
{file = "packaging-
|
| 1906 |
]
|
| 1907 |
|
| 1908 |
[[package]]
|
|
@@ -2088,6 +2208,23 @@ tests = ["check-manifest", "coverage (>=7.4.2)", "defusedxml", "markdown2", "ole
|
|
| 2088 |
typing = ["typing-extensions ; python_version < \"3.10\""]
|
| 2089 |
xmp = ["defusedxml"]
|
| 2090 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2091 |
[[package]]
|
| 2092 |
name = "posthog"
|
| 2093 |
version = "4.0.0"
|
|
@@ -2341,6 +2478,31 @@ files = [
|
|
| 2341 |
[package.extras]
|
| 2342 |
windows-terminal = ["colorama (>=0.4.6)"]
|
| 2343 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2344 |
[[package]]
|
| 2345 |
name = "pymupdf"
|
| 2346 |
version = "1.25.5"
|
|
@@ -2966,7 +3128,7 @@ version = "0.13.2"
|
|
| 2966 |
description = "Style preserving TOML library"
|
| 2967 |
optional = false
|
| 2968 |
python-versions = ">=3.8"
|
| 2969 |
-
groups = ["main"]
|
| 2970 |
files = [
|
| 2971 |
{file = "tomlkit-0.13.2-py3-none-any.whl", hash = "sha256:7a974427f6e119197f670fbbbeae7bef749a6c14e793db934baefc1b5f03efde"},
|
| 2972 |
{file = "tomlkit-0.13.2.tar.gz", hash = "sha256:fff5fe59a87295b278abd31bec92c15d9bc4a06885ab12bcea52c71119392e79"},
|
|
@@ -3451,5 +3613,5 @@ type = ["pytest-mypy"]
|
|
| 3451 |
|
| 3452 |
[metadata]
|
| 3453 |
lock-version = "2.1"
|
| 3454 |
-
python-versions = ">=3.13"
|
| 3455 |
-
content-hash = "
|
|
|
|
| 60 |
[package.extras]
|
| 61 |
tests = ["mypy (>=0.800)", "pytest", "pytest-asyncio"]
|
| 62 |
|
| 63 |
+
[[package]]
|
| 64 |
+
name = "astroid"
|
| 65 |
+
version = "3.3.10"
|
| 66 |
+
description = "An abstract syntax tree for Python with inference support."
|
| 67 |
+
optional = false
|
| 68 |
+
python-versions = ">=3.9.0"
|
| 69 |
+
groups = ["dev"]
|
| 70 |
+
files = [
|
| 71 |
+
{file = "astroid-3.3.10-py3-none-any.whl", hash = "sha256:104fb9cb9b27ea95e847a94c003be03a9e039334a8ebca5ee27dafaf5c5711eb"},
|
| 72 |
+
{file = "astroid-3.3.10.tar.gz", hash = "sha256:c332157953060c6deb9caa57303ae0d20b0fbdb2e59b4a4f2a6ba49d0a7961ce"},
|
| 73 |
+
]
|
| 74 |
+
|
| 75 |
[[package]]
|
| 76 |
name = "attrs"
|
| 77 |
version = "25.3.0"
|
|
|
|
| 495 |
description = "Cross-platform colored terminal text."
|
| 496 |
optional = false
|
| 497 |
python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7"
|
| 498 |
+
groups = ["main", "dev"]
|
|
|
|
| 499 |
files = [
|
| 500 |
{file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"},
|
| 501 |
{file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"},
|
| 502 |
]
|
| 503 |
+
markers = {main = "os_name == \"nt\" or platform_system == \"Windows\" or sys_platform == \"win32\"", dev = "sys_platform == \"win32\""}
|
| 504 |
|
| 505 |
[[package]]
|
| 506 |
name = "coloredlogs"
|
|
|
|
| 538 |
[package.extras]
|
| 539 |
dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "setuptools ; python_version >= \"3.12\"", "tox"]
|
| 540 |
|
| 541 |
+
[[package]]
|
| 542 |
+
name = "dill"
|
| 543 |
+
version = "0.4.0"
|
| 544 |
+
description = "serialize all of Python"
|
| 545 |
+
optional = false
|
| 546 |
+
python-versions = ">=3.8"
|
| 547 |
+
groups = ["dev"]
|
| 548 |
+
files = [
|
| 549 |
+
{file = "dill-0.4.0-py3-none-any.whl", hash = "sha256:44f54bf6412c2c8464c14e8243eb163690a9800dbe2c367330883b19c7561049"},
|
| 550 |
+
{file = "dill-0.4.0.tar.gz", hash = "sha256:0633f1d2df477324f53a895b02c901fb961bdbf65a17122586ea7019292cbcf0"},
|
| 551 |
+
]
|
| 552 |
+
|
| 553 |
+
[package.extras]
|
| 554 |
+
graph = ["objgraph (>=1.7.2)"]
|
| 555 |
+
profile = ["gprof2dot (>=2022.7.29)"]
|
| 556 |
+
|
| 557 |
[[package]]
|
| 558 |
name = "distro"
|
| 559 |
version = "1.9.0"
|
|
|
|
| 1219 |
test = ["jaraco.test (>=5.4)", "pytest (>=6,!=8.1.*)", "zipp (>=3.17)"]
|
| 1220 |
type = ["pytest-mypy"]
|
| 1221 |
|
| 1222 |
+
[[package]]
|
| 1223 |
+
name = "isort"
|
| 1224 |
+
version = "6.0.1"
|
| 1225 |
+
description = "A Python utility / library to sort Python imports."
|
| 1226 |
+
optional = false
|
| 1227 |
+
python-versions = ">=3.9.0"
|
| 1228 |
+
groups = ["dev"]
|
| 1229 |
+
files = [
|
| 1230 |
+
{file = "isort-6.0.1-py3-none-any.whl", hash = "sha256:2dc5d7f65c9678d94c88dfc29161a320eec67328bc97aad576874cb4be1e9615"},
|
| 1231 |
+
{file = "isort-6.0.1.tar.gz", hash = "sha256:1cb5df28dfbc742e490c5e41bad6da41b805b0a8be7bc93cd0fb2a8a890ac450"},
|
| 1232 |
+
]
|
| 1233 |
+
|
| 1234 |
+
[package.extras]
|
| 1235 |
+
colors = ["colorama"]
|
| 1236 |
+
plugins = ["setuptools"]
|
| 1237 |
+
|
| 1238 |
[[package]]
|
| 1239 |
name = "jinja2"
|
| 1240 |
version = "3.1.6"
|
|
|
|
| 1318 |
[package.extras]
|
| 1319 |
adal = ["adal (>=1.0.2)"]
|
| 1320 |
|
| 1321 |
+
[[package]]
|
| 1322 |
+
name = "langfuse"
|
| 1323 |
+
version = "2.60.5"
|
| 1324 |
+
description = "A client library for accessing langfuse"
|
| 1325 |
+
optional = false
|
| 1326 |
+
python-versions = "<4.0,>=3.9"
|
| 1327 |
+
groups = ["main"]
|
| 1328 |
+
files = [
|
| 1329 |
+
{file = "langfuse-2.60.5-py3-none-any.whl", hash = "sha256:fd27d52017f36d6fa5ca652615213a2535dc93dd88c3375eeb811af26384d285"},
|
| 1330 |
+
{file = "langfuse-2.60.5.tar.gz", hash = "sha256:a33ecddc98cf6d12289372e63071b77b72230e7bc8260ee349f1465d53bf425b"},
|
| 1331 |
+
]
|
| 1332 |
+
|
| 1333 |
+
[package.dependencies]
|
| 1334 |
+
anyio = ">=4.4.0,<5.0.0"
|
| 1335 |
+
backoff = ">=1.10.0"
|
| 1336 |
+
httpx = ">=0.15.4,<1.0"
|
| 1337 |
+
idna = ">=3.7,<4.0"
|
| 1338 |
+
packaging = ">=23.2,<25.0"
|
| 1339 |
+
pydantic = ">=1.10.7,<3.0"
|
| 1340 |
+
requests = ">=2,<3"
|
| 1341 |
+
wrapt = ">=1.14,<2.0"
|
| 1342 |
+
|
| 1343 |
+
[package.extras]
|
| 1344 |
+
langchain = ["langchain (>=0.0.309)"]
|
| 1345 |
+
llama-index = ["llama-index (>=0.10.12,<2.0.0)"]
|
| 1346 |
+
openai = ["openai (>=0.27.8)"]
|
| 1347 |
+
|
| 1348 |
[[package]]
|
| 1349 |
name = "markdown-it-py"
|
| 1350 |
version = "3.0.0"
|
|
|
|
| 1441 |
{file = "markupsafe-3.0.2.tar.gz", hash = "sha256:ee55d3edf80167e48ea11a923c7386f4669df67d7994554387f84e7d8b0a2bf0"},
|
| 1442 |
]
|
| 1443 |
|
| 1444 |
+
[[package]]
|
| 1445 |
+
name = "mccabe"
|
| 1446 |
+
version = "0.7.0"
|
| 1447 |
+
description = "McCabe checker, plugin for flake8"
|
| 1448 |
+
optional = false
|
| 1449 |
+
python-versions = ">=3.6"
|
| 1450 |
+
groups = ["dev"]
|
| 1451 |
+
files = [
|
| 1452 |
+
{file = "mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e"},
|
| 1453 |
+
{file = "mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"},
|
| 1454 |
+
]
|
| 1455 |
+
|
| 1456 |
[[package]]
|
| 1457 |
name = "mdurl"
|
| 1458 |
version = "0.1.2"
|
|
|
|
| 1714 |
|
| 1715 |
[[package]]
|
| 1716 |
name = "opentelemetry-api"
|
| 1717 |
+
version = "1.33.1"
|
| 1718 |
description = "OpenTelemetry Python API"
|
| 1719 |
optional = false
|
| 1720 |
python-versions = ">=3.8"
|
| 1721 |
groups = ["main"]
|
| 1722 |
files = [
|
| 1723 |
+
{file = "opentelemetry_api-1.33.1-py3-none-any.whl", hash = "sha256:4db83ebcf7ea93e64637ec6ee6fabee45c5cbe4abd9cf3da95c43828ddb50b83"},
|
| 1724 |
+
{file = "opentelemetry_api-1.33.1.tar.gz", hash = "sha256:1c6055fc0a2d3f23a50c7e17e16ef75ad489345fd3df1f8b8af7c0bbf8a109e8"},
|
| 1725 |
]
|
| 1726 |
|
| 1727 |
[package.dependencies]
|
| 1728 |
deprecated = ">=1.2.6"
|
| 1729 |
importlib-metadata = ">=6.0,<8.7.0"
|
| 1730 |
|
| 1731 |
+
[[package]]
|
| 1732 |
+
name = "opentelemetry-exporter-otlp"
|
| 1733 |
+
version = "1.33.1"
|
| 1734 |
+
description = "OpenTelemetry Collector Exporters"
|
| 1735 |
+
optional = false
|
| 1736 |
+
python-versions = ">=3.8"
|
| 1737 |
+
groups = ["main"]
|
| 1738 |
+
files = [
|
| 1739 |
+
{file = "opentelemetry_exporter_otlp-1.33.1-py3-none-any.whl", hash = "sha256:9bcf1def35b880b55a49e31ebd63910edac14b294fd2ab884953c4deaff5b300"},
|
| 1740 |
+
{file = "opentelemetry_exporter_otlp-1.33.1.tar.gz", hash = "sha256:4d050311ea9486e3994575aa237e32932aad58330a31fba24fdba5c0d531cf04"},
|
| 1741 |
+
]
|
| 1742 |
+
|
| 1743 |
+
[package.dependencies]
|
| 1744 |
+
opentelemetry-exporter-otlp-proto-grpc = "1.33.1"
|
| 1745 |
+
opentelemetry-exporter-otlp-proto-http = "1.33.1"
|
| 1746 |
+
|
| 1747 |
[[package]]
|
| 1748 |
name = "opentelemetry-exporter-otlp-proto-common"
|
| 1749 |
+
version = "1.33.1"
|
| 1750 |
description = "OpenTelemetry Protobuf encoding"
|
| 1751 |
optional = false
|
| 1752 |
python-versions = ">=3.8"
|
| 1753 |
groups = ["main"]
|
| 1754 |
files = [
|
| 1755 |
+
{file = "opentelemetry_exporter_otlp_proto_common-1.33.1-py3-none-any.whl", hash = "sha256:b81c1de1ad349785e601d02715b2d29d6818aed2c809c20219f3d1f20b038c36"},
|
| 1756 |
+
{file = "opentelemetry_exporter_otlp_proto_common-1.33.1.tar.gz", hash = "sha256:c57b3fa2d0595a21c4ed586f74f948d259d9949b58258f11edb398f246bec131"},
|
| 1757 |
]
|
| 1758 |
|
| 1759 |
[package.dependencies]
|
| 1760 |
+
opentelemetry-proto = "1.33.1"
|
| 1761 |
|
| 1762 |
[[package]]
|
| 1763 |
name = "opentelemetry-exporter-otlp-proto-grpc"
|
| 1764 |
+
version = "1.33.1"
|
| 1765 |
description = "OpenTelemetry Collector Protobuf over gRPC Exporter"
|
| 1766 |
optional = false
|
| 1767 |
python-versions = ">=3.8"
|
| 1768 |
groups = ["main"]
|
| 1769 |
files = [
|
| 1770 |
+
{file = "opentelemetry_exporter_otlp_proto_grpc-1.33.1-py3-none-any.whl", hash = "sha256:7e8da32c7552b756e75b4f9e9c768a61eb47dee60b6550b37af541858d669ce1"},
|
| 1771 |
+
{file = "opentelemetry_exporter_otlp_proto_grpc-1.33.1.tar.gz", hash = "sha256:345696af8dc19785fac268c8063f3dc3d5e274c774b308c634f39d9c21955728"},
|
| 1772 |
]
|
| 1773 |
|
| 1774 |
[package.dependencies]
|
|
|
|
| 1776 |
googleapis-common-protos = ">=1.52,<2.0"
|
| 1777 |
grpcio = {version = ">=1.66.2,<2.0.0", markers = "python_version >= \"3.13\""}
|
| 1778 |
opentelemetry-api = ">=1.15,<2.0"
|
| 1779 |
+
opentelemetry-exporter-otlp-proto-common = "1.33.1"
|
| 1780 |
+
opentelemetry-proto = "1.33.1"
|
| 1781 |
+
opentelemetry-sdk = ">=1.33.1,<1.34.0"
|
| 1782 |
+
|
| 1783 |
+
[[package]]
|
| 1784 |
+
name = "opentelemetry-exporter-otlp-proto-http"
|
| 1785 |
+
version = "1.33.1"
|
| 1786 |
+
description = "OpenTelemetry Collector Protobuf over HTTP Exporter"
|
| 1787 |
+
optional = false
|
| 1788 |
+
python-versions = ">=3.8"
|
| 1789 |
+
groups = ["main"]
|
| 1790 |
+
files = [
|
| 1791 |
+
{file = "opentelemetry_exporter_otlp_proto_http-1.33.1-py3-none-any.whl", hash = "sha256:ebd6c523b89a2ecba0549adb92537cc2bf647b4ee61afbbd5a4c6535aa3da7cf"},
|
| 1792 |
+
{file = "opentelemetry_exporter_otlp_proto_http-1.33.1.tar.gz", hash = "sha256:46622d964a441acb46f463ebdc26929d9dec9efb2e54ef06acdc7305e8593c38"},
|
| 1793 |
+
]
|
| 1794 |
+
|
| 1795 |
+
[package.dependencies]
|
| 1796 |
+
deprecated = ">=1.2.6"
|
| 1797 |
+
googleapis-common-protos = ">=1.52,<2.0"
|
| 1798 |
+
opentelemetry-api = ">=1.15,<2.0"
|
| 1799 |
+
opentelemetry-exporter-otlp-proto-common = "1.33.1"
|
| 1800 |
+
opentelemetry-proto = "1.33.1"
|
| 1801 |
+
opentelemetry-sdk = ">=1.33.1,<1.34.0"
|
| 1802 |
+
requests = ">=2.7,<3.0"
|
| 1803 |
|
| 1804 |
[[package]]
|
| 1805 |
name = "opentelemetry-instrumentation"
|
| 1806 |
+
version = "0.54b1"
|
| 1807 |
description = "Instrumentation Tools & Auto Instrumentation for OpenTelemetry Python"
|
| 1808 |
optional = false
|
| 1809 |
python-versions = ">=3.8"
|
| 1810 |
groups = ["main"]
|
| 1811 |
files = [
|
| 1812 |
+
{file = "opentelemetry_instrumentation-0.54b1-py3-none-any.whl", hash = "sha256:a4ae45f4a90c78d7006c51524f57cd5aa1231aef031eae905ee34d5423f5b198"},
|
| 1813 |
+
{file = "opentelemetry_instrumentation-0.54b1.tar.gz", hash = "sha256:7658bf2ff914b02f246ec14779b66671508125c0e4227361e56b5ebf6cef0aec"},
|
| 1814 |
]
|
| 1815 |
|
| 1816 |
[package.dependencies]
|
| 1817 |
opentelemetry-api = ">=1.4,<2.0"
|
| 1818 |
+
opentelemetry-semantic-conventions = "0.54b1"
|
| 1819 |
packaging = ">=18.0"
|
| 1820 |
wrapt = ">=1.0.0,<2.0.0"
|
| 1821 |
|
| 1822 |
[[package]]
|
| 1823 |
name = "opentelemetry-instrumentation-asgi"
|
| 1824 |
+
version = "0.54b1"
|
| 1825 |
description = "ASGI instrumentation for OpenTelemetry"
|
| 1826 |
optional = false
|
| 1827 |
python-versions = ">=3.8"
|
| 1828 |
groups = ["main"]
|
| 1829 |
files = [
|
| 1830 |
+
{file = "opentelemetry_instrumentation_asgi-0.54b1-py3-none-any.whl", hash = "sha256:84674e822b89af563b283a5283c2ebb9ed585d1b80a1c27fb3ac20b562e9f9fc"},
|
| 1831 |
+
{file = "opentelemetry_instrumentation_asgi-0.54b1.tar.gz", hash = "sha256:ab4df9776b5f6d56a78413c2e8bbe44c90694c67c844a1297865dc1bd926ed3c"},
|
| 1832 |
]
|
| 1833 |
|
| 1834 |
[package.dependencies]
|
| 1835 |
asgiref = ">=3.0,<4.0"
|
| 1836 |
opentelemetry-api = ">=1.12,<2.0"
|
| 1837 |
+
opentelemetry-instrumentation = "0.54b1"
|
| 1838 |
+
opentelemetry-semantic-conventions = "0.54b1"
|
| 1839 |
+
opentelemetry-util-http = "0.54b1"
|
| 1840 |
|
| 1841 |
[package.extras]
|
| 1842 |
instruments = ["asgiref (>=3.0,<4.0)"]
|
| 1843 |
|
| 1844 |
[[package]]
|
| 1845 |
name = "opentelemetry-instrumentation-fastapi"
|
| 1846 |
+
version = "0.54b1"
|
| 1847 |
description = "OpenTelemetry FastAPI Instrumentation"
|
| 1848 |
optional = false
|
| 1849 |
python-versions = ">=3.8"
|
| 1850 |
groups = ["main"]
|
| 1851 |
files = [
|
| 1852 |
+
{file = "opentelemetry_instrumentation_fastapi-0.54b1-py3-none-any.whl", hash = "sha256:fb247781cfa75fd09d3d8713c65e4a02bd1e869b00e2c322cc516d4b5429860c"},
|
| 1853 |
+
{file = "opentelemetry_instrumentation_fastapi-0.54b1.tar.gz", hash = "sha256:1fcad19cef0db7092339b571a59e6f3045c9b58b7fd4670183f7addc459d78df"},
|
| 1854 |
]
|
| 1855 |
|
| 1856 |
[package.dependencies]
|
| 1857 |
opentelemetry-api = ">=1.12,<2.0"
|
| 1858 |
+
opentelemetry-instrumentation = "0.54b1"
|
| 1859 |
+
opentelemetry-instrumentation-asgi = "0.54b1"
|
| 1860 |
+
opentelemetry-semantic-conventions = "0.54b1"
|
| 1861 |
+
opentelemetry-util-http = "0.54b1"
|
| 1862 |
|
| 1863 |
[package.extras]
|
| 1864 |
instruments = ["fastapi (>=0.58,<1.0)"]
|
| 1865 |
|
| 1866 |
[[package]]
|
| 1867 |
name = "opentelemetry-proto"
|
| 1868 |
+
version = "1.33.1"
|
| 1869 |
description = "OpenTelemetry Python Proto"
|
| 1870 |
optional = false
|
| 1871 |
python-versions = ">=3.8"
|
| 1872 |
groups = ["main"]
|
| 1873 |
files = [
|
| 1874 |
+
{file = "opentelemetry_proto-1.33.1-py3-none-any.whl", hash = "sha256:243d285d9f29663fc7ea91a7171fcc1ccbbfff43b48df0774fd64a37d98eda70"},
|
| 1875 |
+
{file = "opentelemetry_proto-1.33.1.tar.gz", hash = "sha256:9627b0a5c90753bf3920c398908307063e4458b287bb890e5c1d6fa11ad50b68"},
|
| 1876 |
]
|
| 1877 |
|
| 1878 |
[package.dependencies]
|
|
|
|
| 1880 |
|
| 1881 |
[[package]]
|
| 1882 |
name = "opentelemetry-sdk"
|
| 1883 |
+
version = "1.33.1"
|
| 1884 |
description = "OpenTelemetry Python SDK"
|
| 1885 |
optional = false
|
| 1886 |
python-versions = ">=3.8"
|
| 1887 |
groups = ["main"]
|
| 1888 |
files = [
|
| 1889 |
+
{file = "opentelemetry_sdk-1.33.1-py3-none-any.whl", hash = "sha256:19ea73d9a01be29cacaa5d6c8ce0adc0b7f7b4d58cc52f923e4413609f670112"},
|
| 1890 |
+
{file = "opentelemetry_sdk-1.33.1.tar.gz", hash = "sha256:85b9fcf7c3d23506fbc9692fd210b8b025a1920535feec50bd54ce203d57a531"},
|
| 1891 |
]
|
| 1892 |
|
| 1893 |
[package.dependencies]
|
| 1894 |
+
opentelemetry-api = "1.33.1"
|
| 1895 |
+
opentelemetry-semantic-conventions = "0.54b1"
|
| 1896 |
typing-extensions = ">=3.7.4"
|
| 1897 |
|
| 1898 |
[[package]]
|
| 1899 |
name = "opentelemetry-semantic-conventions"
|
| 1900 |
+
version = "0.54b1"
|
| 1901 |
description = "OpenTelemetry Semantic Conventions"
|
| 1902 |
optional = false
|
| 1903 |
python-versions = ">=3.8"
|
| 1904 |
groups = ["main"]
|
| 1905 |
files = [
|
| 1906 |
+
{file = "opentelemetry_semantic_conventions-0.54b1-py3-none-any.whl", hash = "sha256:29dab644a7e435b58d3a3918b58c333c92686236b30f7891d5e51f02933ca60d"},
|
| 1907 |
+
{file = "opentelemetry_semantic_conventions-0.54b1.tar.gz", hash = "sha256:d1cecedae15d19bdaafca1e56b29a66aa286f50b5d08f036a145c7f3e9ef9cee"},
|
| 1908 |
]
|
| 1909 |
|
| 1910 |
[package.dependencies]
|
| 1911 |
deprecated = ">=1.2.6"
|
| 1912 |
+
opentelemetry-api = "1.33.1"
|
| 1913 |
|
| 1914 |
[[package]]
|
| 1915 |
name = "opentelemetry-util-http"
|
| 1916 |
+
version = "0.54b1"
|
| 1917 |
description = "Web util for OpenTelemetry"
|
| 1918 |
optional = false
|
| 1919 |
python-versions = ">=3.8"
|
| 1920 |
groups = ["main"]
|
| 1921 |
files = [
|
| 1922 |
+
{file = "opentelemetry_util_http-0.54b1-py3-none-any.whl", hash = "sha256:b1c91883f980344a1c3c486cffd47ae5c9c1dd7323f9cbe9fdb7cadb401c87c9"},
|
| 1923 |
+
{file = "opentelemetry_util_http-0.54b1.tar.gz", hash = "sha256:f0b66868c19fbaf9c9d4e11f4a7599fa15d5ea50b884967a26ccd9d72c7c9d15"},
|
| 1924 |
]
|
| 1925 |
|
| 1926 |
[[package]]
|
|
|
|
| 2015 |
|
| 2016 |
[[package]]
|
| 2017 |
name = "packaging"
|
| 2018 |
+
version = "24.2"
|
| 2019 |
description = "Core utilities for Python packages"
|
| 2020 |
optional = false
|
| 2021 |
python-versions = ">=3.8"
|
| 2022 |
groups = ["main"]
|
| 2023 |
files = [
|
| 2024 |
+
{file = "packaging-24.2-py3-none-any.whl", hash = "sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759"},
|
| 2025 |
+
{file = "packaging-24.2.tar.gz", hash = "sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f"},
|
| 2026 |
]
|
| 2027 |
|
| 2028 |
[[package]]
|
|
|
|
| 2208 |
typing = ["typing-extensions ; python_version < \"3.10\""]
|
| 2209 |
xmp = ["defusedxml"]
|
| 2210 |
|
| 2211 |
+
[[package]]
|
| 2212 |
+
name = "platformdirs"
|
| 2213 |
+
version = "4.3.8"
|
| 2214 |
+
description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`."
|
| 2215 |
+
optional = false
|
| 2216 |
+
python-versions = ">=3.9"
|
| 2217 |
+
groups = ["dev"]
|
| 2218 |
+
files = [
|
| 2219 |
+
{file = "platformdirs-4.3.8-py3-none-any.whl", hash = "sha256:ff7059bb7eb1179e2685604f4aaf157cfd9535242bd23742eadc3c13542139b4"},
|
| 2220 |
+
{file = "platformdirs-4.3.8.tar.gz", hash = "sha256:3d512d96e16bcb959a814c9f348431070822a6496326a4be0911c40b5a74c2bc"},
|
| 2221 |
+
]
|
| 2222 |
+
|
| 2223 |
+
[package.extras]
|
| 2224 |
+
docs = ["furo (>=2024.8.6)", "proselint (>=0.14)", "sphinx (>=8.1.3)", "sphinx-autodoc-typehints (>=3)"]
|
| 2225 |
+
test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=8.3.4)", "pytest-cov (>=6)", "pytest-mock (>=3.14)"]
|
| 2226 |
+
type = ["mypy (>=1.14.1)"]
|
| 2227 |
+
|
| 2228 |
[[package]]
|
| 2229 |
name = "posthog"
|
| 2230 |
version = "4.0.0"
|
|
|
|
| 2478 |
[package.extras]
|
| 2479 |
windows-terminal = ["colorama (>=0.4.6)"]
|
| 2480 |
|
| 2481 |
+
[[package]]
|
| 2482 |
+
name = "pylint"
|
| 2483 |
+
version = "3.3.7"
|
| 2484 |
+
description = "python code static checker"
|
| 2485 |
+
optional = false
|
| 2486 |
+
python-versions = ">=3.9.0"
|
| 2487 |
+
groups = ["dev"]
|
| 2488 |
+
files = [
|
| 2489 |
+
{file = "pylint-3.3.7-py3-none-any.whl", hash = "sha256:43860aafefce92fca4cf6b61fe199cdc5ae54ea28f9bf4cd49de267b5195803d"},
|
| 2490 |
+
{file = "pylint-3.3.7.tar.gz", hash = "sha256:2b11de8bde49f9c5059452e0c310c079c746a0a8eeaa789e5aa966ecc23e4559"},
|
| 2491 |
+
]
|
| 2492 |
+
|
| 2493 |
+
[package.dependencies]
|
| 2494 |
+
astroid = ">=3.3.8,<=3.4.0.dev0"
|
| 2495 |
+
colorama = {version = ">=0.4.5", markers = "sys_platform == \"win32\""}
|
| 2496 |
+
dill = {version = ">=0.3.7", markers = "python_version >= \"3.12\""}
|
| 2497 |
+
isort = ">=4.2.5,<5.13 || >5.13,<7"
|
| 2498 |
+
mccabe = ">=0.6,<0.8"
|
| 2499 |
+
platformdirs = ">=2.2"
|
| 2500 |
+
tomlkit = ">=0.10.1"
|
| 2501 |
+
|
| 2502 |
+
[package.extras]
|
| 2503 |
+
spelling = ["pyenchant (>=3.2,<4.0)"]
|
| 2504 |
+
testutils = ["gitpython (>3)"]
|
| 2505 |
+
|
| 2506 |
[[package]]
|
| 2507 |
name = "pymupdf"
|
| 2508 |
version = "1.25.5"
|
|
|
|
| 3128 |
description = "Style preserving TOML library"
|
| 3129 |
optional = false
|
| 3130 |
python-versions = ">=3.8"
|
| 3131 |
+
groups = ["main", "dev"]
|
| 3132 |
files = [
|
| 3133 |
{file = "tomlkit-0.13.2-py3-none-any.whl", hash = "sha256:7a974427f6e119197f670fbbbeae7bef749a6c14e793db934baefc1b5f03efde"},
|
| 3134 |
{file = "tomlkit-0.13.2.tar.gz", hash = "sha256:fff5fe59a87295b278abd31bec92c15d9bc4a06885ab12bcea52c71119392e79"},
|
|
|
|
| 3613 |
|
| 3614 |
[metadata]
|
| 3615 |
lock-version = "2.1"
|
| 3616 |
+
python-versions = ">=3.13,<4.0"
|
| 3617 |
+
content-hash = "72ca03bb7d298a295843d749afd4a8b26e6be77edea422880a5a00bcecaa96ef"
|
pyproject.toml
CHANGED
|
@@ -8,7 +8,7 @@ authors = [
|
|
| 8 |
]
|
| 9 |
license = {text = "Apache 2.0"}
|
| 10 |
readme = "README.md"
|
| 11 |
-
requires-python = ">=3.13"
|
| 12 |
dependencies = [
|
| 13 |
"google-genai (==1.11.0)",
|
| 14 |
"chromadb (==1.0.5)",
|
|
@@ -19,9 +19,17 @@ dependencies = [
|
|
| 19 |
"rtree (>=1.4.0,<2.0.0)",
|
| 20 |
"pymupdf (>=1.25.5,<2.0.0)",
|
| 21 |
"gradio (>=5.27.1,<6.0.0)",
|
|
|
|
|
|
|
|
|
|
|
|
|
| 22 |
]
|
| 23 |
|
| 24 |
|
| 25 |
[build-system]
|
| 26 |
requires = ["poetry-core>=2.0.0,<3.0.0"]
|
| 27 |
build-backend = "poetry.core.masonry.api"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8 |
]
|
| 9 |
license = {text = "Apache 2.0"}
|
| 10 |
readme = "README.md"
|
| 11 |
+
requires-python = ">=3.13,<4.0"
|
| 12 |
dependencies = [
|
| 13 |
"google-genai (==1.11.0)",
|
| 14 |
"chromadb (==1.0.5)",
|
|
|
|
| 19 |
"rtree (>=1.4.0,<2.0.0)",
|
| 20 |
"pymupdf (>=1.25.5,<2.0.0)",
|
| 21 |
"gradio (>=5.27.1,<6.0.0)",
|
| 22 |
+
"langfuse (>=2.60.5,<3.0.0)",
|
| 23 |
+
"opentelemetry-api (>=1.33.1,<2.0.0)",
|
| 24 |
+
"opentelemetry-sdk (>=1.33.1,<2.0.0)",
|
| 25 |
+
"opentelemetry-exporter-otlp (>=1.33.1,<2.0.0)",
|
| 26 |
]
|
| 27 |
|
| 28 |
|
| 29 |
[build-system]
|
| 30 |
requires = ["poetry-core>=2.0.0,<3.0.0"]
|
| 31 |
build-backend = "poetry.core.masonry.api"
|
| 32 |
+
|
| 33 |
+
[tool.poetry.group.dev.dependencies]
|
| 34 |
+
pylint = "^3.3.7"
|
| 35 |
+
|
ragntex.py
CHANGED
|
@@ -1,3 +1,8 @@
|
|
| 1 |
-
|
| 2 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3 |
demo.launch()
|
|
|
|
| 1 |
+
"""RAGnTex: A Retrieval-Augmented Generation for LaTeX Documents"""
|
| 2 |
|
| 3 |
+
from src import demo, init_telemetry
|
| 4 |
+
|
| 5 |
+
# Setup OpenTelemetry
|
| 6 |
+
init_telemetry()
|
| 7 |
+
# Run Gradio
|
| 8 |
demo.launch()
|
src/__init__.py
CHANGED
|
@@ -1,22 +1,9 @@
|
|
| 1 |
-
|
| 2 |
-
# from .processing import process_documents, save_pdf_images, save_pdf_figures, get_prompt
|
| 3 |
-
# from .compilation import CompilePresentation
|
| 4 |
-
# from .services import client, generate_presentation
|
| 5 |
-
# from .utils import upload_files, Logger
|
| 6 |
|
| 7 |
from .interface import demo
|
|
|
|
| 8 |
|
| 9 |
__all__ = [
|
| 10 |
-
# "db",
|
| 11 |
-
# "embed_fn",
|
| 12 |
-
# "process_documents",
|
| 13 |
-
# "save_pdf_images",
|
| 14 |
-
# "save_pdf_figures",
|
| 15 |
-
# "get_prompt",
|
| 16 |
-
# "CompilePresentation",
|
| 17 |
-
# "client",
|
| 18 |
-
# "upload_files",
|
| 19 |
-
# "Logger",
|
| 20 |
"demo",
|
| 21 |
-
|
| 22 |
]
|
|
|
|
| 1 |
+
"""Initialize the package by exposing the main interface and telemetry setup."""
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2 |
|
| 3 |
from .interface import demo
|
| 4 |
+
from .telemetry import init_telemetry
|
| 5 |
|
| 6 |
__all__ = [
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7 |
"demo",
|
| 8 |
+
"init_telemetry",
|
| 9 |
]
|
src/compilation/__init__.py
CHANGED
|
@@ -1,3 +1,5 @@
|
|
| 1 |
-
|
| 2 |
|
| 3 |
-
|
|
|
|
|
|
|
|
|
| 1 |
+
"""This module initializes the latex compilation package."""
|
| 2 |
|
| 3 |
+
from .latex_compilation import compile_presentation
|
| 4 |
+
|
| 5 |
+
__all__ = ["compile_presentation"]
|
src/compilation/latex_compilation.py
CHANGED
|
@@ -1,12 +1,19 @@
|
|
| 1 |
-
|
|
|
|
| 2 |
import os
|
| 3 |
import subprocess
|
| 4 |
-
from ..
|
| 5 |
|
| 6 |
LOGGER = Logger.get_logger()
|
| 7 |
|
| 8 |
|
| 9 |
-
def
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 10 |
# Remove possible Markdown wrapping around the output
|
| 11 |
if latex_code.startswith("```latex"):
|
| 12 |
latex_code = latex_code.split("\n", 1)[1] # Remove the first line
|
|
@@ -18,7 +25,6 @@ def CompilePresentation(latex_code, work_dir):
|
|
| 18 |
with open(tex_file, "w") as f:
|
| 19 |
f.write(latex_code)
|
| 20 |
|
| 21 |
-
LOGGER.info("=" * 100)
|
| 22 |
LOGGER.info("📄 Files in the directory: %s", work_dir)
|
| 23 |
|
| 24 |
# List files in the directory using Python (instead of `!ls`)
|
|
@@ -28,27 +34,35 @@ def CompilePresentation(latex_code, work_dir):
|
|
| 28 |
# Compile with pdflatex (using subprocess instead of `!`)
|
| 29 |
os.chdir(work_dir) # Change working directory
|
| 30 |
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
LOGGER.error(
|
|
|
|
|
|
|
| 53 |
else:
|
| 54 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Module for compiling LaTeX presentations into PDF format."""
|
| 2 |
+
|
| 3 |
import os
|
| 4 |
import subprocess
|
| 5 |
+
from ..telemetry import Logger, traced_block
|
| 6 |
|
| 7 |
LOGGER = Logger.get_logger()
|
| 8 |
|
| 9 |
|
| 10 |
+
def compile_presentation(latex_code, work_dir) -> None:
|
| 11 |
+
"""Compile LaTeX code into a PDF presentation.
|
| 12 |
+
Args:
|
| 13 |
+
latex_code (str): LaTeX code to be compiled.
|
| 14 |
+
work_dir (str): Directory where the LaTeX file will be saved and compiled.
|
| 15 |
+
"""
|
| 16 |
+
LOGGER.info("Compiling presentation...")
|
| 17 |
# Remove possible Markdown wrapping around the output
|
| 18 |
if latex_code.startswith("```latex"):
|
| 19 |
latex_code = latex_code.split("\n", 1)[1] # Remove the first line
|
|
|
|
| 25 |
with open(tex_file, "w") as f:
|
| 26 |
f.write(latex_code)
|
| 27 |
|
|
|
|
| 28 |
LOGGER.info("📄 Files in the directory: %s", work_dir)
|
| 29 |
|
| 30 |
# List files in the directory using Python (instead of `!ls`)
|
|
|
|
| 34 |
# Compile with pdflatex (using subprocess instead of `!`)
|
| 35 |
os.chdir(work_dir) # Change working directory
|
| 36 |
|
| 37 |
+
with traced_block("🧱 compile_presentation") as span:
|
| 38 |
+
try:
|
| 39 |
+
# Run pdflatex twice to ensure proper slide enumeration
|
| 40 |
+
subprocess.run(
|
| 41 |
+
["pdflatex", "-interaction=nonstopmode", "presentation.tex"],
|
| 42 |
+
check=True,
|
| 43 |
+
stdout=subprocess.PIPE,
|
| 44 |
+
stderr=subprocess.PIPE,
|
| 45 |
+
)
|
| 46 |
+
subprocess.run(
|
| 47 |
+
["pdflatex", "-interaction=nonstopmode", "presentation.tex"],
|
| 48 |
+
check=True,
|
| 49 |
+
stdout=subprocess.PIPE,
|
| 50 |
+
stderr=subprocess.PIPE,
|
| 51 |
+
)
|
| 52 |
+
|
| 53 |
+
except subprocess.CalledProcessError as e:
|
| 54 |
+
# Handle error in case of failed LaTeX compilation
|
| 55 |
+
span.set_attribute("pdf.success", False)
|
| 56 |
+
span.set_attribute("pdf.error_log", e.stderr.decode())
|
| 57 |
+
span.record_exception(e)
|
| 58 |
+
LOGGER.error(
|
| 59 |
+
"❌ PDF generation failed. Here's the log: %s", e.stderr.decode()
|
| 60 |
+
)
|
| 61 |
else:
|
| 62 |
+
# Check for PDF output
|
| 63 |
+
if not os.path.exists("presentation.pdf"):
|
| 64 |
+
span.set_attribute("pdf.success", False)
|
| 65 |
+
LOGGER.error("❌ PDF generation failed. No PDF file found.")
|
| 66 |
+
else:
|
| 67 |
+
span.set_attribute("pdf.success", True)
|
| 68 |
+
LOGGER.info("💾 PDF generated successfully in: %s", work_dir)
|
src/database/__init__.py
CHANGED
|
@@ -1,4 +1,7 @@
|
|
|
|
|
|
|
|
| 1 |
from .database import db, embed_fn
|
| 2 |
from .ingest import ingest_files_to_db
|
|
|
|
| 3 |
|
| 4 |
-
__all__ = ["db", "embed_fn", "ingest_files_to_db"]
|
|
|
|
| 1 |
+
"""This module initializes the database package."""
|
| 2 |
+
|
| 3 |
from .database import db, embed_fn
|
| 4 |
from .ingest import ingest_files_to_db
|
| 5 |
+
from .retrieve import retrive_files_from_db
|
| 6 |
|
| 7 |
+
__all__ = ["db", "embed_fn", "ingest_files_to_db", "retrive_files_from_db"]
|
src/database/database.py
CHANGED
|
@@ -1,4 +1,5 @@
|
|
| 1 |
-
|
|
|
|
| 2 |
from google.api_core import retry
|
| 3 |
from google.genai import types
|
| 4 |
from google import genai
|
|
@@ -12,6 +13,8 @@ is_retriable = lambda e: (isinstance(e, genai.errors.APIError) and e.code in {42
|
|
| 12 |
|
| 13 |
|
| 14 |
class GeminiEmbeddingFunction(EmbeddingFunction):
|
|
|
|
|
|
|
| 15 |
def __init__(self):
|
| 16 |
self.document_mode = True
|
| 17 |
|
|
|
|
| 1 |
+
"""Database module for user-uploaded files."""
|
| 2 |
+
|
| 3 |
from google.api_core import retry
|
| 4 |
from google.genai import types
|
| 5 |
from google import genai
|
|
|
|
| 13 |
|
| 14 |
|
| 15 |
class GeminiEmbeddingFunction(EmbeddingFunction):
|
| 16 |
+
"""A class to handle retrieval embedding functions using Google Gemini API."""
|
| 17 |
+
|
| 18 |
def __init__(self):
|
| 19 |
self.document_mode = True
|
| 20 |
|
src/database/ingest.py
CHANGED
|
@@ -1,8 +1,10 @@
|
|
| 1 |
-
|
|
|
|
| 2 |
from pathlib import Path
|
| 3 |
from .database import db
|
| 4 |
-
from ..
|
| 5 |
from ..processing import process_documents
|
|
|
|
| 6 |
|
| 7 |
LOGGER = Logger.get_logger()
|
| 8 |
|
|
@@ -31,11 +33,14 @@ def ingest_files_to_db(pdf_files) -> None:
|
|
| 31 |
# Process new files only
|
| 32 |
documents, metadatas = process_documents(new_pdfs)
|
| 33 |
|
| 34 |
-
LOGGER.info(
|
| 35 |
if documents:
|
| 36 |
db.add(
|
| 37 |
documents=documents,
|
| 38 |
ids=[str(i) for i in range(len(documents))],
|
| 39 |
metadatas=metadatas,
|
| 40 |
)
|
|
|
|
|
|
|
|
|
|
| 41 |
LOGGER.info("Documents ingested successfully.")
|
|
|
|
| 1 |
+
"""Ingest module for adding new PDF files to the database."""
|
| 2 |
+
|
| 3 |
from pathlib import Path
|
| 4 |
from .database import db
|
| 5 |
+
from ..telemetry import Logger
|
| 6 |
from ..processing import process_documents
|
| 7 |
+
from ..telemetry import traced_block
|
| 8 |
|
| 9 |
LOGGER = Logger.get_logger()
|
| 10 |
|
|
|
|
| 33 |
# Process new files only
|
| 34 |
documents, metadatas = process_documents(new_pdfs)
|
| 35 |
|
| 36 |
+
LOGGER.info("Adding %d new PDFs to the database...", len(new_pdfs))
|
| 37 |
if documents:
|
| 38 |
db.add(
|
| 39 |
documents=documents,
|
| 40 |
ids=[str(i) for i in range(len(documents))],
|
| 41 |
metadatas=metadatas,
|
| 42 |
)
|
| 43 |
+
with traced_block("📑 injest_docs_to_db") as span:
|
| 44 |
+
span.set_attribute("input.n_documents", len(documents))
|
| 45 |
+
span.set_attribute("output.n_documents_added", len(documents))
|
| 46 |
LOGGER.info("Documents ingested successfully.")
|
src/database/retrieve.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from .database import db, embed_fn
|
| 2 |
+
from ..telemetry import Logger, traced_block
|
| 3 |
+
|
| 4 |
+
LOGGER = Logger.get_logger()
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
def retrive_files_from_db(topic: str) -> tuple[list[str], list[dict]]:
|
| 8 |
+
"""Retrieve relevant documents from the database based on the provided topic.
|
| 9 |
+
Args:
|
| 10 |
+
topic (str): The topic for which to retrieve documents.
|
| 11 |
+
Returns:
|
| 12 |
+
tuple: A 2-element tuple:
|
| 13 |
+
- list[str]: List of retrieved documents.
|
| 14 |
+
- list[dict]: List of metadata associated with the documents.
|
| 15 |
+
"""
|
| 16 |
+
LOGGER.info("Retrieving relevant documents...")
|
| 17 |
+
embed_fn.document_mode = False
|
| 18 |
+
query_oneline = topic.replace("\n", " ")
|
| 19 |
+
with traced_block("🔎 retrieve_documents") as span:
|
| 20 |
+
span.set_attribute("input.query_oneline", query_oneline)
|
| 21 |
+
n_yelded_docs = 2
|
| 22 |
+
result = db.query(query_texts=[query_oneline], n_results=n_yelded_docs)
|
| 23 |
+
span.set_attribute("output.n_results", n_yelded_docs)
|
| 24 |
+
span.set_attribute("output.metadatas", result["metadatas"])
|
| 25 |
+
[documents] = result["documents"]
|
| 26 |
+
[metadatas] = result["metadatas"]
|
| 27 |
+
|
| 28 |
+
return [documents], [metadatas]
|
src/interface/__init__.py
CHANGED
|
@@ -1,3 +1,6 @@
|
|
|
|
|
|
|
|
| 1 |
from .gradio_interface import demo
|
|
|
|
| 2 |
|
| 3 |
-
__all__ = ["demo"]
|
|
|
|
| 1 |
+
"""This module initializes the UI package."""
|
| 2 |
+
|
| 3 |
from .gradio_interface import demo
|
| 4 |
+
from .upload_files import upload_files
|
| 5 |
|
| 6 |
+
__all__ = ["upload_files", "demo"]
|
src/interface/gradio_interface.py
CHANGED
|
@@ -1,15 +1,22 @@
|
|
| 1 |
# interface/gradio_interface.py
|
| 2 |
|
| 3 |
import gradio as gr
|
| 4 |
-
from .
|
| 5 |
from ..services import generate_presentation
|
| 6 |
|
| 7 |
|
| 8 |
-
def upload_and_update_list(files, uploaded_list):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9 |
status, new_paths = upload_files(files)
|
| 10 |
updated_list = uploaded_list + [p for p in new_paths if p not in uploaded_list]
|
| 11 |
-
|
| 12 |
-
return status, updated_list # , file_list_display
|
| 13 |
|
| 14 |
|
| 15 |
def generate_iframe(file_path):
|
|
|
|
| 1 |
# interface/gradio_interface.py
|
| 2 |
|
| 3 |
import gradio as gr
|
| 4 |
+
from .upload_files import upload_files
|
| 5 |
from ..services import generate_presentation
|
| 6 |
|
| 7 |
|
| 8 |
+
def upload_and_update_list(files: list, uploaded_list) -> tuple[str, list[str]]:
|
| 9 |
+
"""Helper function to handle uploaded documents and update the list of uploaded files.
|
| 10 |
+
Args:
|
| 11 |
+
files (list): List of file-like objects to be uploaded.
|
| 12 |
+
uploaded_list (list): Current list of uploaded file paths.
|
| 13 |
+
Returns:
|
| 14 |
+
tuple: A 2-element tuple:
|
| 15 |
+
- str: Status message indicating the result of the upload operation.
|
| 16 |
+
- list[str]: Updated list of uploaded file paths."""
|
| 17 |
status, new_paths = upload_files(files)
|
| 18 |
updated_list = uploaded_list + [p for p in new_paths if p not in uploaded_list]
|
| 19 |
+
return status, updated_list
|
|
|
|
| 20 |
|
| 21 |
|
| 22 |
def generate_iframe(file_path):
|
src/{utils/gradio_user_settings.py → interface/upload_files.py}
RENAMED
|
@@ -1,13 +1,20 @@
|
|
| 1 |
-
|
| 2 |
|
| 3 |
from pathlib import Path
|
| 4 |
import shutil
|
| 5 |
-
from .logging_utils import Logger
|
| 6 |
|
| 7 |
LOGGER = Logger.get_logger()
|
| 8 |
|
| 9 |
|
| 10 |
-
def upload_files(files):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 11 |
if not files:
|
| 12 |
return "No files selected."
|
| 13 |
|
|
|
|
| 1 |
+
"""Module to handle file uploads via UI."""
|
| 2 |
|
| 3 |
from pathlib import Path
|
| 4 |
import shutil
|
| 5 |
+
from ..telemetry.logging_utils import Logger
|
| 6 |
|
| 7 |
LOGGER = Logger.get_logger()
|
| 8 |
|
| 9 |
|
| 10 |
+
def upload_files(files: list) -> tuple[str, list[str]]:
|
| 11 |
+
"""Upload files and save them in the temporary directory.
|
| 12 |
+
Args:
|
| 13 |
+
files (list): List of file-like objects to be uploaded.
|
| 14 |
+
Returns:
|
| 15 |
+
str: Message indicating the result of the upload operation.
|
| 16 |
+
list[str]: List of saved file paths.
|
| 17 |
+
"""
|
| 18 |
if not files:
|
| 19 |
return "No files selected."
|
| 20 |
|
src/processing/__init__.py
CHANGED
|
@@ -1,5 +1,16 @@
|
|
| 1 |
-
|
| 2 |
-
|
|
|
|
|
|
|
|
|
|
| 3 |
from .prompt import get_prompt
|
| 4 |
|
| 5 |
-
__all__ = [
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""This module initializes the document processing package."""
|
| 2 |
+
|
| 3 |
+
from .document_processing import process_documents, delete_uploaded_files
|
| 4 |
+
from .images_processing import save_pdf_images, save_pdf_figures, find_used_fgx
|
| 5 |
+
from .output_folder import create_output_folder
|
| 6 |
from .prompt import get_prompt
|
| 7 |
|
| 8 |
+
__all__ = [
|
| 9 |
+
"process_documents",
|
| 10 |
+
"delete_uploaded_files",
|
| 11 |
+
"save_pdf_images",
|
| 12 |
+
"save_pdf_figures",
|
| 13 |
+
"find_used_fgx",
|
| 14 |
+
"create_output_folder",
|
| 15 |
+
"get_prompt",
|
| 16 |
+
]
|
src/processing/document_processing.py
CHANGED
|
@@ -1,10 +1,14 @@
|
|
| 1 |
-
|
|
|
|
| 2 |
import re
|
| 3 |
-
import fitz
|
| 4 |
import os
|
| 5 |
-
import
|
|
|
|
| 6 |
|
| 7 |
from .images_processing import extract_images, extract_vector
|
|
|
|
|
|
|
|
|
|
| 8 |
|
| 9 |
|
| 10 |
def extract_pdf_content(pdf_path: str):
|
|
@@ -67,3 +71,18 @@ def process_documents(pdf_files):
|
|
| 67 |
metadatas.append(fixed_metadata)
|
| 68 |
|
| 69 |
return documents, metadatas
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Module handling PDF document processing and content extraction."""
|
| 2 |
+
|
| 3 |
import re
|
|
|
|
| 4 |
import os
|
| 5 |
+
from pathlib import Path
|
| 6 |
+
import fitz
|
| 7 |
|
| 8 |
from .images_processing import extract_images, extract_vector
|
| 9 |
+
from ..telemetry import Logger
|
| 10 |
+
|
| 11 |
+
LOGGER = Logger.get_logger()
|
| 12 |
|
| 13 |
|
| 14 |
def extract_pdf_content(pdf_path: str):
|
|
|
|
| 71 |
metadatas.append(fixed_metadata)
|
| 72 |
|
| 73 |
return documents, metadatas
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def delete_uploaded_files(uploaded_files):
|
| 77 |
+
"""Delete uploaded files after processing."""
|
| 78 |
+
for file in uploaded_files:
|
| 79 |
+
try:
|
| 80 |
+
path = Path(file)
|
| 81 |
+
if path.exists():
|
| 82 |
+
path.unlink() # Delete the file
|
| 83 |
+
LOGGER.info(f"Deleted file: {file}")
|
| 84 |
+
else:
|
| 85 |
+
LOGGER.warning(f"File not found for deletion: {file}")
|
| 86 |
+
LOGGER.info("🧹 Cleaned up uploaded files.")
|
| 87 |
+
except Exception as e:
|
| 88 |
+
LOGGER.warning(f"Failed to delete {file}", exc_info=e)
|
src/processing/images_processing.py
CHANGED
|
@@ -1,13 +1,19 @@
|
|
| 1 |
-
|
| 2 |
-
|
| 3 |
import re
|
| 4 |
import os
|
| 5 |
import hashlib
|
| 6 |
from collections import defaultdict
|
|
|
|
| 7 |
from rtree import index
|
| 8 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9 |
|
| 10 |
def find_image_caption(page, image_bbox, max_distance=100):
|
|
|
|
| 11 |
# Get all text blocks on the page
|
| 12 |
blocks = page.get_text("dict")["blocks"]
|
| 13 |
image_bottom = image_bbox.y1
|
|
@@ -56,6 +62,7 @@ def find_image_caption(page, image_bbox, max_distance=100):
|
|
| 56 |
|
| 57 |
|
| 58 |
def extract_images(pdf, doc, page, page_num):
|
|
|
|
| 59 |
images = page.get_images(full=True)
|
| 60 |
|
| 61 |
imgs = []
|
|
@@ -210,7 +217,6 @@ def extract_vector(pdf, doc, page, page_num):
|
|
| 210 |
min_size = page_size * MIN_SIZE
|
| 211 |
max_size = page_size * MAX_SIZE
|
| 212 |
|
| 213 |
-
all_text = page.get_text()
|
| 214 |
drawings = page.get_drawings()
|
| 215 |
|
| 216 |
# Group drawings into figures
|
|
@@ -268,6 +274,7 @@ def extract_vector(pdf, doc, page, page_num):
|
|
| 268 |
|
| 269 |
|
| 270 |
def save_pdf_images(pdf_path: str, req_imgs: list, images_dir: str):
|
|
|
|
| 271 |
doc = fitz.open(pdf_path)
|
| 272 |
pdf = os.path.splitext(os.path.basename(pdf_path))[0]
|
| 273 |
|
|
@@ -328,7 +335,6 @@ def save_pdf_figures(pdf_path: str, req_figs: list, figures_dir: str):
|
|
| 328 |
min_size = page_size * MIN_SIZE
|
| 329 |
max_size = page_size * MAX_SIZE
|
| 330 |
|
| 331 |
-
all_text = page.get_text()
|
| 332 |
drawings = page.get_drawings()
|
| 333 |
|
| 334 |
# Group drawings into figures
|
|
@@ -370,3 +376,53 @@ def save_pdf_figures(pdf_path: str, req_figs: list, figures_dir: str):
|
|
| 370 |
f.write(figure_bytes)
|
| 371 |
|
| 372 |
return True
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Module handling images exrtraction and processing from the uploaded PDF documents."""
|
| 2 |
+
|
| 3 |
import re
|
| 4 |
import os
|
| 5 |
import hashlib
|
| 6 |
from collections import defaultdict
|
| 7 |
+
import fitz
|
| 8 |
from rtree import index
|
| 9 |
|
| 10 |
+
from ..telemetry import Logger
|
| 11 |
+
|
| 12 |
+
LOGGER = Logger.get_logger()
|
| 13 |
+
|
| 14 |
|
| 15 |
def find_image_caption(page, image_bbox, max_distance=100):
|
| 16 |
+
"""Method to find and extract image caption based on the image bounding box."""
|
| 17 |
# Get all text blocks on the page
|
| 18 |
blocks = page.get_text("dict")["blocks"]
|
| 19 |
image_bottom = image_bbox.y1
|
|
|
|
| 62 |
|
| 63 |
|
| 64 |
def extract_images(pdf, doc, page, page_num):
|
| 65 |
+
"""Extract images from a PDF page and classify them based on their aspect ratio."""
|
| 66 |
images = page.get_images(full=True)
|
| 67 |
|
| 68 |
imgs = []
|
|
|
|
| 217 |
min_size = page_size * MIN_SIZE
|
| 218 |
max_size = page_size * MAX_SIZE
|
| 219 |
|
|
|
|
| 220 |
drawings = page.get_drawings()
|
| 221 |
|
| 222 |
# Group drawings into figures
|
|
|
|
| 274 |
|
| 275 |
|
| 276 |
def save_pdf_images(pdf_path: str, req_imgs: list, images_dir: str):
|
| 277 |
+
"""Save images used in the presentation to the specified directory."""
|
| 278 |
doc = fitz.open(pdf_path)
|
| 279 |
pdf = os.path.splitext(os.path.basename(pdf_path))[0]
|
| 280 |
|
|
|
|
| 335 |
min_size = page_size * MIN_SIZE
|
| 336 |
max_size = page_size * MAX_SIZE
|
| 337 |
|
|
|
|
| 338 |
drawings = page.get_drawings()
|
| 339 |
|
| 340 |
# Group drawings into figures
|
|
|
|
| 376 |
f.write(figure_bytes)
|
| 377 |
|
| 378 |
return True
|
| 379 |
+
|
| 380 |
+
|
| 381 |
+
def find_used_fgx(answer, work_dir: str, metadatas: list):
|
| 382 |
+
"""Finds and saves the figures used in the LLM output to the gfx directory
|
| 383 |
+
where the presentation will be compiled.
|
| 384 |
+
Args:
|
| 385 |
+
answer (str): The LLM answer containing names of the figures.
|
| 386 |
+
work_dir (str): The working directory where the presentation will be compiled.
|
| 387 |
+
metadatas (list): List of metadata dictionaries containing PDF paths.
|
| 388 |
+
"""
|
| 389 |
+
graphics_dir = os.path.join(work_dir, "gfx")
|
| 390 |
+
os.makedirs(graphics_dir, exist_ok=True)
|
| 391 |
+
LOGGER.info("🌄 Images will be saved to: %s", graphics_dir)
|
| 392 |
+
|
| 393 |
+
# Find images, which are used in the presentation
|
| 394 |
+
pattern_img = re.compile(
|
| 395 |
+
r"doc(?P<doc>[a-zA-Z0-9_]+)_page(?P<page>\d+)_img(?P<img>\d+)_hash(?P<hash>[a-fA-F0-9]{8})\.png"
|
| 396 |
+
)
|
| 397 |
+
matches_img = pattern_img.finditer(answer.text)
|
| 398 |
+
|
| 399 |
+
req_imgs = []
|
| 400 |
+
for match in matches_img:
|
| 401 |
+
req_img = {
|
| 402 |
+
"doc": match.group("doc"),
|
| 403 |
+
"page": int(match.group("page")),
|
| 404 |
+
"img": int(match.group("img")),
|
| 405 |
+
"hash": match.group("hash"),
|
| 406 |
+
}
|
| 407 |
+
req_imgs.append(req_img)
|
| 408 |
+
|
| 409 |
+
# Find figures, which are used in the presentation
|
| 410 |
+
pattern_fig = re.compile(
|
| 411 |
+
r"doc(?P<doc>[a-zA-Z0-9_]+)_page(?P<page>\d+)_fig(?P<fig>\d+)_hash(?P<hash>[a-fA-F0-9]{8})\.png"
|
| 412 |
+
)
|
| 413 |
+
matches_fig = pattern_fig.finditer(answer.text)
|
| 414 |
+
|
| 415 |
+
req_figs = []
|
| 416 |
+
for match in matches_fig:
|
| 417 |
+
req_fig = {
|
| 418 |
+
"doc": match.group("doc"),
|
| 419 |
+
"page": int(match.group("page")),
|
| 420 |
+
"fig": int(match.group("fig")),
|
| 421 |
+
"hash": match.group("hash"),
|
| 422 |
+
}
|
| 423 |
+
req_figs.append(req_fig)
|
| 424 |
+
|
| 425 |
+
# Save the required graphics
|
| 426 |
+
for metadata in metadatas:
|
| 427 |
+
save_pdf_images(metadata["pdf_path"], req_imgs, graphics_dir)
|
| 428 |
+
save_pdf_figures(metadata["pdf_path"], req_figs, graphics_dir)
|
src/processing/output_folder.py
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Module to create an output folder for saving results."""
|
| 2 |
+
|
| 3 |
+
import os
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
from datetime import datetime
|
| 6 |
+
from ..telemetry import Logger
|
| 7 |
+
|
| 8 |
+
LOGGER = Logger.get_logger()
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
def create_output_folder() -> str:
|
| 12 |
+
"""Create a new folder where the output will be saved.
|
| 13 |
+
Returns:
|
| 14 |
+
str: Path to the newly created output folder.
|
| 15 |
+
"""
|
| 16 |
+
LOGGER.info("💾 Creating new subfolder...")
|
| 17 |
+
cwd = Path.cwd()
|
| 18 |
+
one_level_up = cwd.parent
|
| 19 |
+
base_path = one_level_up / "output"
|
| 20 |
+
timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
|
| 21 |
+
work_dir = os.path.join(base_path, timestamp)
|
| 22 |
+
print("work_dir", work_dir)
|
| 23 |
+
os.makedirs(work_dir, exist_ok=False)
|
| 24 |
+
LOGGER.info("📁 Created folder: %s", work_dir)
|
| 25 |
+
return work_dir
|
src/processing/prompt.py
CHANGED
|
@@ -1,4 +1,6 @@
|
|
| 1 |
-
|
|
|
|
|
|
|
| 2 |
def get_prompt(
|
| 3 |
presentation_theme: str = "default", color_theme: str = "default"
|
| 4 |
) -> str:
|
|
@@ -7,19 +9,27 @@ def get_prompt(
|
|
| 7 |
Args:
|
| 8 |
presentation_theme (str): The theme for the presentation, default is "default".
|
| 9 |
color_theme (str): The color theme for the presentation, default is "default".
|
|
|
|
|
|
|
|
|
|
| 10 |
"""
|
| 11 |
-
prompt = f"""You are a presentation assistant that creates clear, concise, and engaging slide decks from the
|
|
|
|
|
|
|
| 12 |
|
| 13 |
Structure your slides as follows:
|
| 14 |
1. **Introduction**: Present the topic and explain why it's important or interesting.
|
| 15 |
-
2. **Main Content**: Break down the topic into 2–4 core ideas, one idea per slide. Explain each with clear
|
|
|
|
| 16 |
3. **Summary**: Recap the key takeaways, what was learned, and what it means for the audience.
|
| 17 |
|
| 18 |
-
Use friendly and accessible language that anyone can understand. Avoid technical jargon unless it's
|
|
|
|
| 19 |
|
| 20 |
Output only valid LaTeX Beamer code. Each slide must be defined using \\begin{{frame}} ... \\end{{frame}}.
|
| 21 |
|
| 22 |
-
Here is some reference content retrieved from a document. Please generate a LaTeX Beamer presentation based on
|
|
|
|
| 23 |
|
| 24 |
1. Introduction (what is the topic and why it matters)
|
| 25 |
2. Main Part (a few slides on the core ideas)
|
|
@@ -90,7 +100,8 @@ def get_prompt(
|
|
| 90 |
|
| 91 |
\\end{{document}}
|
| 92 |
|
| 93 |
-
You must use **at least one image** when creating a presentation. In your output, include `\\includegraphics`
|
|
|
|
| 94 |
|
| 95 |
Prioritize the slide structure of **Core Idea 2** (two-column layout) over **Core Idea 3** (single image on top).
|
| 96 |
Only use Core Idea 3 when appropriate based on the image orientation.
|
|
@@ -102,7 +113,8 @@ def get_prompt(
|
|
| 102 |
|
| 103 |
Strict rules:
|
| 104 |
- Never invent, change, or guess the image path. Use the `"path"` value exactly as written.
|
| 105 |
-
- Never change or edit anything inside the square brackets `[...]` in any `\\includegraphics`
|
|
|
|
| 106 |
- Skip any image with caption `"None"`.
|
| 107 |
- Use **Core Idea 2** layout for **vertical** and **square** images.
|
| 108 |
- Use **Core Idea 3** layout for **horizontal** images (when a two-column layout is not suitable).\n\n"""
|
|
|
|
| 1 |
+
"""Module contatining the master prompt."""
|
| 2 |
+
|
| 3 |
+
|
| 4 |
def get_prompt(
|
| 5 |
presentation_theme: str = "default", color_theme: str = "default"
|
| 6 |
) -> str:
|
|
|
|
| 9 |
Args:
|
| 10 |
presentation_theme (str): The theme for the presentation, default is "default".
|
| 11 |
color_theme (str): The color theme for the presentation, default is "default".
|
| 12 |
+
Returns:
|
| 13 |
+
str: the prompt with the specified themes and structure for generating a
|
| 14 |
+
LaTeX Beamer presentation.
|
| 15 |
"""
|
| 16 |
+
prompt = f"""You are a presentation assistant that creates clear, concise, and engaging slide decks from the
|
| 17 |
+
reference material provided. You extract the most relevant and important information, organize it logically,
|
| 18 |
+
and generate LaTeX code for a presentation using the Beamer class.
|
| 19 |
|
| 20 |
Structure your slides as follows:
|
| 21 |
1. **Introduction**: Present the topic and explain why it's important or interesting.
|
| 22 |
+
2. **Main Content**: Break down the topic into 2–4 core ideas, one idea per slide. Explain each with clear
|
| 23 |
+
language, bullet points, or short sentences.
|
| 24 |
3. **Summary**: Recap the key takeaways, what was learned, and what it means for the audience.
|
| 25 |
|
| 26 |
+
Use friendly and accessible language that anyone can understand. Avoid technical jargon unless it's
|
| 27 |
+
essential—and when you use it, explain it simply.
|
| 28 |
|
| 29 |
Output only valid LaTeX Beamer code. Each slide must be defined using \\begin{{frame}} ... \\end{{frame}}.
|
| 30 |
|
| 31 |
+
Here is some reference content retrieved from a document. Please generate a LaTeX Beamer presentation based on
|
| 32 |
+
the content, following this structure:
|
| 33 |
|
| 34 |
1. Introduction (what is the topic and why it matters)
|
| 35 |
2. Main Part (a few slides on the core ideas)
|
|
|
|
| 100 |
|
| 101 |
\\end{{document}}
|
| 102 |
|
| 103 |
+
You must use **at least one image** when creating a presentation. In your output, include `\\includegraphics`
|
| 104 |
+
commands in **at least half of the slides** where appropriate.
|
| 105 |
|
| 106 |
Prioritize the slide structure of **Core Idea 2** (two-column layout) over **Core Idea 3** (single image on top).
|
| 107 |
Only use Core Idea 3 when appropriate based on the image orientation.
|
|
|
|
| 113 |
|
| 114 |
Strict rules:
|
| 115 |
- Never invent, change, or guess the image path. Use the `"path"` value exactly as written.
|
| 116 |
+
- Never change or edit anything inside the square brackets `[...]` in any `\\includegraphics`
|
| 117 |
+
command. All image formatting options (height, width, keepaspectratio) must be left untouched.
|
| 118 |
- Skip any image with caption `"None"`.
|
| 119 |
- Use **Core Idea 2** layout for **vertical** and **square** images.
|
| 120 |
- Use **Core Idea 3** layout for **horizontal** images (when a two-column layout is not suitable).\n\n"""
|
src/services/__init__.py
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
|
|
|
|
|
| 1 |
from .google_client import client
|
| 2 |
from .generator import generate_presentation
|
| 3 |
|
|
|
|
| 1 |
+
"""This module initializes the services package."""
|
| 2 |
+
|
| 3 |
from .google_client import client
|
| 4 |
from .generator import generate_presentation
|
| 5 |
|
src/services/generator.py
CHANGED
|
@@ -1,116 +1,63 @@
|
|
| 1 |
-
|
| 2 |
-
import os
|
| 3 |
-
import re
|
| 4 |
-
from pathlib import Path
|
| 5 |
-
from datetime import datetime
|
| 6 |
-
from ..database import db, embed_fn, ingest_files_to_db
|
| 7 |
-
from ..utils import Logger
|
| 8 |
-
from ..processing import get_prompt
|
| 9 |
-
from . import client
|
| 10 |
-
from ..compilation import CompilePresentation
|
| 11 |
-
from ..processing import save_pdf_images, save_pdf_figures
|
| 12 |
-
|
| 13 |
-
LOGGER = Logger.get_logger()
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
def generate_presentation(theme, color, topic, uploaded_files):
|
| 17 |
-
print(f"Theme: {theme}, Color: {color}, Topic: {topic}")
|
| 18 |
-
ingest_files_to_db(uploaded_files)
|
| 19 |
-
|
| 20 |
-
LOGGER.info("Retrieving relevant documents...")
|
| 21 |
-
embed_fn.document_mode = False
|
| 22 |
-
|
| 23 |
-
query_oneline = topic.replace(
|
| 24 |
-
"\n", " "
|
| 25 |
-
) # (Optional) For cleaner input in case of newlines
|
| 26 |
-
result = db.query(query_texts=[query_oneline], n_results=2)
|
| 27 |
-
[documents] = result["documents"]
|
| 28 |
-
[metadatas] = result["metadatas"]
|
| 29 |
-
|
| 30 |
-
prompt = get_prompt(theme, color)
|
| 31 |
|
| 32 |
-
|
| 33 |
-
passage_oneline = passage.replace("\n", " ")
|
| 34 |
-
images_passage = metas["images_passage"]
|
| 35 |
-
|
| 36 |
-
prompt += f"PASSAGE: {passage_oneline}\n"
|
| 37 |
-
prompt += f"IMAGES: {images_passage}\n"
|
| 38 |
-
|
| 39 |
-
# Create a new subfolder
|
| 40 |
-
LOGGER.info("💾 Creating new subfolder...")
|
| 41 |
-
cwd = Path.cwd()
|
| 42 |
-
one_level_up = cwd.parent
|
| 43 |
-
base_path = one_level_up / "output"
|
| 44 |
-
timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
|
| 45 |
-
work_dir = os.path.join(base_path, timestamp)
|
| 46 |
-
print("work_dir", work_dir)
|
| 47 |
-
os.makedirs(work_dir, exist_ok=False)
|
| 48 |
-
LOGGER.info("📁 Created folder: %s", work_dir)
|
| 49 |
-
|
| 50 |
-
# Generate the presentation code
|
| 51 |
-
model_name = "gemini-2.0-flash"
|
| 52 |
-
LOGGER.info("⭐️ Generating response...")
|
| 53 |
-
answer = client.models.generate_content(model=model_name, contents=prompt)
|
| 54 |
-
LOGGER.info("🎲 Generated the response using: %s", model_name)
|
| 55 |
-
# Create a subfolder for graphics
|
| 56 |
-
|
| 57 |
-
graphics_dir = os.path.join(work_dir, "gfx")
|
| 58 |
-
os.makedirs(graphics_dir, exist_ok=True)
|
| 59 |
-
LOGGER.info("🌄 Images will be saved to: %s", graphics_dir)
|
| 60 |
-
|
| 61 |
-
# Find images, which are used in the presentation
|
| 62 |
-
pattern_img = re.compile(
|
| 63 |
-
r"doc(?P<doc>[a-zA-Z0-9_]+)_page(?P<page>\d+)_img(?P<img>\d+)_hash(?P<hash>[a-fA-F0-9]{8})\.png"
|
| 64 |
-
)
|
| 65 |
-
matches_img = pattern_img.finditer(answer.text)
|
| 66 |
-
|
| 67 |
-
req_imgs = []
|
| 68 |
-
for match in matches_img:
|
| 69 |
-
req_img = {
|
| 70 |
-
"doc": match.group("doc"),
|
| 71 |
-
"page": int(match.group("page")),
|
| 72 |
-
"img": int(match.group("img")),
|
| 73 |
-
"hash": match.group("hash"),
|
| 74 |
-
}
|
| 75 |
-
req_imgs.append(req_img)
|
| 76 |
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
|
|
|
|
|
|
|
| 82 |
|
| 83 |
-
req_figs = []
|
| 84 |
-
for match in matches_fig:
|
| 85 |
-
req_fig = {
|
| 86 |
-
"doc": match.group("doc"),
|
| 87 |
-
"page": int(match.group("page")),
|
| 88 |
-
"fig": int(match.group("fig")),
|
| 89 |
-
"hash": match.group("hash"),
|
| 90 |
-
}
|
| 91 |
-
req_figs.append(req_fig)
|
| 92 |
|
| 93 |
-
|
| 94 |
-
for metadata in metadatas:
|
| 95 |
-
save_pdf_images(metadata["pdf_path"], req_imgs, graphics_dir)
|
| 96 |
-
save_pdf_figures(metadata["pdf_path"], req_figs, graphics_dir)
|
| 97 |
|
| 98 |
-
LOGGER.info("Compiling presentation...")
|
| 99 |
-
CompilePresentation(answer.text, work_dir)
|
| 100 |
-
LOGGER.info("Presentation is ready!")
|
| 101 |
-
# Clean up uploaded files
|
| 102 |
-
print("uploaded_files", uploaded_files)
|
| 103 |
-
for file in uploaded_files:
|
| 104 |
-
try:
|
| 105 |
-
path = Path(file)
|
| 106 |
-
if path.exists():
|
| 107 |
-
path.unlink() # Delete the file
|
| 108 |
-
LOGGER.info(f"Deleted file: {file}")
|
| 109 |
-
else:
|
| 110 |
-
LOGGER.warning(f"File not found for deletion: {file}")
|
| 111 |
-
except Exception as e:
|
| 112 |
-
LOGGER.warning(f"Failed to delete {file}", exc_info=e)
|
| 113 |
|
| 114 |
-
|
| 115 |
-
|
| 116 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Module to put it all together and manage the generation of a presentation."""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2 |
|
| 3 |
+
from pathlib import Path
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4 |
|
| 5 |
+
# from datetime import datetime
|
| 6 |
+
from ..database import ingest_files_to_db, retrive_files_from_db
|
| 7 |
+
from ..telemetry import Logger, generate_with_trace, traced_block
|
| 8 |
+
from ..processing import get_prompt, create_output_folder
|
| 9 |
+
from . import client
|
| 10 |
+
from ..compilation import compile_presentation
|
| 11 |
+
from ..processing import delete_uploaded_files, find_used_fgx
|
| 12 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 13 |
|
| 14 |
+
LOGGER = Logger.get_logger()
|
|
|
|
|
|
|
|
|
|
| 15 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 16 |
|
| 17 |
+
def generate_presentation(theme, color, topic, uploaded_files) -> tuple[str, str, str]:
|
| 18 |
+
"""Main function to generate a presentation based on the provided theme, color, and topic.
|
| 19 |
+
Args:
|
| 20 |
+
theme (str): The theme of the presentation, chosen by user via UI, defaults to "default".
|
| 21 |
+
color (str): The color scheme for the presentation, chosen by user via UI,
|
| 22 |
+
defaults to "default".
|
| 23 |
+
topic (str): The topic of the presentation, set by user via UI, defaults to "default".
|
| 24 |
+
uploaded_files (list): List of file paths to be ingested, provided by user via UI.
|
| 25 |
+
Returns:
|
| 26 |
+
tuple: A 3-element tuple:
|
| 27 |
+
- str: Success message.
|
| 28 |
+
- str: Path to the generated presentation.
|
| 29 |
+
- str: Placeholder for the output viewer.
|
| 30 |
+
"""
|
| 31 |
+
with traced_block(
|
| 32 |
+
"📽️ generate_presentation",
|
| 33 |
+
input_theme=theme,
|
| 34 |
+
input_color=color,
|
| 35 |
+
input_topic=topic,
|
| 36 |
+
file_count=len(uploaded_files),
|
| 37 |
+
):
|
| 38 |
+
ingest_files_to_db(uploaded_files)
|
| 39 |
+
|
| 40 |
+
[documents], [metadatas] = retrive_files_from_db(topic)
|
| 41 |
+
|
| 42 |
+
prompt = get_prompt(theme, color)
|
| 43 |
+
|
| 44 |
+
for passage, metas in zip(documents, metadatas):
|
| 45 |
+
passage_oneline = passage.replace("\n", " ")
|
| 46 |
+
images_passage = metas["images_passage"]
|
| 47 |
+
|
| 48 |
+
prompt += f"PASSAGE: {passage_oneline}\n"
|
| 49 |
+
prompt += f"IMAGES: {images_passage}\n"
|
| 50 |
+
|
| 51 |
+
work_dir = create_output_folder()
|
| 52 |
+
# Generate the presentation code
|
| 53 |
+
model_name = "gemini-2.0-flash"
|
| 54 |
+
LOGGER.info("⭐️ Generating response...")
|
| 55 |
+
answer = generate_with_trace(client, model_name, prompt)
|
| 56 |
+
LOGGER.info("🎲 Generated the response using: %s", model_name)
|
| 57 |
+
|
| 58 |
+
find_used_fgx(answer, work_dir, metadatas)
|
| 59 |
+
compile_presentation(answer.text, work_dir)
|
| 60 |
+
|
| 61 |
+
delete_uploaded_files(uploaded_files)
|
| 62 |
+
output_path = Path(work_dir + "/presentation.pdf")
|
| 63 |
+
return "Presentation generated successfully!", str(output_path), ""
|
src/services/google_client.py
CHANGED
|
@@ -1,4 +1,5 @@
|
|
| 1 |
-
|
|
|
|
| 2 |
import os
|
| 3 |
from google import genai
|
| 4 |
from dotenv import load_dotenv
|
|
|
|
| 1 |
+
"""Initialize and export a GenAI client using the provided Google API key."""
|
| 2 |
+
|
| 3 |
import os
|
| 4 |
from google import genai
|
| 5 |
from dotenv import load_dotenv
|
src/telemetry/__init__.py
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Initialize telemetry and logging utilities."""
|
| 2 |
+
|
| 3 |
+
from .logging_utils import Logger
|
| 4 |
+
from .telemetry import generate_with_trace, init_telemetry, traced_block
|
| 5 |
+
|
| 6 |
+
__all__ = [
|
| 7 |
+
"generate_with_trace",
|
| 8 |
+
"init_telemetry",
|
| 9 |
+
"traced_block",
|
| 10 |
+
"Logger",
|
| 11 |
+
]
|
src/{utils → telemetry}/logging_utils.py
RENAMED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
-
|
| 2 |
|
| 3 |
import logging
|
| 4 |
|
|
@@ -23,6 +23,16 @@ class Logger:
|
|
| 23 |
)
|
| 24 |
cls._logger = logging.getLogger("Logger")
|
| 25 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 26 |
@classmethod
|
| 27 |
def get_logger(cls) -> logging.Logger:
|
| 28 |
"""
|
|
|
|
| 1 |
+
"""Module for setting up and managing a global logger instance."""
|
| 2 |
|
| 3 |
import logging
|
| 4 |
|
|
|
|
| 23 |
)
|
| 24 |
cls._logger = logging.getLogger("Logger")
|
| 25 |
|
| 26 |
+
# add compatibility with opentelemetry
|
| 27 |
+
otel_loggers = [
|
| 28 |
+
"opentelemetry.sdk",
|
| 29 |
+
"opentelemetry.trace",
|
| 30 |
+
"opentelemetry.exporter.otlp.proto.http.trace_exporter",
|
| 31 |
+
]
|
| 32 |
+
|
| 33 |
+
for name in otel_loggers:
|
| 34 |
+
logging.getLogger(name).setLevel(logging.DEBUG)
|
| 35 |
+
|
| 36 |
@classmethod
|
| 37 |
def get_logger(cls) -> logging.Logger:
|
| 38 |
"""
|
src/telemetry/telemetry.py
ADDED
|
@@ -0,0 +1,89 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Telemetry utilities for Langfuse integration and OpenTelemetry tracing."""
|
| 2 |
+
|
| 3 |
+
import os
|
| 4 |
+
import hashlib
|
| 5 |
+
from typing import Union
|
| 6 |
+
from contextlib import contextmanager
|
| 7 |
+
|
| 8 |
+
# import psycopg2
|
| 9 |
+
import base64
|
| 10 |
+
from opentelemetry import trace
|
| 11 |
+
from opentelemetry.sdk.trace import TracerProvider
|
| 12 |
+
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
|
| 13 |
+
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
| 14 |
+
|
| 15 |
+
from .logging_utils import Logger
|
| 16 |
+
|
| 17 |
+
LOGGER = Logger.get_logger()
|
| 18 |
+
|
| 19 |
+
tracer = None
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def init_telemetry() -> None:
|
| 23 |
+
"""Setup langfuse, initialize OpenTelemetry for tracing."""
|
| 24 |
+
if os.getenv("USE_LANGFUSE", "false").lower() == "true":
|
| 25 |
+
os.environ["OTEL_EXPORTER_OTLP_ENDPOINT"] = (
|
| 26 |
+
os.environ["LANGFUSE_HOST"] + "/api/public/otel"
|
| 27 |
+
)
|
| 28 |
+
auth = f"{os.getenv('LANGFUSE_PUBLIC_KEY')}:{os.getenv('LANGFUSE_SECRET_KEY')}"
|
| 29 |
+
auth_b64 = base64.b64encode(auth.encode()).decode()
|
| 30 |
+
os.environ["OTEL_EXPORTER_OTLP_HEADERS"] = f"Authorization=Basic {auth_b64}"
|
| 31 |
+
|
| 32 |
+
provider = TracerProvider()
|
| 33 |
+
exporter = OTLPSpanExporter()
|
| 34 |
+
provider.add_span_processor(SimpleSpanProcessor(exporter))
|
| 35 |
+
trace.set_tracer_provider(provider)
|
| 36 |
+
|
| 37 |
+
global tracer
|
| 38 |
+
tracer = trace.get_tracer("langfuse.genai")
|
| 39 |
+
|
| 40 |
+
else:
|
| 41 |
+
LOGGER.warning("Langfuse telemetry is disabled (USE_LANGFUSE != true)")
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def hash_prompt(text: str) -> str:
|
| 45 |
+
"""Generate a SHA-256 hash of the prompt.
|
| 46 |
+
Args:
|
| 47 |
+
text (str): The input text to hash."""
|
| 48 |
+
return hashlib.sha256(text.encode()).hexdigest()
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
@contextmanager
|
| 52 |
+
def traced_block(name: str, **attrs):
|
| 53 |
+
"""Context manager to create a traced block.
|
| 54 |
+
Args:
|
| 55 |
+
name (str): The name of the span.
|
| 56 |
+
**attrs: Additional attributes to set on the span."""
|
| 57 |
+
with tracer.start_as_current_span(name) as span:
|
| 58 |
+
for key, value in attrs.items():
|
| 59 |
+
span.set_attribute(key, value)
|
| 60 |
+
try:
|
| 61 |
+
yield span # Control to the block
|
| 62 |
+
except Exception as e:
|
| 63 |
+
# Record exception and mark span status as error
|
| 64 |
+
span.record_exception(e)
|
| 65 |
+
span.set_status(trace.Status(trace.StatusCode.ERROR, str(e)))
|
| 66 |
+
LOGGER.error("Error in span %s while generating content: %s", name, e)
|
| 67 |
+
raise
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
def generate_with_trace(client, model_name: str, prompt: str) -> Union[object, None]:
|
| 71 |
+
"""Save traces from the generation.
|
| 72 |
+
Args:
|
| 73 |
+
client: google genai client instance.
|
| 74 |
+
model_name (str): The name of the model to use.
|
| 75 |
+
prompt (str): The input prompt for generation."""
|
| 76 |
+
with tracer.start_as_current_span("🤖 genai_generate") as span:
|
| 77 |
+
span.set_attribute("input.prompt_hash", hash_prompt(prompt))
|
| 78 |
+
span.set_attribute("input.prompt_length", len(prompt))
|
| 79 |
+
span.set_attribute("input.model_name", model_name)
|
| 80 |
+
try:
|
| 81 |
+
response = client.models.generate_content(model=model_name, contents=prompt)
|
| 82 |
+
span.set_attribute("output.response_length", len(str(response)))
|
| 83 |
+
span.set_attribute("output.success", True)
|
| 84 |
+
return response
|
| 85 |
+
except Exception as e:
|
| 86 |
+
span.set_attribute("output.success", False)
|
| 87 |
+
span.record_exception(e)
|
| 88 |
+
span.set_status(trace.Status(trace.StatusCode.ERROR, str(e)))
|
| 89 |
+
return LOGGER.error("Error in spans while generating content: %s", e)
|
src/utils/__init__.py
DELETED
|
@@ -1,9 +0,0 @@
|
|
| 1 |
-
from .gradio_user_settings import (
|
| 2 |
-
upload_files,
|
| 3 |
-
)
|
| 4 |
-
from .logging_utils import Logger
|
| 5 |
-
|
| 6 |
-
__all__ = [
|
| 7 |
-
"upload_files",
|
| 8 |
-
"Logger",
|
| 9 |
-
]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|