Agentic-Service-Data-Eyond-Catalog

Running

Agentic-Service-Data-Eyond-Catalog / pyproject.toml

[NOTICKET] feat(knowledge-parsing): wire Mistral OCR backend end-to-end

6856ea8 about 2 months ago

5.17 kB

[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"

[project]
name = "agent-service-data-eyond"
version = "0.1.0"
description = "Agentic Service Data Eyond — Multi-Agent AI Backend"
requires-python = ">=3.12,<3.13"

dependencies = [
    # --- Web Framework ---
    "fastapi[standard]==0.115.6",
    "uvicorn[standard]==0.32.1",
    "python-multipart==0.0.12",
    "starlette==0.41.3",
    "sse-starlette==2.1.3",
    # --- LangChain Core Ecosystem (NO LiteLLM) ---
    "langchain==0.3.13",
    "langchain-core==0.3.28",
    "langchain-community==0.3.13",
    "langchain-openai==0.2.14",
    "langchain-postgres>=0.0.13",
    "langgraph==0.2.60",
    "langgraph-checkpoint-postgres==2.0.9",
    # --- LLM / Azure OpenAI ---
    "openai==1.58.1",
    "tiktoken==0.8.0",
    # --- Database ---
    "sqlalchemy[asyncio]==2.0.36",
    "asyncpg==0.30.0",
    "psycopg[binary,pool]==3.2.3",
    "pgvector==0.3.6",
    "alembic==1.14.0",
    # --- Azure ---
    "azure-storage-blob==12.23.1",
    "azure-identity==1.19.0",
    "azure-ai-documentintelligence==1.0.0",
    # --- Pydantic / Validation ---
    "pydantic==2.10.3",
    "pydantic-settings==2.7.0",
    # --- Observability ---
    "langfuse==2.57.4",
    "structlog==24.4.0",
    "prometheus-client==0.21.1",
    # --- Security ---
    "passlib[bcrypt]==1.7.4",
    "cryptography==44.0.0",
    # --- Rate Limiting ---
    "slowapi==0.1.9",
    "redis==5.2.1",
    # --- Retry ---
    "tenacity==9.0.0",
    # --- Document Processing (for reading existing docs from blob) ---
    "pypdf==5.1.0",
    "python-docx==1.1.2",
    "openpyxl==3.1.5",
    "pandas==2.2.3",
    # --- Chart/Visualization ---
    "matplotlib==3.9.3",
    "plotly==5.24.1",
    "kaleido==0.2.1",
    # --- MCP ---
    "mcp==1.2.0",
    # --- Advanced RAG ---
    "rank-bm25==0.2.2",
    "sentence-transformers==3.3.1",
    # --- PII Detection (no LiteLLM) ---
    "presidio-analyzer==2.2.355",
    "presidio-anonymizer==2.2.355",
    "spacy==3.8.3",
    # --- Utilities ---
    "httpx==0.28.1",
    "anyio==4.7.0",
    "python-dotenv==1.0.1",
    "orjson==3.10.12",
    "cachetools==5.5.0",
    "apscheduler==3.10.4",
    "jsonpatch>=1.33",
    "psycopg2>=2.9.11",
    # --- SQL parsing / guardrails ---
    "sqlglot>=25.0.0",
    # --- User-DB connectors (db_pipeline) ---
    "pymysql>=1.1.1",
    "pymssql>=2.3.0",
    "sqlalchemy-bigquery>=1.11.0",
    "snowflake-sqlalchemy>=1.7.0",
    # --- OCR (pdf processing) ---
    "pdf2image>=1.17.0",
    "pytesseract>=0.3.13",
    "pypdf2>=3.0.1",
    "pyarrow>=24.0.0",
    "rapidfuzz>=3.14.5",
    "gliner>=0.2.13",
    "torch>=2.6",
]

[project.optional-dependencies]
dev = [
    "pytest==8.3.4",
    "pytest-asyncio==0.24.0",
    "pytest-cov==6.0.0",
    "httpx==0.28.1",
    "ruff==0.8.4",
    "mypy==1.13.0",
    "pre-commit==4.0.1",
]

# Document parsing for the knowledge pipeline (src/knowledge_parsing/).
#
# Deliberately an EXTRA, not a main dependency: MinerU pulls torch and its model
# tail (GBs), and the agent service never parses documents at request time —
# parsing is an offline, admin-triggered batch job. Keeping it here means the
# deployed Space does not build or ship any of it.
#
#     pip install -e ".[parsing]"
knowledge-parsing = [
    "mineru==3.4.4",
    # Mistral OCR backend (primary parser; MinerU kept as fallback). Needs
    # pydantic>=2.11.2 — see the note on the pinned pydantic in [dependencies].
    "mistralai>=1.9",
]

[tool.hatch.build.targets.wheel]
packages = ["src/agent_service"]

[tool.ruff]
target-version = "py312"
line-length = 100

[tool.ruff.lint]
select = ["E", "F", "I", "N", "UP", "S", "B", "A", "C4", "T20"]
ignore = [
    "S101",   # assert statements OK in tests
    "S105",   # hardcoded passwords — false positives in config
    "S106",
    "B008",   # FastAPI Depends() calls OK in function args
]

[tool.ruff.lint.per-file-ignores]
# S608: golden compiler tests assert literal SQL strings (incl. concatenated
# suffixes) — they never execute against a DB, so it's a false positive here.
"tests/**" = ["S101", "S105", "S106", "S608"]
# T201: eval/ scripts are CLIs — print() is their intended output channel.
"eval/**" = ["T201"]
# Same rule, same reason: this is an operator-run CLI, and its printed output IS
# the deliverable.
"src/knowledge_extraction/cli.py" = ["T201"]
# Same again for the knowledge-parsing CLI entry points. Scoped to the two files
# rather than the package, so the library modules stay print-free.
"src/knowledge_parsing/run.py" = ["T201"]
"src/knowledge_parsing/report.py" = ["T201"]
# Pembungkus pengembangan: print() memang saluran keluarannya, dan subprocess
# memang tugasnya — dia menjalankan CLI parsing dengan env var yang sudah
# disiapkan. Argumennya daftar, tidak pernah lewat shell.
"scripts/parse.py" = ["T201", "S603"]

[tool.mypy]
python_version = "3.12"
strict = true
ignore_missing_imports = true
plugins = ["pydantic.mypy"]

[tool.pytest.ini_options]
asyncio_mode = "auto"
testpaths = ["tests"]
filterwarnings = [
    "ignore::DeprecationWarning",
]

[dependency-groups]
dev = [
    "pytest>=8.3.4",
    "pytest-asyncio>=0.24.0",
    "ruff>=0.8.4",
]

	[build-system]
	requires = ["hatchling"]
	build-backend = "hatchling.build"

	[project]
	name = "agent-service-data-eyond"
	version = "0.1.0"
	description = "Agentic Service Data Eyond — Multi-Agent AI Backend"
	requires-python = ">=3.12,<3.13"

	dependencies = [
	# --- Web Framework ---
	"fastapi[standard]==0.115.6",
	"uvicorn[standard]==0.32.1",
	"python-multipart==0.0.12",
	"starlette==0.41.3",
	"sse-starlette==2.1.3",
	# --- LangChain Core Ecosystem (NO LiteLLM) ---
	"langchain==0.3.13",
	"langchain-core==0.3.28",
	"langchain-community==0.3.13",
	"langchain-openai==0.2.14",
	"langchain-postgres>=0.0.13",
	"langgraph==0.2.60",
	"langgraph-checkpoint-postgres==2.0.9",
	# --- LLM / Azure OpenAI ---
	"openai==1.58.1",
	"tiktoken==0.8.0",
	# --- Database ---
	"sqlalchemy[asyncio]==2.0.36",
	"asyncpg==0.30.0",
	"psycopg[binary,pool]==3.2.3",
	"pgvector==0.3.6",
	"alembic==1.14.0",
	# --- Azure ---
	"azure-storage-blob==12.23.1",
	"azure-identity==1.19.0",
	"azure-ai-documentintelligence==1.0.0",
	# --- Pydantic / Validation ---
	"pydantic==2.10.3",
	"pydantic-settings==2.7.0",
	# --- Observability ---
	"langfuse==2.57.4",
	"structlog==24.4.0",
	"prometheus-client==0.21.1",
	# --- Security ---
	"passlib[bcrypt]==1.7.4",
	"cryptography==44.0.0",
	# --- Rate Limiting ---
	"slowapi==0.1.9",
	"redis==5.2.1",
	# --- Retry ---
	"tenacity==9.0.0",
	# --- Document Processing (for reading existing docs from blob) ---
	"pypdf==5.1.0",
	"python-docx==1.1.2",
	"openpyxl==3.1.5",
	"pandas==2.2.3",
	# --- Chart/Visualization ---
	"matplotlib==3.9.3",
	"plotly==5.24.1",
	"kaleido==0.2.1",
	# --- MCP ---
	"mcp==1.2.0",
	# --- Advanced RAG ---
	"rank-bm25==0.2.2",
	"sentence-transformers==3.3.1",
	# --- PII Detection (no LiteLLM) ---
	"presidio-analyzer==2.2.355",
	"presidio-anonymizer==2.2.355",
	"spacy==3.8.3",
	# --- Utilities ---
	"httpx==0.28.1",
	"anyio==4.7.0",
	"python-dotenv==1.0.1",
	"orjson==3.10.12",
	"cachetools==5.5.0",
	"apscheduler==3.10.4",
	"jsonpatch>=1.33",
	"psycopg2>=2.9.11",
	# --- SQL parsing / guardrails ---
	"sqlglot>=25.0.0",
	# --- User-DB connectors (db_pipeline) ---
	"pymysql>=1.1.1",
	"pymssql>=2.3.0",
	"sqlalchemy-bigquery>=1.11.0",
	"snowflake-sqlalchemy>=1.7.0",
	# --- OCR (pdf processing) ---
	"pdf2image>=1.17.0",
	"pytesseract>=0.3.13",
	"pypdf2>=3.0.1",
	"pyarrow>=24.0.0",
	"rapidfuzz>=3.14.5",
	"gliner>=0.2.13",
	"torch>=2.6",
	]

	[project.optional-dependencies]
	dev = [
	"pytest==8.3.4",
	"pytest-asyncio==0.24.0",
	"pytest-cov==6.0.0",
	"httpx==0.28.1",
	"ruff==0.8.4",
	"mypy==1.13.0",
	"pre-commit==4.0.1",
	]

	# Document parsing for the knowledge pipeline (src/knowledge_parsing/).
	#
	# Deliberately an EXTRA, not a main dependency: MinerU pulls torch and its model
	# tail (GBs), and the agent service never parses documents at request time —
	# parsing is an offline, admin-triggered batch job. Keeping it here means the
	# deployed Space does not build or ship any of it.
	#
	# pip install -e ".[parsing]"
	knowledge-parsing = [
	"mineru==3.4.4",
	# Mistral OCR backend (primary parser; MinerU kept as fallback). Needs
	# pydantic>=2.11.2 — see the note on the pinned pydantic in [dependencies].
	"mistralai>=1.9",
	]

	[tool.hatch.build.targets.wheel]
	packages = ["src/agent_service"]

	[tool.ruff]
	target-version = "py312"
	line-length = 100

	[tool.ruff.lint]
	select = ["E", "F", "I", "N", "UP", "S", "B", "A", "C4", "T20"]
	ignore = [
	"S101", # assert statements OK in tests
	"S105", # hardcoded passwords — false positives in config
	"S106",
	"B008", # FastAPI Depends() calls OK in function args
	]

	[tool.ruff.lint.per-file-ignores]
	# S608: golden compiler tests assert literal SQL strings (incl. concatenated
	# suffixes) — they never execute against a DB, so it's a false positive here.
	"tests/**" = ["S101", "S105", "S106", "S608"]
	# T201: eval/ scripts are CLIs — print() is their intended output channel.
	"eval/**" = ["T201"]
	# Same rule, same reason: this is an operator-run CLI, and its printed output IS
	# the deliverable.
	"src/knowledge_extraction/cli.py" = ["T201"]
	# Same again for the knowledge-parsing CLI entry points. Scoped to the two files
	# rather than the package, so the library modules stay print-free.
	"src/knowledge_parsing/run.py" = ["T201"]
	"src/knowledge_parsing/report.py" = ["T201"]
	# Pembungkus pengembangan: print() memang saluran keluarannya, dan subprocess
	# memang tugasnya — dia menjalankan CLI parsing dengan env var yang sudah
	# disiapkan. Argumennya daftar, tidak pernah lewat shell.
	"scripts/parse.py" = ["T201", "S603"]

	[tool.mypy]
	python_version = "3.12"
	strict = true
	ignore_missing_imports = true
	plugins = ["pydantic.mypy"]

	[tool.pytest.ini_options]
	asyncio_mode = "auto"
	testpaths = ["tests"]
	filterwarnings = [
	"ignore::DeprecationWarning",
	]

	[dependency-groups]
	dev = [
	"pytest>=8.3.4",
	"pytest-asyncio>=0.24.0",
	"ruff>=0.8.4",
	]