Spaces:
Sleeping
Sleeping
Commit ·
697be7d
1
Parent(s): d5eb307
update
Browse files- .gitattributes +13 -0
- .gitignore +3 -1
- Dockerfile +2 -1
- __init__.py +1 -1
- api/server.py +34 -61
- app/banner.py +0 -21
- app/core/exceptions.py +0 -5
- app/main.py +0 -95
- app/services/ping.py +6 -3
- banner.py +1 -1
- core/__init__.py +0 -7
- core/converter.py +7 -8
- extraction/json_extractor.py +5 -4
- extraction/spacy_extractor.py +2 -6
- logger.py +7 -74
- self_ping.py +0 -19
.gitattributes
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
* text=auto
|
| 2 |
+
*.sh text eol=lf
|
| 3 |
+
*.py text eol=lf
|
| 4 |
+
Dockerfile text eol=lf
|
| 5 |
+
*.ini text eol=lf
|
| 6 |
+
*.cfg text eol=lf
|
| 7 |
+
*.yml text eol=lf
|
| 8 |
+
*.yaml text eol=lf
|
| 9 |
+
*.json text eol=lf
|
| 10 |
+
*.md text eol=lf
|
| 11 |
+
.gitignore text eol=lf
|
| 12 |
+
.dockerignore text eol=lf
|
| 13 |
+
.env text eol=lf
|
.gitignore
CHANGED
|
@@ -125,4 +125,6 @@ run_docker.py
|
|
| 125 |
docs
|
| 126 |
.claude
|
| 127 |
start_docker.txt
|
| 128 |
-
Reconciliation-file-processing-service.postman_collection.json
|
|
|
|
|
|
|
|
|
| 125 |
docs
|
| 126 |
.claude
|
| 127 |
start_docker.txt
|
| 128 |
+
Reconciliation-file-processing-service.postman_collection.json
|
| 129 |
+
pytest.ini
|
| 130 |
+
tests/*
|
Dockerfile
CHANGED
|
@@ -5,9 +5,10 @@
|
|
| 5 |
|
| 6 |
FROM python:3.12-slim
|
| 7 |
|
| 8 |
-
LABEL maintainer="Reconciliation API"
|
| 9 |
LABEL description="Document-to-Markdown & PDF-to-image API"
|
| 10 |
LABEL version="2.2.0"
|
|
|
|
| 11 |
|
| 12 |
# ── System dependencies ──────────────────────────────────────
|
| 13 |
RUN apt-get update && apt-get install -y --no-install-recommends \
|
|
|
|
| 5 |
|
| 6 |
FROM python:3.12-slim
|
| 7 |
|
| 8 |
+
LABEL maintainer="Reconciliation File Processing Service API"
|
| 9 |
LABEL description="Document-to-Markdown & PDF-to-image API"
|
| 10 |
LABEL version="2.2.0"
|
| 11 |
+
LABEL description="Production-ready: consolidated logging, unified app, cleanup loop, self-ping"
|
| 12 |
|
| 13 |
# ── System dependencies ──────────────────────────────────────
|
| 14 |
RUN apt-get update && apt-get install -y --no-install-recommends \
|
__init__.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
-
from
|
| 2 |
|
| 3 |
__all__ = [
|
| 4 |
"ConversionError",
|
|
|
|
| 1 |
+
from core.converter import ConversionError, ConversionResult, DocumentConverter, SUPPORTED_EXTENSIONS
|
| 2 |
|
| 3 |
__all__ = [
|
| 4 |
"ConversionError",
|
api/server.py
CHANGED
|
@@ -20,7 +20,6 @@ PDF-to-image conversion (under /api/v1):
|
|
| 20 |
GET /api/v1/files/{id}/{fn}
|
| 21 |
GET /api/v1/health
|
| 22 |
GET /api/v1/ready
|
| 23 |
-
GET /api/v1/ping
|
| 24 |
|
| 25 |
System:
|
| 26 |
GET / Root info
|
|
@@ -34,11 +33,10 @@ from __future__ import annotations
|
|
| 34 |
|
| 35 |
import asyncio
|
| 36 |
import concurrent.futures
|
|
|
|
| 37 |
import os
|
| 38 |
-
import threading
|
| 39 |
import time
|
| 40 |
import uuid
|
| 41 |
-
import urllib.request
|
| 42 |
from contextlib import asynccontextmanager
|
| 43 |
from datetime import datetime, timezone
|
| 44 |
from pathlib import Path
|
|
@@ -46,6 +44,7 @@ from typing import Annotated, Any, Dict, List, Optional
|
|
| 46 |
from urllib.parse import urlparse
|
| 47 |
|
| 48 |
import httpx
|
|
|
|
| 49 |
from fastapi import APIRouter, Depends, FastAPI, File, Form, HTTPException, Request, UploadFile, status
|
| 50 |
from fastapi.responses import JSONResponse, PlainTextResponse
|
| 51 |
from fastapi.middleware.cors import CORSMiddleware
|
|
@@ -62,6 +61,8 @@ from app.core.config import settings
|
|
| 62 |
from app.core.exceptions import AppException
|
| 63 |
from app.core.logging import configure_logging
|
| 64 |
from app.core.rate_limit import RateLimitMiddleware
|
|
|
|
|
|
|
| 65 |
|
| 66 |
configure_logging()
|
| 67 |
logger = get_logger(__name__)
|
|
@@ -69,43 +70,14 @@ logger = get_logger(__name__)
|
|
| 69 |
_START_TIME = time.time()
|
| 70 |
APP_NAME = "reconciliation-file-processing-service"
|
| 71 |
|
| 72 |
-
MAX_UPLOAD_BYTES = 100 * 1024 * 1024
|
| 73 |
|
| 74 |
MAX_WORKERS = min(32, (os.cpu_count() or 1) + 4)
|
| 75 |
_thread_pool = concurrent.futures.ThreadPoolExecutor(max_workers=MAX_WORKERS)
|
| 76 |
|
| 77 |
_converter = DocumentConverter()
|
| 78 |
|
| 79 |
-
logger.info("
|
| 80 |
-
|
| 81 |
-
|
| 82 |
-
# ---------------------------------------------------------------------------
|
| 83 |
-
# Self-ping
|
| 84 |
-
# ---------------------------------------------------------------------------
|
| 85 |
-
|
| 86 |
-
PING_URL = os.environ.get("PING_URL", "https://validops-us-data-extract.hf.space/health")
|
| 87 |
-
PING_INTERVAL_SECONDS = 30 * 60
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
def _ping_once() -> None:
|
| 91 |
-
ts = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC")
|
| 92 |
-
try:
|
| 93 |
-
with urllib.request.urlopen(PING_URL, timeout=10) as resp:
|
| 94 |
-
logger.info("self_ping | status=%d | url=%s | ts=%s", resp.status, PING_URL, ts)
|
| 95 |
-
except Exception as exc:
|
| 96 |
-
logger.warning("self_ping | failed | url=%s | error=%s | ts=%s", PING_URL, exc, ts)
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
def _ping_loop() -> None:
|
| 100 |
-
logger.info("self_ping | scheduler started | interval_minutes=30 | url=%s", PING_URL)
|
| 101 |
-
while True:
|
| 102 |
-
time.sleep(PING_INTERVAL_SECONDS)
|
| 103 |
-
_ping_once()
|
| 104 |
-
|
| 105 |
-
|
| 106 |
-
def _start_ping_scheduler() -> None:
|
| 107 |
-
thread = threading.Thread(target=_ping_loop, name="self-ping", daemon=True)
|
| 108 |
-
thread.start()
|
| 109 |
|
| 110 |
|
| 111 |
# ---------------------------------------------------------------------------
|
|
@@ -114,12 +86,14 @@ def _start_ping_scheduler() -> None:
|
|
| 114 |
|
| 115 |
@asynccontextmanager
|
| 116 |
async def lifespan(app: FastAPI):
|
| 117 |
-
logger.info(
|
| 118 |
-
|
| 119 |
-
)
|
| 120 |
-
_start_ping_scheduler()
|
| 121 |
yield
|
| 122 |
-
|
|
|
|
|
|
|
|
|
|
| 123 |
|
| 124 |
|
| 125 |
# ---------------------------------------------------------------------------
|
|
@@ -132,7 +106,7 @@ app = FastAPI(
|
|
| 132 |
"Unified API for document-to-Markdown conversion (Microsoft MarkItDown + RapidOCR) "
|
| 133 |
"and PDF-to-image conversion with async job support."
|
| 134 |
),
|
| 135 |
-
version="2.
|
| 136 |
docs_url="/docs",
|
| 137 |
redoc_url="/redoc",
|
| 138 |
lifespan=lifespan,
|
|
@@ -154,9 +128,8 @@ async def request_context_middleware(request: Request, call_next):
|
|
| 154 |
request_id = str(uuid.uuid4())
|
| 155 |
start = time.perf_counter()
|
| 156 |
|
| 157 |
-
|
| 158 |
-
|
| 159 |
-
_structlog.contextvars.bind_contextvars(
|
| 160 |
request_id=request_id,
|
| 161 |
method=request.method,
|
| 162 |
path=request.url.path,
|
|
@@ -165,7 +138,7 @@ async def request_context_middleware(request: Request, call_next):
|
|
| 165 |
response = await call_next(request)
|
| 166 |
elapsed = round((time.perf_counter() - start) * 1000, 2)
|
| 167 |
|
| 168 |
-
logger.info("request_completed
|
| 169 |
response.headers["X-Request-ID"] = request_id
|
| 170 |
response.headers["X-Response-Time-Ms"] = str(elapsed)
|
| 171 |
return response
|
|
@@ -174,7 +147,7 @@ async def request_context_middleware(request: Request, call_next):
|
|
| 174 |
# -- Exception handlers --
|
| 175 |
@app.exception_handler(AppException)
|
| 176 |
async def app_exception_handler(request: Request, exc: AppException):
|
| 177 |
-
logger.warning("app_exception
|
| 178 |
return JSONResponse(
|
| 179 |
status_code=exc.status_code,
|
| 180 |
content={"error": exc.detail, "request_id": getattr(request.state, "request_id", "")},
|
|
@@ -190,10 +163,10 @@ async def generic_exception_handler(request: Request, exc: Exception):
|
|
| 190 |
)
|
| 191 |
|
| 192 |
|
| 193 |
-
# -- Root-level routes (only / , /ping
|
| 194 |
@app.get("/", tags=["System"], summary="Root", include_in_schema=False)
|
| 195 |
async def root():
|
| 196 |
-
return {"service": APP_NAME, "version": "2.
|
| 197 |
|
| 198 |
|
| 199 |
@app.get("/ping", tags=["System"], summary="Ping", include_in_schema=False)
|
|
@@ -427,20 +400,20 @@ async def convert_file(
|
|
| 427 |
except _json.JSONDecodeError:
|
| 428 |
raise HTTPException(status_code=400, detail={"success": False, "message": "Invalid JSON in mappings parameter."})
|
| 429 |
|
| 430 |
-
logger.info("convert_file
|
| 431 |
raw = await file.read()
|
| 432 |
if len(raw) > MAX_UPLOAD_BYTES:
|
| 433 |
-
logger.warning("convert_file | file too large
|
| 434 |
raise HTTPException(status_code=413, detail={"success": False, "message": "File exceeds 100 MB limit."})
|
| 435 |
|
| 436 |
loop = asyncio.get_running_loop()
|
| 437 |
outcome = await loop.run_in_executor(_thread_pool, _converter.convert_stream, raw, file.filename or "upload")
|
| 438 |
|
| 439 |
if isinstance(outcome, ConversionError):
|
| 440 |
-
logger.error("convert_file | conversion failed
|
| 441 |
_raise_for_error(outcome)
|
| 442 |
|
| 443 |
-
logger.info("convert_file | success
|
| 444 |
|
| 445 |
if plain_text:
|
| 446 |
return PlainTextResponse(outcome.markdown)
|
|
@@ -455,7 +428,7 @@ async def convert_file(
|
|
| 455 |
summary="Convert a public URL to Markdown",
|
| 456 |
)
|
| 457 |
async def convert_url(body: UrlRequest):
|
| 458 |
-
logger.info("convert_url
|
| 459 |
parsed = urlparse(body.url)
|
| 460 |
filename = Path(parsed.path).name or "url_content"
|
| 461 |
loop = asyncio.get_running_loop()
|
|
@@ -466,7 +439,7 @@ async def convert_url(body: UrlRequest):
|
|
| 466 |
resp = await client.get(body.url)
|
| 467 |
resp.raise_for_status()
|
| 468 |
except httpx.HTTPError as exc:
|
| 469 |
-
logger.error("convert_url | fetch failed
|
| 470 |
raise HTTPException(status_code=400, detail={"success": False, "message": f"Failed to fetch URL: {exc}"})
|
| 471 |
|
| 472 |
raw_data = resp.content
|
|
@@ -475,18 +448,18 @@ async def convert_url(body: UrlRequest):
|
|
| 475 |
|
| 476 |
outcome = await loop.run_in_executor(_thread_pool, _converter.convert_stream, raw_data, filename)
|
| 477 |
if isinstance(outcome, ConversionError):
|
| 478 |
-
logger.error("convert_url | conversion failed
|
| 479 |
_raise_for_error(outcome)
|
| 480 |
|
| 481 |
-
logger.info("convert_url | success
|
| 482 |
return await _build_response(outcome, return_json=body.return_json, filename=filename, raw_data=raw_data, mappings=body.mappings)
|
| 483 |
|
| 484 |
outcome = await loop.run_in_executor(_thread_pool, _converter.convert_url, body.url)
|
| 485 |
if isinstance(outcome, ConversionError):
|
| 486 |
-
logger.error("convert_url | conversion failed
|
| 487 |
_raise_for_error(outcome)
|
| 488 |
|
| 489 |
-
logger.info("convert_url | success
|
| 490 |
return await _build_response(outcome, return_json=body.return_json, filename=filename, mappings=body.mappings)
|
| 491 |
|
| 492 |
|
|
@@ -505,7 +478,7 @@ async def batch_files(
|
|
| 505 |
raise HTTPException(status_code=400, detail={"success": False, "message": "Maximum 10 files per batch."})
|
| 506 |
|
| 507 |
batch_start = time.perf_counter()
|
| 508 |
-
logger.info("batch_files
|
| 509 |
|
| 510 |
async def _process_file(f: UploadFile) -> BatchFileResult:
|
| 511 |
if f is None:
|
|
@@ -520,7 +493,7 @@ async def batch_files(
|
|
| 520 |
results = await asyncio.gather(*[_process_file(f) for f in files])
|
| 521 |
total_ms = round((time.perf_counter() - batch_start) * 1000, 3)
|
| 522 |
succeeded = sum(1 for r in results if r.success)
|
| 523 |
-
logger.info("batch_files | done
|
| 524 |
|
| 525 |
return BatchResponse(total=len(results), succeeded=succeeded, failed=len(results) - succeeded, total_time_ms=total_ms, results=results)
|
| 526 |
|
|
@@ -533,7 +506,7 @@ async def batch_files(
|
|
| 533 |
)
|
| 534 |
async def batch_urls(body: BatchUrlRequest):
|
| 535 |
batch_start = time.perf_counter()
|
| 536 |
-
logger.info("batch_urls
|
| 537 |
|
| 538 |
async def _process_url(url: str) -> BatchFileResult:
|
| 539 |
loop = asyncio.get_running_loop()
|
|
@@ -543,7 +516,7 @@ async def batch_urls(body: BatchUrlRequest):
|
|
| 543 |
results = await asyncio.gather(*[_process_url(url) for url in body.urls])
|
| 544 |
total_ms = round((time.perf_counter() - batch_start) * 1000, 3)
|
| 545 |
succeeded = sum(1 for r in results if r.success)
|
| 546 |
-
logger.info("batch_urls | done
|
| 547 |
|
| 548 |
return BatchResponse(total=len(results), succeeded=succeeded, failed=len(results) - succeeded, total_time_ms=total_ms, results=results)
|
| 549 |
|
|
|
|
| 20 |
GET /api/v1/files/{id}/{fn}
|
| 21 |
GET /api/v1/health
|
| 22 |
GET /api/v1/ready
|
|
|
|
| 23 |
|
| 24 |
System:
|
| 25 |
GET / Root info
|
|
|
|
| 33 |
|
| 34 |
import asyncio
|
| 35 |
import concurrent.futures
|
| 36 |
+
import json
|
| 37 |
import os
|
|
|
|
| 38 |
import time
|
| 39 |
import uuid
|
|
|
|
| 40 |
from contextlib import asynccontextmanager
|
| 41 |
from datetime import datetime, timezone
|
| 42 |
from pathlib import Path
|
|
|
|
| 44 |
from urllib.parse import urlparse
|
| 45 |
|
| 46 |
import httpx
|
| 47 |
+
import structlog
|
| 48 |
from fastapi import APIRouter, Depends, FastAPI, File, Form, HTTPException, Request, UploadFile, status
|
| 49 |
from fastapi.responses import JSONResponse, PlainTextResponse
|
| 50 |
from fastapi.middleware.cors import CORSMiddleware
|
|
|
|
| 61 |
from app.core.exceptions import AppException
|
| 62 |
from app.core.logging import configure_logging
|
| 63 |
from app.core.rate_limit import RateLimitMiddleware
|
| 64 |
+
from app.services.ping import start_self_ping
|
| 65 |
+
from app.utils.cleanup import cleanup_loop
|
| 66 |
|
| 67 |
configure_logging()
|
| 68 |
logger = get_logger(__name__)
|
|
|
|
| 70 |
_START_TIME = time.time()
|
| 71 |
APP_NAME = "reconciliation-file-processing-service"
|
| 72 |
|
| 73 |
+
MAX_UPLOAD_BYTES = int(os.getenv("MAX_UPLOAD_BYTES", str(100 * 1024 * 1024)))
|
| 74 |
|
| 75 |
MAX_WORKERS = min(32, (os.cpu_count() or 1) + 4)
|
| 76 |
_thread_pool = concurrent.futures.ThreadPoolExecutor(max_workers=MAX_WORKERS)
|
| 77 |
|
| 78 |
_converter = DocumentConverter()
|
| 79 |
|
| 80 |
+
logger.info("thread_pool_initialised", workers=MAX_WORKERS)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 81 |
|
| 82 |
|
| 83 |
# ---------------------------------------------------------------------------
|
|
|
|
| 86 |
|
| 87 |
@asynccontextmanager
|
| 88 |
async def lifespan(app: FastAPI):
|
| 89 |
+
logger.info("api_starting", version="2.2.0", host="0.0.0.0:7860")
|
| 90 |
+
ping_task = await start_self_ping()
|
| 91 |
+
cleanup_task = asyncio.create_task(cleanup_loop())
|
|
|
|
| 92 |
yield
|
| 93 |
+
if ping_task:
|
| 94 |
+
ping_task.cancel()
|
| 95 |
+
cleanup_task.cancel()
|
| 96 |
+
logger.info("api_shutting_down")
|
| 97 |
|
| 98 |
|
| 99 |
# ---------------------------------------------------------------------------
|
|
|
|
| 106 |
"Unified API for document-to-Markdown conversion (Microsoft MarkItDown + RapidOCR) "
|
| 107 |
"and PDF-to-image conversion with async job support."
|
| 108 |
),
|
| 109 |
+
version="2.2.0",
|
| 110 |
docs_url="/docs",
|
| 111 |
redoc_url="/redoc",
|
| 112 |
lifespan=lifespan,
|
|
|
|
| 128 |
request_id = str(uuid.uuid4())
|
| 129 |
start = time.perf_counter()
|
| 130 |
|
| 131 |
+
structlog.contextvars.clear_contextvars()
|
| 132 |
+
structlog.contextvars.bind_contextvars(
|
|
|
|
| 133 |
request_id=request_id,
|
| 134 |
method=request.method,
|
| 135 |
path=request.url.path,
|
|
|
|
| 138 |
response = await call_next(request)
|
| 139 |
elapsed = round((time.perf_counter() - start) * 1000, 2)
|
| 140 |
|
| 141 |
+
logger.info("request_completed", status=response.status_code, duration_ms=elapsed)
|
| 142 |
response.headers["X-Request-ID"] = request_id
|
| 143 |
response.headers["X-Response-Time-Ms"] = str(elapsed)
|
| 144 |
return response
|
|
|
|
| 147 |
# -- Exception handlers --
|
| 148 |
@app.exception_handler(AppException)
|
| 149 |
async def app_exception_handler(request: Request, exc: AppException):
|
| 150 |
+
logger.warning("app_exception", detail=exc.detail, status_code=exc.status_code)
|
| 151 |
return JSONResponse(
|
| 152 |
status_code=exc.status_code,
|
| 153 |
content={"error": exc.detail, "request_id": getattr(request.state, "request_id", "")},
|
|
|
|
| 163 |
)
|
| 164 |
|
| 165 |
|
| 166 |
+
# -- Root-level routes (only / , /ping) --
|
| 167 |
@app.get("/", tags=["System"], summary="Root", include_in_schema=False)
|
| 168 |
async def root():
|
| 169 |
+
return {"service": APP_NAME, "version": "2.2.0", "status": "running"}
|
| 170 |
|
| 171 |
|
| 172 |
@app.get("/ping", tags=["System"], summary="Ping", include_in_schema=False)
|
|
|
|
| 400 |
except _json.JSONDecodeError:
|
| 401 |
raise HTTPException(status_code=400, detail={"success": False, "message": "Invalid JSON in mappings parameter."})
|
| 402 |
|
| 403 |
+
logger.info("convert_file", filename=file.filename)
|
| 404 |
raw = await file.read()
|
| 405 |
if len(raw) > MAX_UPLOAD_BYTES:
|
| 406 |
+
logger.warning("convert_file | file too large", filename=file.filename, size=len(raw))
|
| 407 |
raise HTTPException(status_code=413, detail={"success": False, "message": "File exceeds 100 MB limit."})
|
| 408 |
|
| 409 |
loop = asyncio.get_running_loop()
|
| 410 |
outcome = await loop.run_in_executor(_thread_pool, _converter.convert_stream, raw, file.filename or "upload")
|
| 411 |
|
| 412 |
if isinstance(outcome, ConversionError):
|
| 413 |
+
logger.error("convert_file | conversion failed", filename=file.filename, error=outcome.message)
|
| 414 |
_raise_for_error(outcome)
|
| 415 |
|
| 416 |
+
logger.info("convert_file | success", filename=file.filename, chars=outcome.char_count, time_ms=round(outcome.duration_ms, 1))
|
| 417 |
|
| 418 |
if plain_text:
|
| 419 |
return PlainTextResponse(outcome.markdown)
|
|
|
|
| 428 |
summary="Convert a public URL to Markdown",
|
| 429 |
)
|
| 430 |
async def convert_url(body: UrlRequest):
|
| 431 |
+
logger.info("convert_url", url=body.url)
|
| 432 |
parsed = urlparse(body.url)
|
| 433 |
filename = Path(parsed.path).name or "url_content"
|
| 434 |
loop = asyncio.get_running_loop()
|
|
|
|
| 439 |
resp = await client.get(body.url)
|
| 440 |
resp.raise_for_status()
|
| 441 |
except httpx.HTTPError as exc:
|
| 442 |
+
logger.error("convert_url | fetch failed", url=body.url, error=str(exc))
|
| 443 |
raise HTTPException(status_code=400, detail={"success": False, "message": f"Failed to fetch URL: {exc}"})
|
| 444 |
|
| 445 |
raw_data = resp.content
|
|
|
|
| 448 |
|
| 449 |
outcome = await loop.run_in_executor(_thread_pool, _converter.convert_stream, raw_data, filename)
|
| 450 |
if isinstance(outcome, ConversionError):
|
| 451 |
+
logger.error("convert_url | conversion failed", url=body.url, error=outcome.message)
|
| 452 |
_raise_for_error(outcome)
|
| 453 |
|
| 454 |
+
logger.info("convert_url | success", url=body.url, chars=outcome.char_count, time_ms=round(outcome.duration_ms, 1))
|
| 455 |
return await _build_response(outcome, return_json=body.return_json, filename=filename, raw_data=raw_data, mappings=body.mappings)
|
| 456 |
|
| 457 |
outcome = await loop.run_in_executor(_thread_pool, _converter.convert_url, body.url)
|
| 458 |
if isinstance(outcome, ConversionError):
|
| 459 |
+
logger.error("convert_url | conversion failed", url=body.url, error=outcome.message)
|
| 460 |
_raise_for_error(outcome)
|
| 461 |
|
| 462 |
+
logger.info("convert_url | success", url=body.url, chars=outcome.char_count, time_ms=round(outcome.duration_ms, 1))
|
| 463 |
return await _build_response(outcome, return_json=body.return_json, filename=filename, mappings=body.mappings)
|
| 464 |
|
| 465 |
|
|
|
|
| 478 |
raise HTTPException(status_code=400, detail={"success": False, "message": "Maximum 10 files per batch."})
|
| 479 |
|
| 480 |
batch_start = time.perf_counter()
|
| 481 |
+
logger.info("batch_files", count=len(files))
|
| 482 |
|
| 483 |
async def _process_file(f: UploadFile) -> BatchFileResult:
|
| 484 |
if f is None:
|
|
|
|
| 493 |
results = await asyncio.gather(*[_process_file(f) for f in files])
|
| 494 |
total_ms = round((time.perf_counter() - batch_start) * 1000, 3)
|
| 495 |
succeeded = sum(1 for r in results if r.success)
|
| 496 |
+
logger.info("batch_files | done", succeeded=succeeded, failed=len(results) - succeeded, total_ms=round(total_ms, 1))
|
| 497 |
|
| 498 |
return BatchResponse(total=len(results), succeeded=succeeded, failed=len(results) - succeeded, total_time_ms=total_ms, results=results)
|
| 499 |
|
|
|
|
| 506 |
)
|
| 507 |
async def batch_urls(body: BatchUrlRequest):
|
| 508 |
batch_start = time.perf_counter()
|
| 509 |
+
logger.info("batch_urls", count=len(body.urls))
|
| 510 |
|
| 511 |
async def _process_url(url: str) -> BatchFileResult:
|
| 512 |
loop = asyncio.get_running_loop()
|
|
|
|
| 516 |
results = await asyncio.gather(*[_process_url(url) for url in body.urls])
|
| 517 |
total_ms = round((time.perf_counter() - batch_start) * 1000, 3)
|
| 518 |
succeeded = sum(1 for r in results if r.success)
|
| 519 |
+
logger.info("batch_urls | done", succeeded=succeeded, failed=len(results) - succeeded, total_ms=round(total_ms, 1))
|
| 520 |
|
| 521 |
return BatchResponse(total=len(results), succeeded=succeeded, failed=len(results) - succeeded, total_time_ms=total_ms, results=results)
|
| 522 |
|
app/banner.py
DELETED
|
@@ -1,21 +0,0 @@
|
|
| 1 |
-
import pyfiglet
|
| 2 |
-
from rich.console import Console
|
| 3 |
-
from rich.rule import Rule
|
| 4 |
-
from rich.text import Text
|
| 5 |
-
|
| 6 |
-
from app.core.config import get_settings
|
| 7 |
-
|
| 8 |
-
_console = Console()
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
def print_banner() -> None:
|
| 12 |
-
settings = get_settings()
|
| 13 |
-
|
| 14 |
-
art = pyfiglet.figlet_format("ValidOps", font="slant")
|
| 15 |
-
|
| 16 |
-
_console.print(Text(art, style="bold cyan"))
|
| 17 |
-
_console.print(f" [bold white]{'Service:':<14}[/bold white] [cyan]{settings.APP_NAME}[/cyan]")
|
| 18 |
-
_console.print(f" [bold white]{'Version:':<14}[/bold white] [cyan]v{settings.VERSION}[/cyan]")
|
| 19 |
-
_console.print(f" [bold white]{'Environment:':<14}[/bold white] [cyan]{settings.ENV}[/cyan]")
|
| 20 |
-
_console.print(Rule(style="dim cyan"))
|
| 21 |
-
_console.print()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
app/core/exceptions.py
CHANGED
|
@@ -54,8 +54,3 @@ class InvalidParameterError(AppException):
|
|
| 54 |
class FileServiceError(AppException):
|
| 55 |
def __init__(self, detail: str = "File upload service error"):
|
| 56 |
super().__init__(detail, status.HTTP_502_BAD_GATEWAY)
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
class CSVGenerationError(AppException):
|
| 60 |
-
def __init__(self, detail: str = "CSV generation failed"):
|
| 61 |
-
super().__init__(detail, status.HTTP_500_INTERNAL_SERVER_ERROR)
|
|
|
|
| 54 |
class FileServiceError(AppException):
|
| 55 |
def __init__(self, detail: str = "File upload service error"):
|
| 56 |
super().__init__(detail, status.HTTP_502_BAD_GATEWAY)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
app/main.py
DELETED
|
@@ -1,95 +0,0 @@
|
|
| 1 |
-
from __future__ import annotations
|
| 2 |
-
|
| 3 |
-
import time
|
| 4 |
-
import uuid
|
| 5 |
-
from contextlib import asynccontextmanager
|
| 6 |
-
|
| 7 |
-
import structlog
|
| 8 |
-
from fastapi import FastAPI, Request
|
| 9 |
-
from fastapi.middleware.cors import CORSMiddleware
|
| 10 |
-
from fastapi.middleware.gzip import GZipMiddleware
|
| 11 |
-
from fastapi.responses import JSONResponse
|
| 12 |
-
from prometheus_client import make_asgi_app
|
| 13 |
-
|
| 14 |
-
from app.api.routes import router
|
| 15 |
-
from app.core.config import settings
|
| 16 |
-
from app.core.exceptions import AppException
|
| 17 |
-
from app.core.logging import configure_logging
|
| 18 |
-
from app.core.rate_limit import RateLimitMiddleware
|
| 19 |
-
configure_logging()
|
| 20 |
-
logger = structlog.get_logger(__name__)
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
@asynccontextmanager
|
| 24 |
-
async def lifespan(app: FastAPI):
|
| 25 |
-
logger.info("startup", service=settings.APP_NAME, version=settings.VERSION, env=settings.ENV)
|
| 26 |
-
yield
|
| 27 |
-
logger.info("shutdown", service=settings.APP_NAME)
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
app = FastAPI(
|
| 31 |
-
title=settings.APP_NAME,
|
| 32 |
-
description="High-performance PDF to Image conversion and CSV report generation API",
|
| 33 |
-
version=settings.VERSION,
|
| 34 |
-
docs_url="/docs",
|
| 35 |
-
redoc_url="/redoc",
|
| 36 |
-
lifespan=lifespan,
|
| 37 |
-
)
|
| 38 |
-
|
| 39 |
-
app.add_middleware(RateLimitMiddleware)
|
| 40 |
-
app.add_middleware(GZipMiddleware, minimum_size=1000)
|
| 41 |
-
app.add_middleware(
|
| 42 |
-
CORSMiddleware,
|
| 43 |
-
allow_origins=settings.ALLOWED_ORIGINS,
|
| 44 |
-
allow_credentials=True,
|
| 45 |
-
allow_methods=["*"],
|
| 46 |
-
allow_headers=["*"],
|
| 47 |
-
)
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
@app.middleware("http")
|
| 51 |
-
async def request_context_middleware(request: Request, call_next):
|
| 52 |
-
request_id = str(uuid.uuid4())
|
| 53 |
-
start = time.perf_counter()
|
| 54 |
-
request.state.request_id = request_id
|
| 55 |
-
|
| 56 |
-
structlog.contextvars.clear_contextvars()
|
| 57 |
-
structlog.contextvars.bind_contextvars(
|
| 58 |
-
request_id=request_id,
|
| 59 |
-
method=request.method,
|
| 60 |
-
path=request.url.path,
|
| 61 |
-
)
|
| 62 |
-
|
| 63 |
-
response = await call_next(request)
|
| 64 |
-
elapsed = round((time.perf_counter() - start) * 1000, 2)
|
| 65 |
-
|
| 66 |
-
logger.info("request_completed", status_code=response.status_code, duration_ms=elapsed)
|
| 67 |
-
response.headers["X-Request-ID"] = request_id
|
| 68 |
-
response.headers["X-Response-Time-Ms"] = str(elapsed)
|
| 69 |
-
return response
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
@app.exception_handler(AppException)
|
| 73 |
-
async def app_exception_handler(request: Request, exc: AppException):
|
| 74 |
-
logger.warning("app_exception", detail=exc.detail, status_code=exc.status_code)
|
| 75 |
-
return JSONResponse(
|
| 76 |
-
status_code=exc.status_code,
|
| 77 |
-
content={"error": exc.detail, "request_id": request.state.request_id},
|
| 78 |
-
)
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
@app.exception_handler(Exception)
|
| 82 |
-
async def generic_exception_handler(request: Request, exc: Exception):
|
| 83 |
-
logger.exception("unhandled_exception", exc_info=exc)
|
| 84 |
-
return JSONResponse(
|
| 85 |
-
status_code=500,
|
| 86 |
-
content={"error": "Internal server error", "request_id": request.state.request_id},
|
| 87 |
-
)
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
@app.get("/")
|
| 91 |
-
async def root():
|
| 92 |
-
return {"message": f"{settings.APP_NAME} v{settings.VERSION}"}
|
| 93 |
-
|
| 94 |
-
app.include_router(router, prefix="/api/v1")
|
| 95 |
-
app.mount("/metrics", make_asgi_app())
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
app/services/ping.py
CHANGED
|
@@ -1,6 +1,7 @@
|
|
| 1 |
from __future__ import annotations
|
| 2 |
|
| 3 |
import asyncio
|
|
|
|
| 4 |
|
| 5 |
import httpx
|
| 6 |
import structlog
|
|
@@ -10,11 +11,13 @@ from app.core.config import settings
|
|
| 10 |
logger = structlog.get_logger(__name__)
|
| 11 |
|
| 12 |
|
| 13 |
-
async def start_self_ping() ->
|
| 14 |
if not settings.SELF_PING_ENABLED or not settings.SELF_PING_URL:
|
| 15 |
logger.info("self_ping_disabled")
|
| 16 |
-
return
|
| 17 |
-
asyncio.create_task(_ping_loop())
|
|
|
|
|
|
|
| 18 |
|
| 19 |
|
| 20 |
async def _ping_loop() -> None:
|
|
|
|
| 1 |
from __future__ import annotations
|
| 2 |
|
| 3 |
import asyncio
|
| 4 |
+
from typing import Optional
|
| 5 |
|
| 6 |
import httpx
|
| 7 |
import structlog
|
|
|
|
| 11 |
logger = structlog.get_logger(__name__)
|
| 12 |
|
| 13 |
|
| 14 |
+
async def start_self_ping() -> Optional[asyncio.Task]:
|
| 15 |
if not settings.SELF_PING_ENABLED or not settings.SELF_PING_URL:
|
| 16 |
logger.info("self_ping_disabled")
|
| 17 |
+
return None
|
| 18 |
+
task = asyncio.create_task(_ping_loop())
|
| 19 |
+
logger.info("self_ping_started", url=settings.SELF_PING_URL, interval=settings.SELF_PING_INTERVAL_SECONDS)
|
| 20 |
+
return task
|
| 21 |
|
| 22 |
|
| 23 |
async def _ping_loop() -> None:
|
banner.py
CHANGED
|
@@ -8,7 +8,7 @@ from rich.text import Text
|
|
| 8 |
_console = Console()
|
| 9 |
|
| 10 |
SERVICE_NAME = os.getenv("SERVICE_NAME", "reconciliation-file-processing-service")
|
| 11 |
-
API_VERSION = os.getenv("API_VERSION", "2.
|
| 12 |
ENVIRONMENT = os.getenv("ENVIRONMENT", "production")
|
| 13 |
|
| 14 |
|
|
|
|
| 8 |
_console = Console()
|
| 9 |
|
| 10 |
SERVICE_NAME = os.getenv("SERVICE_NAME", "reconciliation-file-processing-service")
|
| 11 |
+
API_VERSION = os.getenv("API_VERSION", "2.2.0")
|
| 12 |
ENVIRONMENT = os.getenv("ENVIRONMENT", "production")
|
| 13 |
|
| 14 |
|
core/__init__.py
CHANGED
|
@@ -1,15 +1,8 @@
|
|
| 1 |
from .converter import ConversionError, ConversionResult, DocumentConverter, SUPPORTED_EXTENSIONS
|
| 2 |
-
from .batch import BatchProcessor, BatchReport
|
| 3 |
-
from .output import OutputWriter
|
| 4 |
-
from .ocr_engine import ocr_image
|
| 5 |
|
| 6 |
__all__ = [
|
| 7 |
"ConversionError",
|
| 8 |
"ConversionResult",
|
| 9 |
"DocumentConverter",
|
| 10 |
"SUPPORTED_EXTENSIONS",
|
| 11 |
-
"BatchProcessor",
|
| 12 |
-
"BatchReport",
|
| 13 |
-
"OutputWriter",
|
| 14 |
-
"ocr_image",
|
| 15 |
]
|
|
|
|
| 1 |
from .converter import ConversionError, ConversionResult, DocumentConverter, SUPPORTED_EXTENSIONS
|
|
|
|
|
|
|
|
|
|
| 2 |
|
| 3 |
__all__ = [
|
| 4 |
"ConversionError",
|
| 5 |
"ConversionResult",
|
| 6 |
"DocumentConverter",
|
| 7 |
"SUPPORTED_EXTENSIONS",
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8 |
]
|
core/converter.py
CHANGED
|
@@ -37,6 +37,13 @@ from logger import get_logger
|
|
| 37 |
|
| 38 |
logger = get_logger(__name__)
|
| 39 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 40 |
|
| 41 |
# ---------------------------------------------------------------------------
|
| 42 |
# Supported formats
|
|
@@ -51,14 +58,6 @@ SUPPORTED_EXTENSIONS = {
|
|
| 51 |
".zip", ".epub",
|
| 52 |
}
|
| 53 |
|
| 54 |
-
# Extensions that route through RapidOCR rather than MarkItDown.
|
| 55 |
-
IMAGE_EXTENSIONS = {
|
| 56 |
-
".jpg", ".jpeg", ".png", ".gif", ".bmp", ".webp", ".tiff",
|
| 57 |
-
}
|
| 58 |
-
|
| 59 |
-
IMAGE_MIME_PREFIXES = {"image/"}
|
| 60 |
-
|
| 61 |
-
|
| 62 |
def _is_image(ext: str, mime: str) -> bool:
|
| 63 |
"""Return True when the input should be routed through RapidOCR."""
|
| 64 |
return ext.lower() in IMAGE_EXTENSIONS or any(
|
|
|
|
| 37 |
|
| 38 |
logger = get_logger(__name__)
|
| 39 |
|
| 40 |
+
# Extensions that route through RapidOCR rather than MarkItDown.
|
| 41 |
+
IMAGE_EXTENSIONS = {
|
| 42 |
+
".jpg", ".jpeg", ".png", ".gif", ".bmp", ".webp", ".tiff",
|
| 43 |
+
}
|
| 44 |
+
|
| 45 |
+
IMAGE_MIME_PREFIXES = {"image/"}
|
| 46 |
+
|
| 47 |
|
| 48 |
# ---------------------------------------------------------------------------
|
| 49 |
# Supported formats
|
|
|
|
| 58 |
".zip", ".epub",
|
| 59 |
}
|
| 60 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 61 |
def _is_image(ext: str, mime: str) -> bool:
|
| 62 |
"""Return True when the input should be routed through RapidOCR."""
|
| 63 |
return ext.lower() in IMAGE_EXTENSIONS or any(
|
extraction/json_extractor.py
CHANGED
|
@@ -20,10 +20,11 @@ logger = get_logger(__name__)
|
|
| 20 |
SUPPORTED_EXTENSIONS = {'.csv', '.xls', '.xlsx'}
|
| 21 |
|
| 22 |
# Resource limits to prevent abuse
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
|
|
|
|
| 27 |
|
| 28 |
|
| 29 |
def _validate_file_size(size: int) -> Optional[str]:
|
|
|
|
| 20 |
SUPPORTED_EXTENSIONS = {'.csv', '.xls', '.xlsx'}
|
| 21 |
|
| 22 |
# Resource limits to prevent abuse
|
| 23 |
+
from app.core.config import settings as _app_settings
|
| 24 |
+
MAX_FILE_SIZE_BYTES = _app_settings.MAX_FILE_SIZE_BYTES
|
| 25 |
+
MAX_CSV_ROWS = 100000
|
| 26 |
+
MAX_EXCEL_ROWS = 50000
|
| 27 |
+
MAX_MEMORY_ROWS = 100000
|
| 28 |
|
| 29 |
|
| 30 |
def _validate_file_size(size: int) -> Optional[str]:
|
extraction/spacy_extractor.py
CHANGED
|
@@ -66,10 +66,6 @@ def _get_nlp():
|
|
| 66 |
return _nlp
|
| 67 |
|
| 68 |
|
| 69 |
-
def clean_text(text: str) -> str:
|
| 70 |
-
return text
|
| 71 |
-
|
| 72 |
-
|
| 73 |
# ---------------------------------------------------------------------------
|
| 74 |
# Normalizer registry
|
| 75 |
# ---------------------------------------------------------------------------
|
|
@@ -600,7 +596,7 @@ def extract_fields(
|
|
| 600 |
Backward-compatible: callers that pass flat scalar rules unchanged still work.
|
| 601 |
"""
|
| 602 |
nlp = _get_nlp()
|
| 603 |
-
cleaned =
|
| 604 |
|
| 605 |
try:
|
| 606 |
doc = next(iter(nlp.pipe([cleaned])))
|
|
@@ -642,7 +638,7 @@ def extract_schema(
|
|
| 642 |
# → [{"date": "Jan 2024", "amount": "$1,200"}, ...]
|
| 643 |
"""
|
| 644 |
nlp = _get_nlp()
|
| 645 |
-
cleaned =
|
| 646 |
|
| 647 |
try:
|
| 648 |
doc = next(iter(nlp.pipe([cleaned])))
|
|
|
|
| 66 |
return _nlp
|
| 67 |
|
| 68 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 69 |
# ---------------------------------------------------------------------------
|
| 70 |
# Normalizer registry
|
| 71 |
# ---------------------------------------------------------------------------
|
|
|
|
| 596 |
Backward-compatible: callers that pass flat scalar rules unchanged still work.
|
| 597 |
"""
|
| 598 |
nlp = _get_nlp()
|
| 599 |
+
cleaned = text
|
| 600 |
|
| 601 |
try:
|
| 602 |
doc = next(iter(nlp.pipe([cleaned])))
|
|
|
|
| 638 |
# → [{"date": "Jan 2024", "amount": "$1,200"}, ...]
|
| 639 |
"""
|
| 640 |
nlp = _get_nlp()
|
| 641 |
+
cleaned = text
|
| 642 |
|
| 643 |
try:
|
| 644 |
doc = next(iter(nlp.pipe([cleaned])))
|
logger.py
CHANGED
|
@@ -1,87 +1,20 @@
|
|
| 1 |
-
"""
|
| 2 |
|
| 3 |
-
|
| 4 |
-
|
| 5 |
|
| 6 |
Usage
|
| 7 |
-----
|
| 8 |
from logger import get_logger
|
| 9 |
|
| 10 |
logger = get_logger(__name__)
|
| 11 |
-
logger.info("
|
| 12 |
-
|
| 13 |
-
Configuration (environment variables):
|
| 14 |
-
|
| 15 |
-
LOG_LEVEL Minimum level (default: INFO)
|
| 16 |
"""
|
| 17 |
|
| 18 |
from __future__ import annotations
|
| 19 |
|
| 20 |
-
import
|
| 21 |
-
import logging
|
| 22 |
-
import os
|
| 23 |
-
import sys
|
| 24 |
-
from datetime import datetime, timezone
|
| 25 |
-
from typing import Any
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
__all__ = ["get_logger", "app_logger"]
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
class JSONFormatter(logging.Formatter):
|
| 32 |
-
"""Render log records as JSON lines.
|
| 33 |
-
|
| 34 |
-
Each line contains ``timestamp`` (ISO 8601 UTC), ``level``,
|
| 35 |
-
``logger``, and ``message``. Exception tracebacks are included
|
| 36 |
-
under ``exception`` when ``exc_info`` is set on the record.
|
| 37 |
-
"""
|
| 38 |
-
|
| 39 |
-
__slots__ = ()
|
| 40 |
-
|
| 41 |
-
def format(self, record: logging.LogRecord) -> str:
|
| 42 |
-
payload: dict[str, Any] = {
|
| 43 |
-
"timestamp": datetime.fromtimestamp(
|
| 44 |
-
record.created, tz=timezone.utc
|
| 45 |
-
).isoformat(),
|
| 46 |
-
"level": record.levelname,
|
| 47 |
-
"logger": record.name,
|
| 48 |
-
"message": record.getMessage(),
|
| 49 |
-
}
|
| 50 |
-
if record.exc_info and record.exc_info[0] is not None:
|
| 51 |
-
payload["exception"] = self.formatException(record.exc_info)
|
| 52 |
-
return json.dumps(payload, default=str, ensure_ascii=False, sort_keys=True)
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
def get_logger(name: str) -> logging.Logger:
|
| 56 |
-
"""Return a named logger.
|
| 57 |
-
|
| 58 |
-
All loggers inherit handlers from the root logger, which is
|
| 59 |
-
configured once at import time with a :class:`JSONFormatter`.
|
| 60 |
-
"""
|
| 61 |
-
return logging.getLogger(name)
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
_initialised: bool = False
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
def _setup() -> None:
|
| 68 |
-
"""Configure the root logger once from environment defaults."""
|
| 69 |
-
global _initialised
|
| 70 |
-
if _initialised:
|
| 71 |
-
return
|
| 72 |
-
_initialised = True
|
| 73 |
-
|
| 74 |
-
level = os.getenv("LOG_LEVEL", "INFO").upper()
|
| 75 |
-
|
| 76 |
-
root = logging.getLogger()
|
| 77 |
-
root.setLevel(getattr(logging, level, logging.INFO))
|
| 78 |
-
|
| 79 |
-
fmt = JSONFormatter()
|
| 80 |
-
|
| 81 |
-
console = logging.StreamHandler(sys.stdout)
|
| 82 |
-
console.setFormatter(fmt)
|
| 83 |
-
root.addHandler(console)
|
| 84 |
|
|
|
|
| 85 |
|
| 86 |
-
|
| 87 |
-
app_logger = get_logger("docx")
|
|
|
|
| 1 |
+
"""Centralised logging — re-exports structlog.getLogger.
|
| 2 |
|
| 3 |
+
All application code should use this module to obtain loggers,
|
| 4 |
+
ensuring consistent structlog configuration throughout the codebase.
|
| 5 |
|
| 6 |
Usage
|
| 7 |
-----
|
| 8 |
from logger import get_logger
|
| 9 |
|
| 10 |
logger = get_logger(__name__)
|
| 11 |
+
logger.info("event", key="value", duration_ms=42.0)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 12 |
"""
|
| 13 |
|
| 14 |
from __future__ import annotations
|
| 15 |
|
| 16 |
+
import structlog
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 17 |
|
| 18 |
+
__all__ = ["get_logger"]
|
| 19 |
|
| 20 |
+
get_logger = structlog.get_logger
|
|
|
self_ping.py
DELETED
|
@@ -1,19 +0,0 @@
|
|
| 1 |
-
#!/usr/bin/env python3
|
| 2 |
-
"""Self-ping script that hits a URL."""
|
| 3 |
-
|
| 4 |
-
import datetime
|
| 5 |
-
import os
|
| 6 |
-
import urllib.request
|
| 7 |
-
|
| 8 |
-
PING_URL = os.environ.get("PING_URL", "https://your-url.com/health")
|
| 9 |
-
|
| 10 |
-
def main():
|
| 11 |
-
timestamp = datetime.datetime.now().isoformat()
|
| 12 |
-
try:
|
| 13 |
-
urllib.request.urlopen(PING_URL, timeout=10)
|
| 14 |
-
print(f"[{timestamp}] Pinged {PING_URL} - OK")
|
| 15 |
-
except Exception as e:
|
| 16 |
-
print(f"[{timestamp}] Ping failed: {e}")
|
| 17 |
-
|
| 18 |
-
if __name__ == "__main__":
|
| 19 |
-
main()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|