validops-east-1 commited on
Commit
697be7d
·
1 Parent(s): d5eb307
.gitattributes ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ * text=auto
2
+ *.sh text eol=lf
3
+ *.py text eol=lf
4
+ Dockerfile text eol=lf
5
+ *.ini text eol=lf
6
+ *.cfg text eol=lf
7
+ *.yml text eol=lf
8
+ *.yaml text eol=lf
9
+ *.json text eol=lf
10
+ *.md text eol=lf
11
+ .gitignore text eol=lf
12
+ .dockerignore text eol=lf
13
+ .env text eol=lf
.gitignore CHANGED
@@ -125,4 +125,6 @@ run_docker.py
125
  docs
126
  .claude
127
  start_docker.txt
128
- Reconciliation-file-processing-service.postman_collection.json
 
 
 
125
  docs
126
  .claude
127
  start_docker.txt
128
+ Reconciliation-file-processing-service.postman_collection.json
129
+ pytest.ini
130
+ tests/*
Dockerfile CHANGED
@@ -5,9 +5,10 @@
5
 
6
  FROM python:3.12-slim
7
 
8
- LABEL maintainer="Reconciliation API"
9
  LABEL description="Document-to-Markdown & PDF-to-image API"
10
  LABEL version="2.2.0"
 
11
 
12
  # ── System dependencies ──────────────────────────────────────
13
  RUN apt-get update && apt-get install -y --no-install-recommends \
 
5
 
6
  FROM python:3.12-slim
7
 
8
+ LABEL maintainer="Reconciliation File Processing Service API"
9
  LABEL description="Document-to-Markdown & PDF-to-image API"
10
  LABEL version="2.2.0"
11
+ LABEL description="Production-ready: consolidated logging, unified app, cleanup loop, self-ping"
12
 
13
  # ── System dependencies ──────────────────────────────────────
14
  RUN apt-get update && apt-get install -y --no-install-recommends \
__init__.py CHANGED
@@ -1,4 +1,4 @@
1
- from .core import ConversionError, ConversionResult, DocumentConverter, SUPPORTED_EXTENSIONS
2
 
3
  __all__ = [
4
  "ConversionError",
 
1
+ from core.converter import ConversionError, ConversionResult, DocumentConverter, SUPPORTED_EXTENSIONS
2
 
3
  __all__ = [
4
  "ConversionError",
api/server.py CHANGED
@@ -20,7 +20,6 @@ PDF-to-image conversion (under /api/v1):
20
  GET /api/v1/files/{id}/{fn}
21
  GET /api/v1/health
22
  GET /api/v1/ready
23
- GET /api/v1/ping
24
 
25
  System:
26
  GET / Root info
@@ -34,11 +33,10 @@ from __future__ import annotations
34
 
35
  import asyncio
36
  import concurrent.futures
 
37
  import os
38
- import threading
39
  import time
40
  import uuid
41
- import urllib.request
42
  from contextlib import asynccontextmanager
43
  from datetime import datetime, timezone
44
  from pathlib import Path
@@ -46,6 +44,7 @@ from typing import Annotated, Any, Dict, List, Optional
46
  from urllib.parse import urlparse
47
 
48
  import httpx
 
49
  from fastapi import APIRouter, Depends, FastAPI, File, Form, HTTPException, Request, UploadFile, status
50
  from fastapi.responses import JSONResponse, PlainTextResponse
51
  from fastapi.middleware.cors import CORSMiddleware
@@ -62,6 +61,8 @@ from app.core.config import settings
62
  from app.core.exceptions import AppException
63
  from app.core.logging import configure_logging
64
  from app.core.rate_limit import RateLimitMiddleware
 
 
65
 
66
  configure_logging()
67
  logger = get_logger(__name__)
@@ -69,43 +70,14 @@ logger = get_logger(__name__)
69
  _START_TIME = time.time()
70
  APP_NAME = "reconciliation-file-processing-service"
71
 
72
- MAX_UPLOAD_BYTES = 100 * 1024 * 1024
73
 
74
  MAX_WORKERS = min(32, (os.cpu_count() or 1) + 4)
75
  _thread_pool = concurrent.futures.ThreadPoolExecutor(max_workers=MAX_WORKERS)
76
 
77
  _converter = DocumentConverter()
78
 
79
- logger.info("Thread pool initialised with %d workers", MAX_WORKERS)
80
-
81
-
82
- # ---------------------------------------------------------------------------
83
- # Self-ping
84
- # ---------------------------------------------------------------------------
85
-
86
- PING_URL = os.environ.get("PING_URL", "https://validops-us-data-extract.hf.space/health")
87
- PING_INTERVAL_SECONDS = 30 * 60
88
-
89
-
90
- def _ping_once() -> None:
91
- ts = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC")
92
- try:
93
- with urllib.request.urlopen(PING_URL, timeout=10) as resp:
94
- logger.info("self_ping | status=%d | url=%s | ts=%s", resp.status, PING_URL, ts)
95
- except Exception as exc:
96
- logger.warning("self_ping | failed | url=%s | error=%s | ts=%s", PING_URL, exc, ts)
97
-
98
-
99
- def _ping_loop() -> None:
100
- logger.info("self_ping | scheduler started | interval_minutes=30 | url=%s", PING_URL)
101
- while True:
102
- time.sleep(PING_INTERVAL_SECONDS)
103
- _ping_once()
104
-
105
-
106
- def _start_ping_scheduler() -> None:
107
- thread = threading.Thread(target=_ping_loop, name="self-ping", daemon=True)
108
- thread.start()
109
 
110
 
111
  # ---------------------------------------------------------------------------
@@ -114,12 +86,14 @@ def _start_ping_scheduler() -> None:
114
 
115
  @asynccontextmanager
116
  async def lifespan(app: FastAPI):
117
- logger.info(
118
- "MarkItDown & PDF Conversion API starting | version=2.1.0 | host=0.0.0.0:7860",
119
- )
120
- _start_ping_scheduler()
121
  yield
122
- logger.info("MarkItDown & PDF Conversion API shutting down")
 
 
 
123
 
124
 
125
  # ---------------------------------------------------------------------------
@@ -132,7 +106,7 @@ app = FastAPI(
132
  "Unified API for document-to-Markdown conversion (Microsoft MarkItDown + RapidOCR) "
133
  "and PDF-to-image conversion with async job support."
134
  ),
135
- version="2.1.0",
136
  docs_url="/docs",
137
  redoc_url="/redoc",
138
  lifespan=lifespan,
@@ -154,9 +128,8 @@ async def request_context_middleware(request: Request, call_next):
154
  request_id = str(uuid.uuid4())
155
  start = time.perf_counter()
156
 
157
- import structlog as _structlog
158
- _structlog.contextvars.clear_contextvars()
159
- _structlog.contextvars.bind_contextvars(
160
  request_id=request_id,
161
  method=request.method,
162
  path=request.url.path,
@@ -165,7 +138,7 @@ async def request_context_middleware(request: Request, call_next):
165
  response = await call_next(request)
166
  elapsed = round((time.perf_counter() - start) * 1000, 2)
167
 
168
- logger.info("request_completed | status=%d | duration_ms=%.2f", response.status_code, elapsed)
169
  response.headers["X-Request-ID"] = request_id
170
  response.headers["X-Response-Time-Ms"] = str(elapsed)
171
  return response
@@ -174,7 +147,7 @@ async def request_context_middleware(request: Request, call_next):
174
  # -- Exception handlers --
175
  @app.exception_handler(AppException)
176
  async def app_exception_handler(request: Request, exc: AppException):
177
- logger.warning("app_exception | detail=%s | status_code=%d", exc.detail, exc.status_code)
178
  return JSONResponse(
179
  status_code=exc.status_code,
180
  content={"error": exc.detail, "request_id": getattr(request.state, "request_id", "")},
@@ -190,10 +163,10 @@ async def generic_exception_handler(request: Request, exc: Exception):
190
  )
191
 
192
 
193
- # -- Root-level routes (only / , /ping , /health) --
194
  @app.get("/", tags=["System"], summary="Root", include_in_schema=False)
195
  async def root():
196
- return {"service": APP_NAME, "version": "2.1.0", "status": "running"}
197
 
198
 
199
  @app.get("/ping", tags=["System"], summary="Ping", include_in_schema=False)
@@ -427,20 +400,20 @@ async def convert_file(
427
  except _json.JSONDecodeError:
428
  raise HTTPException(status_code=400, detail={"success": False, "message": "Invalid JSON in mappings parameter."})
429
 
430
- logger.info("convert_file | filename=%s", file.filename)
431
  raw = await file.read()
432
  if len(raw) > MAX_UPLOAD_BYTES:
433
- logger.warning("convert_file | file too large | filename=%s | size=%d", file.filename, len(raw))
434
  raise HTTPException(status_code=413, detail={"success": False, "message": "File exceeds 100 MB limit."})
435
 
436
  loop = asyncio.get_running_loop()
437
  outcome = await loop.run_in_executor(_thread_pool, _converter.convert_stream, raw, file.filename or "upload")
438
 
439
  if isinstance(outcome, ConversionError):
440
- logger.error("convert_file | conversion failed | filename=%s | error=%s", file.filename, outcome.message)
441
  _raise_for_error(outcome)
442
 
443
- logger.info("convert_file | success | filename=%s | chars=%d | time_ms=%.1f", file.filename, outcome.char_count, outcome.duration_ms)
444
 
445
  if plain_text:
446
  return PlainTextResponse(outcome.markdown)
@@ -455,7 +428,7 @@ async def convert_file(
455
  summary="Convert a public URL to Markdown",
456
  )
457
  async def convert_url(body: UrlRequest):
458
- logger.info("convert_url | url=%s", body.url)
459
  parsed = urlparse(body.url)
460
  filename = Path(parsed.path).name or "url_content"
461
  loop = asyncio.get_running_loop()
@@ -466,7 +439,7 @@ async def convert_url(body: UrlRequest):
466
  resp = await client.get(body.url)
467
  resp.raise_for_status()
468
  except httpx.HTTPError as exc:
469
- logger.error("convert_url | fetch failed | url=%s | error=%s", body.url, exc)
470
  raise HTTPException(status_code=400, detail={"success": False, "message": f"Failed to fetch URL: {exc}"})
471
 
472
  raw_data = resp.content
@@ -475,18 +448,18 @@ async def convert_url(body: UrlRequest):
475
 
476
  outcome = await loop.run_in_executor(_thread_pool, _converter.convert_stream, raw_data, filename)
477
  if isinstance(outcome, ConversionError):
478
- logger.error("convert_url | conversion failed | url=%s | error=%s", body.url, outcome.message)
479
  _raise_for_error(outcome)
480
 
481
- logger.info("convert_url | success | url=%s | chars=%d | time_ms=%.1f", body.url, outcome.char_count, outcome.duration_ms)
482
  return await _build_response(outcome, return_json=body.return_json, filename=filename, raw_data=raw_data, mappings=body.mappings)
483
 
484
  outcome = await loop.run_in_executor(_thread_pool, _converter.convert_url, body.url)
485
  if isinstance(outcome, ConversionError):
486
- logger.error("convert_url | conversion failed | url=%s | error=%s", body.url, outcome.message)
487
  _raise_for_error(outcome)
488
 
489
- logger.info("convert_url | success | url=%s | chars=%d | time_ms=%.1f", body.url, outcome.char_count, outcome.duration_ms)
490
  return await _build_response(outcome, return_json=body.return_json, filename=filename, mappings=body.mappings)
491
 
492
 
@@ -505,7 +478,7 @@ async def batch_files(
505
  raise HTTPException(status_code=400, detail={"success": False, "message": "Maximum 10 files per batch."})
506
 
507
  batch_start = time.perf_counter()
508
- logger.info("batch_files | count=%d", len(files))
509
 
510
  async def _process_file(f: UploadFile) -> BatchFileResult:
511
  if f is None:
@@ -520,7 +493,7 @@ async def batch_files(
520
  results = await asyncio.gather(*[_process_file(f) for f in files])
521
  total_ms = round((time.perf_counter() - batch_start) * 1000, 3)
522
  succeeded = sum(1 for r in results if r.success)
523
- logger.info("batch_files | done | succeeded=%d | failed=%d | total_ms=%.1f", succeeded, len(results) - succeeded, total_ms)
524
 
525
  return BatchResponse(total=len(results), succeeded=succeeded, failed=len(results) - succeeded, total_time_ms=total_ms, results=results)
526
 
@@ -533,7 +506,7 @@ async def batch_files(
533
  )
534
  async def batch_urls(body: BatchUrlRequest):
535
  batch_start = time.perf_counter()
536
- logger.info("batch_urls | count=%d", len(body.urls))
537
 
538
  async def _process_url(url: str) -> BatchFileResult:
539
  loop = asyncio.get_running_loop()
@@ -543,7 +516,7 @@ async def batch_urls(body: BatchUrlRequest):
543
  results = await asyncio.gather(*[_process_url(url) for url in body.urls])
544
  total_ms = round((time.perf_counter() - batch_start) * 1000, 3)
545
  succeeded = sum(1 for r in results if r.success)
546
- logger.info("batch_urls | done | succeeded=%d | failed=%d | total_ms=%.1f", succeeded, len(results) - succeeded, total_ms)
547
 
548
  return BatchResponse(total=len(results), succeeded=succeeded, failed=len(results) - succeeded, total_time_ms=total_ms, results=results)
549
 
 
20
  GET /api/v1/files/{id}/{fn}
21
  GET /api/v1/health
22
  GET /api/v1/ready
 
23
 
24
  System:
25
  GET / Root info
 
33
 
34
  import asyncio
35
  import concurrent.futures
36
+ import json
37
  import os
 
38
  import time
39
  import uuid
 
40
  from contextlib import asynccontextmanager
41
  from datetime import datetime, timezone
42
  from pathlib import Path
 
44
  from urllib.parse import urlparse
45
 
46
  import httpx
47
+ import structlog
48
  from fastapi import APIRouter, Depends, FastAPI, File, Form, HTTPException, Request, UploadFile, status
49
  from fastapi.responses import JSONResponse, PlainTextResponse
50
  from fastapi.middleware.cors import CORSMiddleware
 
61
  from app.core.exceptions import AppException
62
  from app.core.logging import configure_logging
63
  from app.core.rate_limit import RateLimitMiddleware
64
+ from app.services.ping import start_self_ping
65
+ from app.utils.cleanup import cleanup_loop
66
 
67
  configure_logging()
68
  logger = get_logger(__name__)
 
70
  _START_TIME = time.time()
71
  APP_NAME = "reconciliation-file-processing-service"
72
 
73
+ MAX_UPLOAD_BYTES = int(os.getenv("MAX_UPLOAD_BYTES", str(100 * 1024 * 1024)))
74
 
75
  MAX_WORKERS = min(32, (os.cpu_count() or 1) + 4)
76
  _thread_pool = concurrent.futures.ThreadPoolExecutor(max_workers=MAX_WORKERS)
77
 
78
  _converter = DocumentConverter()
79
 
80
+ logger.info("thread_pool_initialised", workers=MAX_WORKERS)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
81
 
82
 
83
  # ---------------------------------------------------------------------------
 
86
 
87
  @asynccontextmanager
88
  async def lifespan(app: FastAPI):
89
+ logger.info("api_starting", version="2.2.0", host="0.0.0.0:7860")
90
+ ping_task = await start_self_ping()
91
+ cleanup_task = asyncio.create_task(cleanup_loop())
 
92
  yield
93
+ if ping_task:
94
+ ping_task.cancel()
95
+ cleanup_task.cancel()
96
+ logger.info("api_shutting_down")
97
 
98
 
99
  # ---------------------------------------------------------------------------
 
106
  "Unified API for document-to-Markdown conversion (Microsoft MarkItDown + RapidOCR) "
107
  "and PDF-to-image conversion with async job support."
108
  ),
109
+ version="2.2.0",
110
  docs_url="/docs",
111
  redoc_url="/redoc",
112
  lifespan=lifespan,
 
128
  request_id = str(uuid.uuid4())
129
  start = time.perf_counter()
130
 
131
+ structlog.contextvars.clear_contextvars()
132
+ structlog.contextvars.bind_contextvars(
 
133
  request_id=request_id,
134
  method=request.method,
135
  path=request.url.path,
 
138
  response = await call_next(request)
139
  elapsed = round((time.perf_counter() - start) * 1000, 2)
140
 
141
+ logger.info("request_completed", status=response.status_code, duration_ms=elapsed)
142
  response.headers["X-Request-ID"] = request_id
143
  response.headers["X-Response-Time-Ms"] = str(elapsed)
144
  return response
 
147
  # -- Exception handlers --
148
  @app.exception_handler(AppException)
149
  async def app_exception_handler(request: Request, exc: AppException):
150
+ logger.warning("app_exception", detail=exc.detail, status_code=exc.status_code)
151
  return JSONResponse(
152
  status_code=exc.status_code,
153
  content={"error": exc.detail, "request_id": getattr(request.state, "request_id", "")},
 
163
  )
164
 
165
 
166
+ # -- Root-level routes (only / , /ping) --
167
  @app.get("/", tags=["System"], summary="Root", include_in_schema=False)
168
  async def root():
169
+ return {"service": APP_NAME, "version": "2.2.0", "status": "running"}
170
 
171
 
172
  @app.get("/ping", tags=["System"], summary="Ping", include_in_schema=False)
 
400
  except _json.JSONDecodeError:
401
  raise HTTPException(status_code=400, detail={"success": False, "message": "Invalid JSON in mappings parameter."})
402
 
403
+ logger.info("convert_file", filename=file.filename)
404
  raw = await file.read()
405
  if len(raw) > MAX_UPLOAD_BYTES:
406
+ logger.warning("convert_file | file too large", filename=file.filename, size=len(raw))
407
  raise HTTPException(status_code=413, detail={"success": False, "message": "File exceeds 100 MB limit."})
408
 
409
  loop = asyncio.get_running_loop()
410
  outcome = await loop.run_in_executor(_thread_pool, _converter.convert_stream, raw, file.filename or "upload")
411
 
412
  if isinstance(outcome, ConversionError):
413
+ logger.error("convert_file | conversion failed", filename=file.filename, error=outcome.message)
414
  _raise_for_error(outcome)
415
 
416
+ logger.info("convert_file | success", filename=file.filename, chars=outcome.char_count, time_ms=round(outcome.duration_ms, 1))
417
 
418
  if plain_text:
419
  return PlainTextResponse(outcome.markdown)
 
428
  summary="Convert a public URL to Markdown",
429
  )
430
  async def convert_url(body: UrlRequest):
431
+ logger.info("convert_url", url=body.url)
432
  parsed = urlparse(body.url)
433
  filename = Path(parsed.path).name or "url_content"
434
  loop = asyncio.get_running_loop()
 
439
  resp = await client.get(body.url)
440
  resp.raise_for_status()
441
  except httpx.HTTPError as exc:
442
+ logger.error("convert_url | fetch failed", url=body.url, error=str(exc))
443
  raise HTTPException(status_code=400, detail={"success": False, "message": f"Failed to fetch URL: {exc}"})
444
 
445
  raw_data = resp.content
 
448
 
449
  outcome = await loop.run_in_executor(_thread_pool, _converter.convert_stream, raw_data, filename)
450
  if isinstance(outcome, ConversionError):
451
+ logger.error("convert_url | conversion failed", url=body.url, error=outcome.message)
452
  _raise_for_error(outcome)
453
 
454
+ logger.info("convert_url | success", url=body.url, chars=outcome.char_count, time_ms=round(outcome.duration_ms, 1))
455
  return await _build_response(outcome, return_json=body.return_json, filename=filename, raw_data=raw_data, mappings=body.mappings)
456
 
457
  outcome = await loop.run_in_executor(_thread_pool, _converter.convert_url, body.url)
458
  if isinstance(outcome, ConversionError):
459
+ logger.error("convert_url | conversion failed", url=body.url, error=outcome.message)
460
  _raise_for_error(outcome)
461
 
462
+ logger.info("convert_url | success", url=body.url, chars=outcome.char_count, time_ms=round(outcome.duration_ms, 1))
463
  return await _build_response(outcome, return_json=body.return_json, filename=filename, mappings=body.mappings)
464
 
465
 
 
478
  raise HTTPException(status_code=400, detail={"success": False, "message": "Maximum 10 files per batch."})
479
 
480
  batch_start = time.perf_counter()
481
+ logger.info("batch_files", count=len(files))
482
 
483
  async def _process_file(f: UploadFile) -> BatchFileResult:
484
  if f is None:
 
493
  results = await asyncio.gather(*[_process_file(f) for f in files])
494
  total_ms = round((time.perf_counter() - batch_start) * 1000, 3)
495
  succeeded = sum(1 for r in results if r.success)
496
+ logger.info("batch_files | done", succeeded=succeeded, failed=len(results) - succeeded, total_ms=round(total_ms, 1))
497
 
498
  return BatchResponse(total=len(results), succeeded=succeeded, failed=len(results) - succeeded, total_time_ms=total_ms, results=results)
499
 
 
506
  )
507
  async def batch_urls(body: BatchUrlRequest):
508
  batch_start = time.perf_counter()
509
+ logger.info("batch_urls", count=len(body.urls))
510
 
511
  async def _process_url(url: str) -> BatchFileResult:
512
  loop = asyncio.get_running_loop()
 
516
  results = await asyncio.gather(*[_process_url(url) for url in body.urls])
517
  total_ms = round((time.perf_counter() - batch_start) * 1000, 3)
518
  succeeded = sum(1 for r in results if r.success)
519
+ logger.info("batch_urls | done", succeeded=succeeded, failed=len(results) - succeeded, total_ms=round(total_ms, 1))
520
 
521
  return BatchResponse(total=len(results), succeeded=succeeded, failed=len(results) - succeeded, total_time_ms=total_ms, results=results)
522
 
app/banner.py DELETED
@@ -1,21 +0,0 @@
1
- import pyfiglet
2
- from rich.console import Console
3
- from rich.rule import Rule
4
- from rich.text import Text
5
-
6
- from app.core.config import get_settings
7
-
8
- _console = Console()
9
-
10
-
11
- def print_banner() -> None:
12
- settings = get_settings()
13
-
14
- art = pyfiglet.figlet_format("ValidOps", font="slant")
15
-
16
- _console.print(Text(art, style="bold cyan"))
17
- _console.print(f" [bold white]{'Service:':<14}[/bold white] [cyan]{settings.APP_NAME}[/cyan]")
18
- _console.print(f" [bold white]{'Version:':<14}[/bold white] [cyan]v{settings.VERSION}[/cyan]")
19
- _console.print(f" [bold white]{'Environment:':<14}[/bold white] [cyan]{settings.ENV}[/cyan]")
20
- _console.print(Rule(style="dim cyan"))
21
- _console.print()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
app/core/exceptions.py CHANGED
@@ -54,8 +54,3 @@ class InvalidParameterError(AppException):
54
  class FileServiceError(AppException):
55
  def __init__(self, detail: str = "File upload service error"):
56
  super().__init__(detail, status.HTTP_502_BAD_GATEWAY)
57
-
58
-
59
- class CSVGenerationError(AppException):
60
- def __init__(self, detail: str = "CSV generation failed"):
61
- super().__init__(detail, status.HTTP_500_INTERNAL_SERVER_ERROR)
 
54
  class FileServiceError(AppException):
55
  def __init__(self, detail: str = "File upload service error"):
56
  super().__init__(detail, status.HTTP_502_BAD_GATEWAY)
 
 
 
 
 
app/main.py DELETED
@@ -1,95 +0,0 @@
1
- from __future__ import annotations
2
-
3
- import time
4
- import uuid
5
- from contextlib import asynccontextmanager
6
-
7
- import structlog
8
- from fastapi import FastAPI, Request
9
- from fastapi.middleware.cors import CORSMiddleware
10
- from fastapi.middleware.gzip import GZipMiddleware
11
- from fastapi.responses import JSONResponse
12
- from prometheus_client import make_asgi_app
13
-
14
- from app.api.routes import router
15
- from app.core.config import settings
16
- from app.core.exceptions import AppException
17
- from app.core.logging import configure_logging
18
- from app.core.rate_limit import RateLimitMiddleware
19
- configure_logging()
20
- logger = structlog.get_logger(__name__)
21
-
22
-
23
- @asynccontextmanager
24
- async def lifespan(app: FastAPI):
25
- logger.info("startup", service=settings.APP_NAME, version=settings.VERSION, env=settings.ENV)
26
- yield
27
- logger.info("shutdown", service=settings.APP_NAME)
28
-
29
-
30
- app = FastAPI(
31
- title=settings.APP_NAME,
32
- description="High-performance PDF to Image conversion and CSV report generation API",
33
- version=settings.VERSION,
34
- docs_url="/docs",
35
- redoc_url="/redoc",
36
- lifespan=lifespan,
37
- )
38
-
39
- app.add_middleware(RateLimitMiddleware)
40
- app.add_middleware(GZipMiddleware, minimum_size=1000)
41
- app.add_middleware(
42
- CORSMiddleware,
43
- allow_origins=settings.ALLOWED_ORIGINS,
44
- allow_credentials=True,
45
- allow_methods=["*"],
46
- allow_headers=["*"],
47
- )
48
-
49
-
50
- @app.middleware("http")
51
- async def request_context_middleware(request: Request, call_next):
52
- request_id = str(uuid.uuid4())
53
- start = time.perf_counter()
54
- request.state.request_id = request_id
55
-
56
- structlog.contextvars.clear_contextvars()
57
- structlog.contextvars.bind_contextvars(
58
- request_id=request_id,
59
- method=request.method,
60
- path=request.url.path,
61
- )
62
-
63
- response = await call_next(request)
64
- elapsed = round((time.perf_counter() - start) * 1000, 2)
65
-
66
- logger.info("request_completed", status_code=response.status_code, duration_ms=elapsed)
67
- response.headers["X-Request-ID"] = request_id
68
- response.headers["X-Response-Time-Ms"] = str(elapsed)
69
- return response
70
-
71
-
72
- @app.exception_handler(AppException)
73
- async def app_exception_handler(request: Request, exc: AppException):
74
- logger.warning("app_exception", detail=exc.detail, status_code=exc.status_code)
75
- return JSONResponse(
76
- status_code=exc.status_code,
77
- content={"error": exc.detail, "request_id": request.state.request_id},
78
- )
79
-
80
-
81
- @app.exception_handler(Exception)
82
- async def generic_exception_handler(request: Request, exc: Exception):
83
- logger.exception("unhandled_exception", exc_info=exc)
84
- return JSONResponse(
85
- status_code=500,
86
- content={"error": "Internal server error", "request_id": request.state.request_id},
87
- )
88
-
89
-
90
- @app.get("/")
91
- async def root():
92
- return {"message": f"{settings.APP_NAME} v{settings.VERSION}"}
93
-
94
- app.include_router(router, prefix="/api/v1")
95
- app.mount("/metrics", make_asgi_app())
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
app/services/ping.py CHANGED
@@ -1,6 +1,7 @@
1
  from __future__ import annotations
2
 
3
  import asyncio
 
4
 
5
  import httpx
6
  import structlog
@@ -10,11 +11,13 @@ from app.core.config import settings
10
  logger = structlog.get_logger(__name__)
11
 
12
 
13
- async def start_self_ping() -> None:
14
  if not settings.SELF_PING_ENABLED or not settings.SELF_PING_URL:
15
  logger.info("self_ping_disabled")
16
- return
17
- asyncio.create_task(_ping_loop())
 
 
18
 
19
 
20
  async def _ping_loop() -> None:
 
1
  from __future__ import annotations
2
 
3
  import asyncio
4
+ from typing import Optional
5
 
6
  import httpx
7
  import structlog
 
11
  logger = structlog.get_logger(__name__)
12
 
13
 
14
+ async def start_self_ping() -> Optional[asyncio.Task]:
15
  if not settings.SELF_PING_ENABLED or not settings.SELF_PING_URL:
16
  logger.info("self_ping_disabled")
17
+ return None
18
+ task = asyncio.create_task(_ping_loop())
19
+ logger.info("self_ping_started", url=settings.SELF_PING_URL, interval=settings.SELF_PING_INTERVAL_SECONDS)
20
+ return task
21
 
22
 
23
  async def _ping_loop() -> None:
banner.py CHANGED
@@ -8,7 +8,7 @@ from rich.text import Text
8
  _console = Console()
9
 
10
  SERVICE_NAME = os.getenv("SERVICE_NAME", "reconciliation-file-processing-service")
11
- API_VERSION = os.getenv("API_VERSION", "2.1.0")
12
  ENVIRONMENT = os.getenv("ENVIRONMENT", "production")
13
 
14
 
 
8
  _console = Console()
9
 
10
  SERVICE_NAME = os.getenv("SERVICE_NAME", "reconciliation-file-processing-service")
11
+ API_VERSION = os.getenv("API_VERSION", "2.2.0")
12
  ENVIRONMENT = os.getenv("ENVIRONMENT", "production")
13
 
14
 
core/__init__.py CHANGED
@@ -1,15 +1,8 @@
1
  from .converter import ConversionError, ConversionResult, DocumentConverter, SUPPORTED_EXTENSIONS
2
- from .batch import BatchProcessor, BatchReport
3
- from .output import OutputWriter
4
- from .ocr_engine import ocr_image
5
 
6
  __all__ = [
7
  "ConversionError",
8
  "ConversionResult",
9
  "DocumentConverter",
10
  "SUPPORTED_EXTENSIONS",
11
- "BatchProcessor",
12
- "BatchReport",
13
- "OutputWriter",
14
- "ocr_image",
15
  ]
 
1
  from .converter import ConversionError, ConversionResult, DocumentConverter, SUPPORTED_EXTENSIONS
 
 
 
2
 
3
  __all__ = [
4
  "ConversionError",
5
  "ConversionResult",
6
  "DocumentConverter",
7
  "SUPPORTED_EXTENSIONS",
 
 
 
 
8
  ]
core/converter.py CHANGED
@@ -37,6 +37,13 @@ from logger import get_logger
37
 
38
  logger = get_logger(__name__)
39
 
 
 
 
 
 
 
 
40
 
41
  # ---------------------------------------------------------------------------
42
  # Supported formats
@@ -51,14 +58,6 @@ SUPPORTED_EXTENSIONS = {
51
  ".zip", ".epub",
52
  }
53
 
54
- # Extensions that route through RapidOCR rather than MarkItDown.
55
- IMAGE_EXTENSIONS = {
56
- ".jpg", ".jpeg", ".png", ".gif", ".bmp", ".webp", ".tiff",
57
- }
58
-
59
- IMAGE_MIME_PREFIXES = {"image/"}
60
-
61
-
62
  def _is_image(ext: str, mime: str) -> bool:
63
  """Return True when the input should be routed through RapidOCR."""
64
  return ext.lower() in IMAGE_EXTENSIONS or any(
 
37
 
38
  logger = get_logger(__name__)
39
 
40
+ # Extensions that route through RapidOCR rather than MarkItDown.
41
+ IMAGE_EXTENSIONS = {
42
+ ".jpg", ".jpeg", ".png", ".gif", ".bmp", ".webp", ".tiff",
43
+ }
44
+
45
+ IMAGE_MIME_PREFIXES = {"image/"}
46
+
47
 
48
  # ---------------------------------------------------------------------------
49
  # Supported formats
 
58
  ".zip", ".epub",
59
  }
60
 
 
 
 
 
 
 
 
 
61
  def _is_image(ext: str, mime: str) -> bool:
62
  """Return True when the input should be routed through RapidOCR."""
63
  return ext.lower() in IMAGE_EXTENSIONS or any(
extraction/json_extractor.py CHANGED
@@ -20,10 +20,11 @@ logger = get_logger(__name__)
20
  SUPPORTED_EXTENSIONS = {'.csv', '.xls', '.xlsx'}
21
 
22
  # Resource limits to prevent abuse
23
- MAX_FILE_SIZE_BYTES = 100 * 1024 * 1024 # 100 MB
24
- MAX_CSV_ROWS = 100000 # Limit rows for CSV
25
- MAX_EXCEL_ROWS = 50000 # Limit rows for Excel
26
- MAX_MEMORY_ROWS = 100000 # Global memory guard
 
27
 
28
 
29
  def _validate_file_size(size: int) -> Optional[str]:
 
20
  SUPPORTED_EXTENSIONS = {'.csv', '.xls', '.xlsx'}
21
 
22
  # Resource limits to prevent abuse
23
+ from app.core.config import settings as _app_settings
24
+ MAX_FILE_SIZE_BYTES = _app_settings.MAX_FILE_SIZE_BYTES
25
+ MAX_CSV_ROWS = 100000
26
+ MAX_EXCEL_ROWS = 50000
27
+ MAX_MEMORY_ROWS = 100000
28
 
29
 
30
  def _validate_file_size(size: int) -> Optional[str]:
extraction/spacy_extractor.py CHANGED
@@ -66,10 +66,6 @@ def _get_nlp():
66
  return _nlp
67
 
68
 
69
- def clean_text(text: str) -> str:
70
- return text
71
-
72
-
73
  # ---------------------------------------------------------------------------
74
  # Normalizer registry
75
  # ---------------------------------------------------------------------------
@@ -600,7 +596,7 @@ def extract_fields(
600
  Backward-compatible: callers that pass flat scalar rules unchanged still work.
601
  """
602
  nlp = _get_nlp()
603
- cleaned = clean_text(text)
604
 
605
  try:
606
  doc = next(iter(nlp.pipe([cleaned])))
@@ -642,7 +638,7 @@ def extract_schema(
642
  # → [{"date": "Jan 2024", "amount": "$1,200"}, ...]
643
  """
644
  nlp = _get_nlp()
645
- cleaned = clean_text(text)
646
 
647
  try:
648
  doc = next(iter(nlp.pipe([cleaned])))
 
66
  return _nlp
67
 
68
 
 
 
 
 
69
  # ---------------------------------------------------------------------------
70
  # Normalizer registry
71
  # ---------------------------------------------------------------------------
 
596
  Backward-compatible: callers that pass flat scalar rules unchanged still work.
597
  """
598
  nlp = _get_nlp()
599
+ cleaned = text
600
 
601
  try:
602
  doc = next(iter(nlp.pipe([cleaned])))
 
638
  # → [{"date": "Jan 2024", "amount": "$1,200"}, ...]
639
  """
640
  nlp = _get_nlp()
641
+ cleaned = text
642
 
643
  try:
644
  doc = next(iter(nlp.pipe([cleaned])))
logger.py CHANGED
@@ -1,87 +1,20 @@
1
- """Production-grade structured JSON logger.
2
 
3
- Emits newline-delimited JSON to stdout (12-factor app pattern) for
4
- seamless ingestion by log aggregators (Datadog, ELK, CloudWatch, etc.).
5
 
6
  Usage
7
  -----
8
  from logger import get_logger
9
 
10
  logger = get_logger(__name__)
11
- logger.info("processed | file=%s | rows=%d", filename, n)
12
-
13
- Configuration (environment variables):
14
-
15
- LOG_LEVEL Minimum level (default: INFO)
16
  """
17
 
18
  from __future__ import annotations
19
 
20
- import json
21
- import logging
22
- import os
23
- import sys
24
- from datetime import datetime, timezone
25
- from typing import Any
26
-
27
-
28
- __all__ = ["get_logger", "app_logger"]
29
-
30
-
31
- class JSONFormatter(logging.Formatter):
32
- """Render log records as JSON lines.
33
-
34
- Each line contains ``timestamp`` (ISO 8601 UTC), ``level``,
35
- ``logger``, and ``message``. Exception tracebacks are included
36
- under ``exception`` when ``exc_info`` is set on the record.
37
- """
38
-
39
- __slots__ = ()
40
-
41
- def format(self, record: logging.LogRecord) -> str:
42
- payload: dict[str, Any] = {
43
- "timestamp": datetime.fromtimestamp(
44
- record.created, tz=timezone.utc
45
- ).isoformat(),
46
- "level": record.levelname,
47
- "logger": record.name,
48
- "message": record.getMessage(),
49
- }
50
- if record.exc_info and record.exc_info[0] is not None:
51
- payload["exception"] = self.formatException(record.exc_info)
52
- return json.dumps(payload, default=str, ensure_ascii=False, sort_keys=True)
53
-
54
-
55
- def get_logger(name: str) -> logging.Logger:
56
- """Return a named logger.
57
-
58
- All loggers inherit handlers from the root logger, which is
59
- configured once at import time with a :class:`JSONFormatter`.
60
- """
61
- return logging.getLogger(name)
62
-
63
-
64
- _initialised: bool = False
65
-
66
-
67
- def _setup() -> None:
68
- """Configure the root logger once from environment defaults."""
69
- global _initialised
70
- if _initialised:
71
- return
72
- _initialised = True
73
-
74
- level = os.getenv("LOG_LEVEL", "INFO").upper()
75
-
76
- root = logging.getLogger()
77
- root.setLevel(getattr(logging, level, logging.INFO))
78
-
79
- fmt = JSONFormatter()
80
-
81
- console = logging.StreamHandler(sys.stdout)
82
- console.setFormatter(fmt)
83
- root.addHandler(console)
84
 
 
85
 
86
- _setup()
87
- app_logger = get_logger("docx")
 
1
+ """Centralised logging re-exports structlog.getLogger.
2
 
3
+ All application code should use this module to obtain loggers,
4
+ ensuring consistent structlog configuration throughout the codebase.
5
 
6
  Usage
7
  -----
8
  from logger import get_logger
9
 
10
  logger = get_logger(__name__)
11
+ logger.info("event", key="value", duration_ms=42.0)
 
 
 
 
12
  """
13
 
14
  from __future__ import annotations
15
 
16
+ import structlog
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
17
 
18
+ __all__ = ["get_logger"]
19
 
20
+ get_logger = structlog.get_logger
 
self_ping.py DELETED
@@ -1,19 +0,0 @@
1
- #!/usr/bin/env python3
2
- """Self-ping script that hits a URL."""
3
-
4
- import datetime
5
- import os
6
- import urllib.request
7
-
8
- PING_URL = os.environ.get("PING_URL", "https://your-url.com/health")
9
-
10
- def main():
11
- timestamp = datetime.datetime.now().isoformat()
12
- try:
13
- urllib.request.urlopen(PING_URL, timeout=10)
14
- print(f"[{timestamp}] Pinged {PING_URL} - OK")
15
- except Exception as e:
16
- print(f"[{timestamp}] Ping failed: {e}")
17
-
18
- if __name__ == "__main__":
19
- main()