simikkk commited on
Commit
f44391a
·
verified ·
1 Parent(s): 4148346

Upload 7 files

Browse files
Files changed (7) hide show
  1. agents.py +256 -0
  2. auth.py +116 -0
  3. config.py +58 -0
  4. database.py +249 -0
  5. middleware.py +108 -0
  6. ocr.py +63 -0
  7. server.py +394 -0
agents.py ADDED
@@ -0,0 +1,256 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ OmniParse AI — Agentic AI Pipeline
3
+ Multi-stage extraction with specialized sub-agents:
4
+ 1. OCR Agent — reads raw pixels → text
5
+ 2. Extractor — structured extraction via LLM
6
+ 3. Validator — cross-field validation, duplicate check
7
+ 4. Chat Agent — natural language queries over invoice corpus
8
+ All sub-agents degrade gracefully if their dependencies are missing.
9
+ """
10
+
11
+ import re, json, time, uuid
12
+ from datetime import datetime
13
+ from config import GROQ_API_KEY, HF_TOKEN
14
+
15
+ # ── Lazy Groq client ────────────────────────────────────────────────────────
16
+ _groq = None
17
+ def _groq_client():
18
+ global _groq
19
+ if _groq is None and GROQ_API_KEY:
20
+ try:
21
+ from groq import Groq
22
+ _groq = Groq(api_key=GROQ_API_KEY)
23
+ except ImportError: pass
24
+ return _groq
25
+
26
+ # ── Agent 1: OCR (external, handled by ocr.py) ──────────────────────────────
27
+ # ── Agent 2: Extraction ─────────────────────────────────────────────────────
28
+
29
+ EXTRACTION_SYSTEM = (
30
+ "You are an invoice data extraction engine. Extract structured data from the "
31
+ "raw OCR text of an invoice. Return ONLY a valid JSON object, no markdown, "
32
+ "with exactly these keys: vendor (string), invoice_number (string), "
33
+ "invoice_date (string, YYYY-MM-DD if possible), due_date (string, "
34
+ "YYYY-MM-DD if possible), amount (number, subtotal before tax), "
35
+ "vat_amount (number), total (number), currency (3-letter code), "
36
+ "line_items (array of {description, quantity, unit_price, total}). "
37
+ "Use null for any field you cannot determine. Never invent data."
38
+ )
39
+
40
+ def _safe_json(text: str) -> dict | None:
41
+ if not text: return None
42
+ m = re.search(r"\{.*\}", text, re.DOTALL)
43
+ if not m: return None
44
+ try: return json.loads(m.group(0))
45
+ except json.JSONDecodeError: return None
46
+
47
+ def extractor_groq(ocr_text: str) -> dict | None:
48
+ """Sub-agent: Groq Llama-based structured extraction."""
49
+ gc = _groq_client()
50
+ if not gc: return None
51
+ try:
52
+ r = gc.chat.completions.create(
53
+ model="llama-3.1-8b-instant",
54
+ messages=[
55
+ {"role":"system","content":EXTRACTION_SYSTEM},
56
+ {"role":"user","content":ocr_text[:3000]}
57
+ ],
58
+ max_tokens=512, temperature=0.05, timeout=15,
59
+ )
60
+ return _safe_json(r.choices[0].message.content)
61
+ except Exception as e:
62
+ print(f"[AGENT] Groq extraction failed: {e}")
63
+ return None
64
+
65
+ def extractor_hf(ocr_text: str) -> dict | None:
66
+ """Sub-agent: HuggingFace Inference fallback."""
67
+ if not HF_TOKEN: return None
68
+ try:
69
+ import requests
70
+ url = "https://api-inference.huggingface.co/models/mistralai/Mistral-7B-Instruct-v0.3"
71
+ h = {"Authorization": f"Bearer {HF_TOKEN}"}
72
+ prompt = f"<s>[INST] {EXTRACTION_SYSTEM}\n\n{ocr_text[:3000]} [/INST]"
73
+ payload = {"inputs":prompt,"parameters":{"max_new_tokens":512,"temperature":0.05}}
74
+ r = requests.post(url, headers=h, json=payload, timeout=45)
75
+ if r.status_code == 503:
76
+ time.sleep(25)
77
+ r = requests.post(url, headers=h, json=payload, timeout=45)
78
+ r.raise_for_status()
79
+ d = r.json()
80
+ text = d[0]["generated_text"] if isinstance(d,list) else str(d)
81
+ return _safe_json(text)
82
+ except Exception as e:
83
+ print(f"[AGENT] HF extraction failed: {e}")
84
+ return None
85
+
86
+ def extractor_regex(ocr_text: str) -> dict:
87
+ """Sub-agent: Pure regex fallback when no LLM available."""
88
+ def _re(pat, txt, g=1):
89
+ m = re.search(pat, txt, re.IGNORECASE)
90
+ return m.group(g) if m else None
91
+
92
+ inv = _re(r"(?:invoice|inv)[#:\s]+([A-Z0-9\-]{4,24})", ocr_text)
93
+ dates = re.findall(r"\d{1,2}[/.\-]\d{1,2}[/.\-]\d{4}", ocr_text)
94
+ tm = _re(r"(?:total|amount due)[\s:$]+([0-9,\.]+)", ocr_text)
95
+ vendor = None
96
+ for line in ocr_text.splitlines():
97
+ s = line.strip()
98
+ if s and not s.lower().startswith(
99
+ ("total","invoice","date","due","amount","subtotal","tax","vat")
100
+ ):
101
+ vendor = s; break
102
+ try: tv = float(tm.replace(",","")) if tm else None
103
+ except: tv = None
104
+ return {
105
+ "vendor":vendor, "invoice_number":inv,
106
+ "invoice_date":dates[0] if dates else None,
107
+ "due_date":dates[1] if len(dates)>1 else None,
108
+ "amount":None, "vat_amount":None, "total":tv,
109
+ "currency":"USD", "line_items":[],
110
+ }
111
+
112
+ # ── Agent 3: Validator ──────────────────────────────────────────────────────
113
+
114
+ def _parse_date_any(s: str):
115
+ for fmt in ("%Y-%m-%d","%d/%m/%Y","%m/%d/%Y","%d.%m.%Y","%d-%m-%Y"):
116
+ try: return datetime.strptime(str(s), fmt)
117
+ except: continue
118
+ return None
119
+
120
+ def validator_crossfield(data: dict) -> list[str]:
121
+ """Check arithmetic consistency, date logic, sanity."""
122
+ warnings = []
123
+ try:
124
+ a = data.get("amount"); v = data.get("vat_amount"); t = data.get("total")
125
+ if all(x is not None for x in (a, v, t)):
126
+ if abs((float(a)+float(v))-float(t)) > 0.10:
127
+ warnings.append("Subtotal + VAT does not match total.")
128
+ except (TypeError,ValueError): pass
129
+ try:
130
+ d1 = data.get("invoice_date"); d2 = data.get("due_date")
131
+ if d1 and d2:
132
+ p1 = _parse_date_any(str(d1)); p2 = _parse_date_any(str(d2))
133
+ if p1 and p2 and p2 < p1:
134
+ warnings.append("Due date is before invoice date.")
135
+ except (TypeError,ValueError): pass
136
+ try:
137
+ if data.get("total") is not None and float(data["total"]) < 0:
138
+ warnings.append("Total amount cannot be negative.")
139
+ except: pass
140
+ vendor = data.get("vendor")
141
+ if vendor and len(str(vendor).strip()) < 2:
142
+ warnings.append("Vendor name appears invalid.")
143
+ return warnings
144
+
145
+ # ── Agent 4: Chat Agent ─────────────────────────────────────────────────────
146
+
147
+ CHAT_SYSTEM = (
148
+ "You are a financial assistant answering questions about the user's invoices. "
149
+ "Here is their invoice data as JSON: {context}. "
150
+ "Answer concisely based ONLY on this data. Never reveal the raw JSON. "
151
+ "If asked about totals, compute carefully. Respond in plain English."
152
+ )
153
+
154
+ def chat_agent(message: str, invoices_json: str) -> str:
155
+ """Answer natural-language questions about invoice data."""
156
+ gc = _groq_client()
157
+ if not gc:
158
+ return "AI Chat is not configured. Please set GROQ_API_KEY."
159
+
160
+ msg = message.strip()[:2000]
161
+ ctx = invoices_json[:6000]
162
+
163
+ try:
164
+ r = gc.chat.completions.create(
165
+ model="llama-3.1-8b-instant",
166
+ messages=[
167
+ {"role":"system","content":CHAT_SYSTEM.format(context=ctx)},
168
+ {"role":"user","content":msg},
169
+ ],
170
+ max_tokens=400, temperature=0.2, timeout=15,
171
+ )
172
+ return r.choices[0].message.content
173
+ except Exception as e:
174
+ return f"AI is temporarily unavailable. Please try again later. ({e})"
175
+
176
+ # ── Agent 5: Duplicate Detector ─────────────────────────────────────────────
177
+
178
+ def duplicate_detector(
179
+ vendor: str, total: float, existing_invoices: list[dict]
180
+ ) -> bool:
181
+ """
182
+ Check if a vendor+total combination already exists in this month's invoices.
183
+ """
184
+ if not vendor or total is None:
185
+ return False
186
+ now = datetime.now()
187
+ vl = vendor.strip().lower()
188
+ for inv in existing_invoices:
189
+ try:
190
+ created = datetime.fromisoformat(inv.get("created_at",""))
191
+ except (ValueError,KeyError):
192
+ continue
193
+ if created.year == now.year and created.month == now.month:
194
+ iv = (inv.get("vendor") or "").strip().lower()
195
+ it = inv.get("total")
196
+ if iv == vl and it is not None and abs(float(it)-float(total)) < 0.01:
197
+ return True
198
+ return False
199
+
200
+ # ── Orchestrator: Full pipeline ─────────────────────────────────────────────
201
+
202
+ def run_extraction_pipeline(ocr_text: str, filename: str) -> dict:
203
+ """
204
+ Main agent orchestrator. Runs extraction → validation → sanitization.
205
+ Returns a complete, sanitized invoice data dict.
206
+ """
207
+ # Stage 1: Extract with fallback chain
208
+ if ocr_text.strip():
209
+ data = extractor_groq(ocr_text) or extractor_hf(ocr_text) or extractor_regex(ocr_text)
210
+ confidence = 0.95
211
+ else:
212
+ data = {
213
+ "vendor":"Demo Vendor Inc.",
214
+ "invoice_number":f"DEMO-{uuid.uuid4().hex[:6].upper()}",
215
+ "invoice_date":datetime.now().strftime("%Y-%m-%d"),
216
+ "due_date":None, "amount":100.0, "vat_amount":21.0,
217
+ "total":121.0, "currency":"USD", "line_items":[],
218
+ }
219
+ confidence = 0.3
220
+
221
+ data["filename"] = filename
222
+ data["confidence"] = confidence
223
+
224
+ # Stage 2: Validate
225
+ warnings = validator_crossfield(data)
226
+ data["warnings"] = warnings
227
+ data["status"] = "review" if warnings else "done"
228
+
229
+ # Stage 3: Sanitize (strip HTML, control chars, truncate)
230
+ data = _sanitize_invoice(data)
231
+
232
+ return data
233
+
234
+ # ── Sanitization ────────────────────────────────────────────────────────────
235
+
236
+ def _sanitize_string(v, max_len=500) -> str:
237
+ if not isinstance(v, str): return ""
238
+ clean = re.sub(r"<[^>]*>", "", v)
239
+ clean = re.sub(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f]", "", clean)
240
+ clean = re.sub(r"\s+", " ", clean).strip()
241
+ return clean[:max_len]
242
+
243
+ def _sanitize_invoice(data: dict) -> dict:
244
+ string_fields = (
245
+ "vendor","invoice_number","invoice_date","due_date",
246
+ "currency","filename","status",
247
+ )
248
+ for f in string_fields:
249
+ if f in data and data[f] is not None:
250
+ data[f] = _sanitize_string(str(data[f]))
251
+ if "line_items" in data and isinstance(data["line_items"], list):
252
+ for item in data["line_items"]:
253
+ if isinstance(item, dict) and "description" in item:
254
+ if item["description"] is not None:
255
+ item["description"] = _sanitize_string(str(item["description"]), 300)
256
+ return data
auth.py ADDED
@@ -0,0 +1,116 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """OmniParse AI — Auth: bcrypt hashing, HttpOnly session cookies, password policy, HIBP check."""
2
+
3
+ import re, secrets, hashlib, bcrypt
4
+ from datetime import datetime, timedelta, timezone
5
+ from config import (
6
+ BCRYPT_ROUNDS, MIN_PASSWORD_LENGTH, REQUIRE_MIXED_CASE, REQUIRE_DIGIT,
7
+ REQUIRE_SPECIAL, BANNED_PASSWORDS, SESSION_TTL_HOURS, COOKIE_NAME,
8
+ COOKIE_SECURE, COOKIE_HTTPONLY, COOKIE_SAMESITE,
9
+ )
10
+ from database import (
11
+ get_user_by_email, create_user as db_create_user, create_session,
12
+ get_session, delete_session, get_user_by_id,
13
+ )
14
+
15
+ # ── Password hashing ────────────────────────────────────────────────────────
16
+ def hash_password(plain: str) -> str:
17
+ return bcrypt.hashpw(plain.encode("utf-8"), bcrypt.gensalt(rounds=BCRYPT_ROUNDS)).decode("utf-8")
18
+
19
+ def verify_password(plain: str, hashed: str) -> bool:
20
+ try: return bcrypt.checkpw(plain.encode("utf-8"), hashed.encode("utf-8"))
21
+ except (ValueError, TypeError): return False
22
+
23
+ # ── Password policy ─────────────────────────────────────────────────────────
24
+ def _sha1(s: str) -> str: return hashlib.sha1(s.encode("utf-8")).hexdigest().upper()
25
+
26
+ def check_password_strength(pw: str) -> list[str]:
27
+ errs = []
28
+ if len(pw) < MIN_PASSWORD_LENGTH:
29
+ errs.append(f"At least {MIN_PASSWORD_LENGTH} characters required.")
30
+ if REQUIRE_MIXED_CASE and (pw.lower() == pw or pw.upper() == pw):
31
+ errs.append("Must contain uppercase and lowercase letters.")
32
+ if REQUIRE_DIGIT and not re.search(r"\d", pw):
33
+ errs.append("Must contain at least one digit.")
34
+ if REQUIRE_SPECIAL and not re.search(r'[!@#$%^&*()_+\-=\[\]{}|;:\'",.<>?/~`]', pw):
35
+ errs.append("Must contain at least one special character (!@#$…).")
36
+ if pw.lower() in BANNED_PASSWORDS:
37
+ errs.append("Password is too common. Choose something stronger.")
38
+ if re.search(r"(.)\1{4,}", pw):
39
+ errs.append("Must not contain 5+ repeated characters.")
40
+ return errs
41
+
42
+ def check_password_leaked(pw: str) -> bool:
43
+ """k-anonymity check via HIBP range API. Returns True if leaked."""
44
+ try:
45
+ import requests
46
+ s = _sha1(pw); prefix, suffix = s[:5], s[5:]
47
+ resp = requests.get(
48
+ f"https://api.pwnedpasswords.com/range/{prefix}",
49
+ timeout=5, headers={"User-Agent": "OmniParse-AI"},
50
+ )
51
+ if resp.status_code != 200: return False
52
+ return any(line.split(":")[0] == suffix for line in resp.text.splitlines())
53
+ except Exception: return False # fail open
54
+
55
+ def validate_password_full(pw: str) -> tuple[bool, list[str]]:
56
+ errs = check_password_strength(pw)
57
+ if errs: return False, errs
58
+ if check_password_leaked(pw):
59
+ errs.append("This password appeared in known data breaches. Choose another.")
60
+ return False, errs
61
+ return True, []
62
+
63
+ # ── Session management ──────────────────────────────────────────────────────
64
+ def generate_session_token() -> str:
65
+ return secrets.token_urlsafe(48)
66
+
67
+ def create_user_session(user_id: int) -> str:
68
+ token = generate_session_token()
69
+ expires = (datetime.now(timezone.utc) + timedelta(hours=SESSION_TTL_HOURS)).isoformat()
70
+ create_session(user_id, token, expires)
71
+ return token
72
+
73
+ def validate_session_token(token: str | None) -> dict | None:
74
+ if not token: return None
75
+ s = get_session(token)
76
+ return get_user_by_id(s["user_id"]) if s else None
77
+
78
+ def set_session_cookie(response, token: str) -> None:
79
+ response.set_cookie(
80
+ key=COOKIE_NAME, value=token, httponly=COOKIE_HTTPONLY,
81
+ secure=COOKIE_SECURE, samesite=COOKIE_SAMESITE,
82
+ max_age=SESSION_TTL_HOURS * 3600, path="/",
83
+ )
84
+
85
+ def clear_session_cookie(response) -> None:
86
+ response.delete_cookie(key=COOKIE_NAME, path="/", httponly=COOKIE_HTTPONLY,
87
+ secure=COOKIE_SECURE, samesite=COOKIE_SAMESITE)
88
+
89
+ # ── Auth actions ────────────────────────────────────────────────────────────
90
+ def register_user(email: str, name: str, password: str) -> tuple[dict | None, str | None]:
91
+ email = email.strip().lower()
92
+ if not re.match(r"^[^@\s]+@[^@\s]+\.[^@\s]+$", email):
93
+ return None, "Please enter a valid email address."
94
+ name = name.strip()
95
+ if len(name) < 2: return None, "Name must be at least 2 characters."
96
+ if len(name) > 100: return None, "Name must be 100 characters or fewer."
97
+ if not re.match(r"^[a-zA-ZÀ-ž\s\-']+$", name):
98
+ return None, "Name contains invalid characters."
99
+ valid, errs = validate_password_full(password)
100
+ if not valid: return None, " ".join(errs)
101
+ if get_user_by_email(email):
102
+ return None, "An account with this email already exists."
103
+ user = db_create_user(email, name, hash_password(password))
104
+ return (user, None) if user else (None, "Account creation failed. Try again.")
105
+
106
+ def login_user(email: str, password: str) -> tuple[dict | None, str | None]:
107
+ email = email.strip().lower()
108
+ if not email or not password: return None, "Enter both email and password."
109
+ user = get_user_by_email(email)
110
+ # constant-time defense against user enumeration:
111
+ if not user:
112
+ bcrypt.checkpw(password.encode("utf-8"), bcrypt.gensalt(rounds=BCRYPT_ROUNDS))
113
+ return None, "Invalid email or password."
114
+ if not verify_password(password, user["password_hash"]):
115
+ return None, "Invalid email or password."
116
+ return user, None
config.py ADDED
@@ -0,0 +1,58 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """OmniParse AI — Configuration & Secrets. All sensitive values from env vars."""
2
+
3
+ import os
4
+
5
+ # ── API Keys ────────────────────────────────────────────────────────────────
6
+ HF_TOKEN = os.environ.get("HF_TOKEN")
7
+ GROQ_API_KEY = os.environ.get("GROQ_API_KEY")
8
+ SUPABASE_URL = os.environ.get("SUPABASE_URL")
9
+ SUPABASE_KEY = os.environ.get("SUPABASE_KEY")
10
+ GOOGLE_VISION_KEY = os.environ.get("GOOGLE_VISION_KEY")
11
+ STRIPE_SECRET_KEY = os.environ.get("STRIPE_SECRET_KEY")
12
+ STRIPE_PRICE_BASIC = os.environ.get("STRIPE_PRICE_BASIC")
13
+ STRIPE_PRICE_PRO = os.environ.get("STRIPE_PRICE_PRO")
14
+ STRIPE_PRICE_ENTERPRISE = os.environ.get("STRIPE_PRICE_ENTERPRISE")
15
+ STRIPE_WEBHOOK_SECRET = os.environ.get("STRIPE_WEBHOOK_SECRET")
16
+ APP_URL = os.environ.get("APP_URL", "http://localhost:7860")
17
+
18
+ # ── Session & Cookies ───────────────────────────────────────────────────────
19
+ SESSION_TTL_HOURS = 168 # 7 days
20
+ COOKIE_NAME = "op_sid"
21
+ COOKIE_SECURE = os.environ.get("COOKIE_SECURE", "false").lower() == "true"
22
+ COOKIE_HTTPONLY = True
23
+ COOKIE_SAMESITE = "lax"
24
+
25
+ # ── Password Policy ─────────────────────────────────────────────────────────
26
+ BCRYPT_ROUNDS = 12
27
+ MIN_PASSWORD_LENGTH = 10
28
+ REQUIRE_MIXED_CASE = True
29
+ REQUIRE_DIGIT = True
30
+ REQUIRE_SPECIAL = True
31
+ BANNED_PASSWORDS = {
32
+ "password", "1234567890", "qwerty12345", "admin12345",
33
+ "letmein123", "welcome123", "monkey1234", "dragon1234",
34
+ "password123", "omniparse123", "invoice1234", "changeme123",
35
+ }
36
+
37
+ # ── File Upload Limits ──────────────────────────────────────────────────────
38
+ MAX_FILE_SIZE_MB = 20
39
+ MAX_FILES_PER_REQ = 10
40
+
41
+ # ── Rate Limits ─────────────────────────────────────────────────────────────
42
+ RATE_GLOBAL = "200/minute"
43
+ RATE_AUTH = "10/minute"
44
+ RATE_UPLOAD = "30/minute"
45
+ RATE_API = "60/minute"
46
+
47
+ # ── Plans ───────────────────────────────────────────────────────────────────
48
+ PLAN_LIMITS = {"free": 20, "basic": 200, "pro": 2000, "enterprise": float("inf")}
49
+ PLAN_LABELS = {"free": "Free", "basic": "Basic", "pro": "Pro", "enterprise": "Enterprise"}
50
+ PLAN_PRICES = {"free": 0, "basic": 29, "pro": 129, "enterprise": 499}
51
+ PLAN_PRICE_IDS = {
52
+ "basic": STRIPE_PRICE_BASIC, "pro": STRIPE_PRICE_PRO,
53
+ "enterprise": STRIPE_PRICE_ENTERPRISE,
54
+ }
55
+
56
+ # ── Database ────────────────────────────────────────────────────────────────
57
+ SQLITE_PATH = os.environ.get("SQLITE_PATH", "omniparse.db")
58
+ USE_SUPABASE = bool(SUPABASE_URL and SUPABASE_KEY)
database.py ADDED
@@ -0,0 +1,249 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """OmniParse AI — Database layer. SQLite fallback + optional Supabase. All queries parameterized."""
2
+
3
+ import sqlite3, json, secrets as _secrets
4
+ from datetime import datetime, timezone
5
+ from config import SQLITE_PATH, USE_SUPABASE, SUPABASE_URL, SUPABASE_KEY
6
+
7
+ _sb = None
8
+
9
+ def _supabase():
10
+ global _sb
11
+ if _sb is None and USE_SUPABASE:
12
+ try:
13
+ from supabase import create_client as sc
14
+ _sb = sc(SUPABASE_URL, SUPABASE_KEY)
15
+ except Exception as e:
16
+ print(f"[WARN] Supabase init failed: {e}")
17
+ return None
18
+ return _sb
19
+
20
+ def _sqlite():
21
+ c = sqlite3.connect(SQLITE_PATH)
22
+ c.row_factory = sqlite3.Row
23
+ c.execute("PRAGMA journal_mode=WAL")
24
+ c.execute("PRAGMA foreign_keys=ON")
25
+ return c
26
+
27
+ # ── Init ────────────────────────────────────────────────────────────────────
28
+ def db_init():
29
+ sb = _supabase()
30
+ if sb: return # tables created in Supabase SQL editor
31
+ c = _sqlite()
32
+ c.executescript("""
33
+ CREATE TABLE IF NOT EXISTS users (
34
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
35
+ email TEXT UNIQUE NOT NULL COLLATE NOCASE,
36
+ name TEXT NOT NULL,
37
+ password_hash TEXT NOT NULL,
38
+ plan TEXT NOT NULL DEFAULT 'free',
39
+ stripe_cid TEXT,
40
+ api_key TEXT UNIQUE,
41
+ email_verified INTEGER NOT NULL DEFAULT 0,
42
+ is_admin INTEGER NOT NULL DEFAULT 0,
43
+ created_at TEXT NOT NULL,
44
+ updated_at TEXT
45
+ );
46
+ CREATE TABLE IF NOT EXISTS sessions (
47
+ token TEXT PRIMARY KEY,
48
+ user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
49
+ expires_at TEXT NOT NULL,
50
+ created_at TEXT NOT NULL
51
+ );
52
+ CREATE TABLE IF NOT EXISTS invoices (
53
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
54
+ user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
55
+ filename TEXT, vendor TEXT, inv_number TEXT,
56
+ inv_date TEXT, due_date TEXT, amount REAL,
57
+ vat_amount REAL, total REAL, currency TEXT DEFAULT 'USD',
58
+ status TEXT DEFAULT 'done', is_duplicate INTEGER DEFAULT 0,
59
+ confidence REAL, raw_json TEXT, created_at TEXT NOT NULL
60
+ );
61
+ CREATE INDEX IF NOT EXISTS ix_inv_user ON invoices(user_id);
62
+ CREATE INDEX IF NOT EXISTS ix_inv_date ON invoices(created_at);
63
+ CREATE INDEX IF NOT EXISTS ix_ses_user ON sessions(user_id);
64
+ CREATE INDEX IF NOT EXISTS ix_ses_exp ON sessions(expires_at);
65
+ """)
66
+ c.commit(); c.close()
67
+
68
+ # ── Users ───────────────────────────────────────────────────────────────────
69
+ def create_user(email: str, name: str, password_hash: str, plan: str = "free") -> dict | None:
70
+ email = email.strip().lower()
71
+ api_key = "op_live_" + _secrets.token_hex(20)
72
+ now = datetime.now(timezone.utc).isoformat()
73
+ sb = _supabase()
74
+ if sb:
75
+ ex = sb.table("users").select("id").eq("email", email).execute()
76
+ if ex.data: return None
77
+ r = sb.table("users").insert({"email":email,"name":name,"password_hash":password_hash,
78
+ "plan":plan,"api_key":api_key,"created_at":now}).execute()
79
+ return r.data[0] if r.data else None
80
+ c = _sqlite()
81
+ try:
82
+ cur = c.execute("INSERT INTO users(email,name,password_hash,plan,api_key,created_at) VALUES(?,?,?,?,?,?)",
83
+ (email, name, password_hash, plan, api_key, now))
84
+ c.commit(); uid = cur.lastrowid
85
+ row = c.execute("SELECT * FROM users WHERE id=?", (uid,)).fetchone()
86
+ return dict(row)
87
+ except sqlite3.IntegrityError: return None
88
+ finally: c.close()
89
+
90
+ def get_user_by_email(email: str) -> dict | None:
91
+ email = email.strip().lower()
92
+ sb = _supabase()
93
+ if sb:
94
+ r = sb.table("users").select("*").eq("email",email).execute()
95
+ return r.data[0] if r.data else None
96
+ c = _sqlite()
97
+ row = c.execute("SELECT * FROM users WHERE email=? COLLATE NOCASE", (email,)).fetchone()
98
+ c.close(); return dict(row) if row else None
99
+
100
+ def get_user_by_id(uid: int) -> dict | None:
101
+ sb = _supabase()
102
+ if sb:
103
+ r = sb.table("users").select("*").eq("id",uid).execute()
104
+ return r.data[0] if r.data else None
105
+ c = _sqlite()
106
+ row = c.execute("SELECT * FROM users WHERE id=?", (uid,)).fetchone()
107
+ c.close(); return dict(row) if row else None
108
+
109
+ def update_user(uid: int, fields: dict) -> None:
110
+ fields["updated_at"] = datetime.now(timezone.utc).isoformat()
111
+ sb = _supabase()
112
+ if sb: sb.table("users").update(fields).eq("id",uid).execute(); return
113
+ c = _sqlite()
114
+ cols = ", ".join(f"{k}=?" for k in fields)
115
+ vals = list(fields.values()) + [uid]
116
+ c.execute(f"UPDATE users SET {cols} WHERE id=?", vals)
117
+ c.commit(); c.close()
118
+
119
+ def delete_user(uid: int) -> None:
120
+ sb = _supabase()
121
+ if sb:
122
+ sb.table("invoices").delete().eq("user_id",uid).execute()
123
+ sb.table("sessions").delete().eq("user_id",uid).execute()
124
+ sb.table("users").delete().eq("id",uid).execute(); return
125
+ c = _sqlite()
126
+ c.execute("DELETE FROM invoices WHERE user_id=?",(uid,))
127
+ c.execute("DELETE FROM sessions WHERE user_id=?",(uid,))
128
+ c.execute("DELETE FROM users WHERE id=?",(uid,))
129
+ c.commit(); c.close()
130
+
131
+ # ── Sessions ────────────────────────────────────────────────────────────────
132
+ def create_session(user_id: int, token: str, expires_at: str) -> None:
133
+ now = datetime.now(timezone.utc).isoformat()
134
+ sb = _supabase()
135
+ if sb:
136
+ sb.table("sessions").insert({"token":token,"user_id":user_id,"expires_at":expires_at,"created_at":now}).execute()
137
+ return
138
+ c = _sqlite()
139
+ c.execute("INSERT INTO sessions(token,user_id,expires_at,created_at) VALUES(?,?,?,?)",(token,user_id,expires_at,now))
140
+ c.commit(); c.close()
141
+
142
+ def get_session(token: str) -> dict | None:
143
+ if not token: return None
144
+ sb = _supabase()
145
+ if sb:
146
+ r = sb.table("sessions").select("*").eq("token",token).execute()
147
+ if not r.data: return None
148
+ session = r.data[0]
149
+ else:
150
+ c = _sqlite()
151
+ row = c.execute("SELECT * FROM sessions WHERE token=?",(token,)).fetchone()
152
+ c.close()
153
+ if not row: return None
154
+ session = dict(row)
155
+ try:
156
+ expires = datetime.fromisoformat(session["expires_at"])
157
+ if expires.tzinfo is None: expires = expires.replace(tzinfo=timezone.utc)
158
+ if expires < datetime.now(timezone.utc): delete_session(token); return None
159
+ except (ValueError, KeyError): return None
160
+ return session
161
+
162
+ def delete_session(token: str) -> None:
163
+ if not token: return
164
+ sb = _supabase()
165
+ if sb: sb.table("sessions").delete().eq("token",token).execute(); return
166
+ c = _sqlite()
167
+ c.execute("DELETE FROM sessions WHERE token=?",(token,))
168
+ c.commit(); c.close()
169
+
170
+ def cleanup_sessions() -> int:
171
+ now = datetime.now(timezone.utc).isoformat()
172
+ sb = _supabase()
173
+ if sb:
174
+ r = sb.table("sessions").delete().lt("expires_at",now).execute()
175
+ return len(r.data) if r.data else 0
176
+ c = _sqlite()
177
+ cur = c.execute("DELETE FROM sessions WHERE expires_at<?",(now,))
178
+ c.commit(); n = cur.rowcount; c.close(); return n
179
+
180
+ # ── Invoices ────────────────────────────────────────────────────────────────
181
+ def insert_invoice(user_id: int, data: dict) -> dict:
182
+ now = datetime.now(timezone.utc).isoformat()
183
+ row = {
184
+ "user_id": user_id,
185
+ "filename": data.get("filename"),
186
+ "vendor": data.get("vendor"),
187
+ "inv_number": data.get("invoice_number"),
188
+ "inv_date": data.get("invoice_date"),
189
+ "due_date": data.get("due_date"),
190
+ "amount": data.get("amount"),
191
+ "vat_amount": data.get("vat_amount"),
192
+ "total": data.get("total"),
193
+ "currency": data.get("currency","USD"),
194
+ "status": data.get("status","done"),
195
+ "is_duplicate": bool(data.get("is_duplicate", False)),
196
+ "confidence": data.get("confidence"),
197
+ "raw_json": json.dumps(data, ensure_ascii=False, default=str),
198
+ "created_at": now,
199
+ }
200
+ sb = _supabase()
201
+ if sb:
202
+ r = sb.table("invoices").insert(row).execute()
203
+ return r.data[0] if r.data else row
204
+ c = _sqlite()
205
+ cols = ", ".join(row.keys())
206
+ phs = ", ".join("?" for _ in row)
207
+ c.execute(f"INSERT INTO invoices({cols}) VALUES({phs})", tuple(row.values()))
208
+ c.commit()
209
+ iid = c.execute("SELECT last_insert_rowid()").fetchone()[0]
210
+ res = c.execute("SELECT * FROM invoices WHERE id=?",(iid,)).fetchone()
211
+ c.close(); return dict(res)
212
+
213
+ def get_invoices(user_id: int) -> list[dict]:
214
+ sb = _supabase()
215
+ if sb:
216
+ r = sb.table("invoices").select("*").eq("user_id",user_id).order("created_at",desc=True).execute()
217
+ return r.data or []
218
+ c = _sqlite()
219
+ rows = c.execute("SELECT * FROM invoices WHERE user_id=? ORDER BY created_at DESC",(user_id,)).fetchall()
220
+ c.close(); return [dict(r) for r in rows]
221
+
222
+ def delete_invoice(user_id: int, invoice_id: int) -> None:
223
+ sb = _supabase()
224
+ if sb: sb.table("invoices").delete().eq("id",invoice_id).eq("user_id",user_id).execute(); return
225
+ c = _sqlite()
226
+ c.execute("DELETE FROM invoices WHERE id=? AND user_id=?",(invoice_id,user_id))
227
+ c.commit(); c.close()
228
+
229
+ def count_invoices_this_month(user_id: int) -> int:
230
+ invs = get_invoices(user_id); now = datetime.now(timezone.utc); n = 0
231
+ for inv in invs:
232
+ try:
233
+ cr = datetime.fromisoformat(inv["created_at"])
234
+ if cr.year == now.year and cr.month == now.month: n += 1
235
+ except (ValueError, KeyError): pass
236
+ return n
237
+
238
+ def check_duplicate(user_id: int, vendor: str, total: float) -> bool:
239
+ if not vendor or total is None: return False
240
+ invs = get_invoices(user_id); now = datetime.now(timezone.utc)
241
+ vl = vendor.strip().lower()
242
+ for inv in invs:
243
+ try: cr = datetime.fromisoformat(inv["created_at"])
244
+ except: continue
245
+ if cr.year == now.year and cr.month == now.month:
246
+ iv = (inv.get("vendor") or "").strip().lower()
247
+ it = inv.get("total")
248
+ if iv == vl and it is not None and abs(float(it)-float(total))<0.01: return True
249
+ return False
middleware.py ADDED
@@ -0,0 +1,108 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """OmniParse AI — Rate limiting + security middleware."""
2
+
3
+ import time, re, os
4
+ from collections import defaultdict
5
+ from fastapi import Request, HTTPException
6
+ from fastapi.responses import Response
7
+ from starlette.middleware.base import BaseHTTPMiddleware
8
+ from config import (
9
+ RATE_GLOBAL, RATE_AUTH, RATE_UPLOAD, RATE_API,
10
+ MAX_FILE_SIZE_MB, MAX_FILES_PER_REQ,
11
+ STRIPE_WEBHOOK_SECRET,
12
+ )
13
+
14
+ # ── Token bucket ────────────────────────────────────────────────────────────
15
+ class TokenBucket:
16
+ def __init__(self, rate: float, cap: int):
17
+ self.tokens = float(cap); self.cap = cap; self.rate = rate
18
+ self.last = time.monotonic()
19
+ def _refill(self):
20
+ now = time.monotonic(); e = now - self.last
21
+ self.tokens = min(self.cap, self.tokens + e * self.rate); self.last = now
22
+ def consume(self, n=1) -> bool:
23
+ self._refill()
24
+ if self.tokens >= n: self.tokens -= n; return True
25
+ return False
26
+
27
+ class RateLimiter:
28
+ def __init__(self):
29
+ self._buckets: dict = defaultdict(dict)
30
+ def _get(self, key: str, limit: str) -> TokenBucket:
31
+ if limit not in self._buckets[key]:
32
+ cnt, period = limit.split("/"); cnt = int(cnt)
33
+ secs = {"second":1,"seconds":1,"minute":60,"minutes":60,"hour":3600,"hours":3600}.get(period,60)
34
+ self._buckets[key][limit] = TokenBucket(cnt/secs, cnt)
35
+ return self._buckets[key][limit]
36
+ def limited(self, key: str, limit: str) -> bool:
37
+ return not self._get(key, limit).consume()
38
+
39
+ limiter = RateLimiter()
40
+
41
+ def client_ip(req: Request) -> str:
42
+ fwd = req.headers.get("X-Forwarded-For")
43
+ if fwd: return fwd.split(",")[0].strip()
44
+ rip = req.headers.get("X-Real-IP")
45
+ if rip: return rip.strip()
46
+ return req.client.host if req.client else "unknown"
47
+
48
+ # ── Security headers middleware ─────────────────────────────────────────────
49
+ class SecurityHeadersMiddleware(BaseHTTPMiddleware):
50
+ async def dispatch(self, req: Request, call_next):
51
+ resp = await call_next(req)
52
+ resp.headers["X-Content-Type-Options"] = "nosniff"
53
+ resp.headers["X-Frame-Options"] = "DENY"
54
+ resp.headers["X-XSS-Protection"] = "1; mode=block"
55
+ resp.headers["Referrer-Policy"] = "strict-origin-when-cross-origin"
56
+ resp.headers["Permissions-Policy"] = "camera=(), microphone=(), geolocation=()"
57
+ if req.url.scheme == "https":
58
+ resp.headers["Strict-Transport-Security"] = "max-age=31536000; includeSubDomains"
59
+ return resp
60
+
61
+ # ── Rate limit helpers ──────────────────────────────────────────────────────
62
+ def global_limit(req: Request):
63
+ ip = client_ip(req)
64
+ if limiter.limited(ip, RATE_GLOBAL):
65
+ raise HTTPException(429, detail="Too many requests. Slow down.", headers={"Retry-After":"60"})
66
+
67
+ def auth_limit(req: Request):
68
+ ip = client_ip(req)
69
+ if limiter.limited(f"auth:{ip}", RATE_AUTH):
70
+ raise HTTPException(429, detail="Too many login attempts. Wait and retry.", headers={"Retry-After":"60"})
71
+
72
+ def upload_limit(req: Request):
73
+ ip = client_ip(req)
74
+ if limiter.limited(f"upload:{ip}", RATE_UPLOAD):
75
+ raise HTTPException(429, detail="Upload rate exceeded. Wait before uploading more.", headers={"Retry-After":"30"})
76
+
77
+ def api_limit(req: Request):
78
+ ip = client_ip(req)
79
+ if limiter.limited(f"api:{ip}", RATE_API):
80
+ raise HTTPException(429, detail="API rate limit exceeded.", headers={"Retry-After":"30"})
81
+
82
+ # ── File validation ─────────────────────────────────────────────────────────
83
+ def validate_file_size(size: int):
84
+ if size > MAX_FILE_SIZE_MB * 1024 * 1024:
85
+ raise HTTPException(413, detail=f"File too large. Maximum {MAX_FILE_SIZE_MB} MB.")
86
+
87
+ def validate_file_count(n: int):
88
+ if n > MAX_FILES_PER_REQ:
89
+ raise HTTPException(400, detail=f"Maximum {MAX_FILES_PER_REQ} files per upload.")
90
+
91
+ def validate_filename(fn: str) -> str:
92
+ clean = os.path.basename(fn)
93
+ clean = re.sub(r'[^a-zA-Z0-9._\- ()]', '_', clean)
94
+ if not clean or clean.startswith('.'): clean = "uploaded_file"
95
+ if len(clean) > 255:
96
+ name, ext = os.path.splitext(clean); clean = name[:250] + ext
97
+ return clean
98
+
99
+ # ── Stripe webhook signature verification ──────────────────────────────────
100
+ async def verify_stripe_webhook(req: Request, raw_body: bytes) -> bool:
101
+ """Verify Stripe webhook signature. Returns True if valid."""
102
+ if not STRIPE_WEBHOOK_SECRET: return False
103
+ try:
104
+ import stripe
105
+ sig = req.headers.get("Stripe-Signature", "")
106
+ stripe.Webhook.construct_event(raw_body, sig, STRIPE_WEBHOOK_SECRET)
107
+ return True
108
+ except Exception: return False
ocr.py ADDED
@@ -0,0 +1,63 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """OmniParse AI — OCR pipeline: Tesseract (primary) + Google Vision (fallback)."""
2
+
3
+ import io, base64
4
+ from PIL import Image as PILImage
5
+ import requests
6
+ from config import GOOGLE_VISION_KEY
7
+
8
+ # Lazy imports
9
+ _pt = None; _pdf2img = None
10
+
11
+ def _get_pt():
12
+ global _pt
13
+ if _pt is None:
14
+ try:
15
+ import pytesseract as t
16
+ _pt = t
17
+ except ImportError: pass
18
+ return _pt
19
+
20
+ def _get_pdf2img():
21
+ global _pdf2img
22
+ if _pdf2img is None:
23
+ try:
24
+ from pdf2image import convert_from_path as c
25
+ _pdf2img = c
26
+ except ImportError: pass
27
+ return _pdf2img
28
+
29
+ def pdf_to_images(path: str, dpi: int = 200) -> list:
30
+ c = _get_pdf2img()
31
+ if not c: return []
32
+ try: return c(path, dpi=dpi)
33
+ except Exception as e: print(f"[WARN] pdf2image: {e}"); return []
34
+
35
+ def load_image(path: str):
36
+ try: return PILImage.open(path).convert("RGB")
37
+ except Exception as e: print(f"[WARN] open image: {e}"); return None
38
+
39
+ def ocr_tesseract(img, lang: str = "eng") -> str:
40
+ pt = _get_pt()
41
+ if not pt: return ""
42
+ try: return pt.image_to_string(img.convert("L"), lang=lang)
43
+ except Exception as e: print(f"[WARN] tesseract: {e}"); return ""
44
+
45
+ def ocr_google_vision(img) -> str:
46
+ if not GOOGLE_VISION_KEY: return ""
47
+ try:
48
+ buf = io.BytesIO(); img.save(buf, format="PNG")
49
+ b64 = base64.b64encode(buf.getvalue()).decode("utf-8")
50
+ payload = {"requests":[{"image":{"content":b64},"features":[{"type":"TEXT_DETECTION"}]}]}
51
+ r = requests.post(
52
+ f"https://vision.googleapis.com/v1/images:annotate?key={GOOGLE_VISION_KEY}",
53
+ json=payload, timeout=15)
54
+ r.raise_for_status()
55
+ return r.json()["responses"][0].get("fullTextAnnotation",{}).get("text","")
56
+ except Exception as e: print(f"[WARN] vision: {e}"); return ""
57
+
58
+ def run_ocr(img) -> str:
59
+ text = ocr_tesseract(img)
60
+ if len(text.strip()) < 100 and GOOGLE_VISION_KEY:
61
+ vt = ocr_google_vision(img)
62
+ if len(vt.strip()) > len(text.strip()): text = vt
63
+ return text
server.py ADDED
@@ -0,0 +1,394 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ OmniParse AI — Main Server (gradio.Server → FastAPI + custom HTML frontend).
3
+
4
+ Run:
5
+ python server.py
6
+
7
+ Deploy as a HuggingFace Space with Gradio SDK.
8
+ Uses gradio.Server for queuing/concurrency + custom HTML/CSS/JS frontend.
9
+ """
10
+
11
+ import os, io, csv, json, uuid, tempfile
12
+ from datetime import datetime, timezone
13
+ from fastapi import Request, HTTPException, UploadFile, File, Form, Depends
14
+ from fastapi.responses import JSONResponse, HTMLResponse, StreamingResponse
15
+ from fastapi.staticfiles import StaticFiles
16
+ from fastapi.middleware.cors import CORSMiddleware
17
+ from gradio import Server
18
+
19
+ # ── Our modules ─────────────────────────────────────────────────────────────
20
+ from config import (
21
+ APP_URL, PLAN_LIMITS, PLAN_LABELS, PLAN_PRICES, PLAN_PRICE_IDS,
22
+ STRIPE_SECRET_KEY, MAX_FILE_SIZE_MB, MAX_FILES_PER_REQ,
23
+ )
24
+ from database import (
25
+ db_init, get_user_by_id, update_user, delete_user,
26
+ insert_invoice, get_invoices, delete_invoice,
27
+ count_invoices_this_month, check_duplicate,
28
+ )
29
+ from auth import (
30
+ register_user, login_user, create_user_session, validate_session_token,
31
+ hash_password, check_password_strength, verify_password,
32
+ set_session_cookie, clear_session_cookie,
33
+ )
34
+ from ocr import run_ocr, pdf_to_images, load_image
35
+ from ai_extraction import (
36
+ ai_extract_groq, ai_extract_hf, regex_extract,
37
+ validate_invoice, sanitize_invoice_data,
38
+ )
39
+ from middleware import (
40
+ SecurityHeadersMiddleware,
41
+ global_limit, auth_limit, upload_limit,
42
+ validate_file_size, validate_file_count, validate_filename,
43
+ )
44
+
45
+ # ── Init ────────────────────────────────────────────────────────────────────
46
+ app = Server()
47
+ app.add_middleware(SecurityHeadersMiddleware)
48
+ app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_credentials=True,
49
+ allow_methods=["GET","POST","DELETE"], allow_headers=["Content-Type","Authorization"])
50
+
51
+ BASE = os.path.dirname(os.path.abspath(__file__))
52
+ STATIC = os.path.join(BASE, "static")
53
+ os.makedirs(os.path.join(STATIC, "css"), exist_ok=True)
54
+ os.makedirs(os.path.join(STATIC, "js"), exist_ok=True)
55
+ os.makedirs(os.path.join(STATIC, "img"), exist_ok=True)
56
+ app.mount("/static", StaticFiles(directory=STATIC), name="static")
57
+
58
+ db_init()
59
+
60
+ # Lazy Stripe
61
+ _stripe = None
62
+ def _get_stripe():
63
+ global _stripe
64
+ if _stripe is None and STRIPE_SECRET_KEY:
65
+ try:
66
+ import stripe; stripe.api_key = STRIPE_SECRET_KEY; _stripe = stripe
67
+ except ImportError: pass
68
+ return _stripe
69
+
70
+ # ── Session dependency ──────────────────────────────────────────────────────
71
+ async def current_user(req: Request):
72
+ token = req.cookies.get("op_sid")
73
+ if not token: return None
74
+ return validate_session_token(token)
75
+
76
+ async def require_auth(req: Request):
77
+ u = await current_user(req)
78
+ if not u: raise HTTPException(401, detail="Authentication required.")
79
+ return u
80
+
81
+ async def require_admin(req: Request):
82
+ u = await require_auth(req)
83
+ if not u.get("is_admin"):
84
+ raise HTTPException(403, detail="Admin access required.")
85
+ return u
86
+
87
+ # ── Demo account bootstrap ──────────────────────────────────────────────────
88
+ def _ensure_demo():
89
+ from database import get_user_by_email
90
+ if not get_user_by_email("demo@omniparse.ai"):
91
+ from database import create_user as cu
92
+ cu("demo@omniparse.ai", "Demo User", hash_password("Demo@12345!"), plan="pro")
93
+ print("[INFO] Demo account: demo@omniparse.ai / Demo@12345! (Pro)")
94
+
95
+ _ensure_demo()
96
+
97
+ # ═══════════════════════════════════════════════════════════════════════════
98
+ # API Routes — Auth
99
+ # ═══════════════════════════════════════════════════════════════════════════
100
+ @app.post("/api/auth/register")
101
+ async def api_register(req: Request, email: str = Form(...), name: str = Form(...), password: str = Form(...)):
102
+ auth_limit(req); global_limit(req)
103
+ user, err = register_user(email, name, password)
104
+ if err: raise HTTPException(400, detail=err)
105
+ token = create_user_session(user["id"])
106
+ resp = JSONResponse({"status":"ok","user":_safe_user(user)})
107
+ set_session_cookie(resp, token)
108
+ return resp
109
+
110
+ @app.post("/api/auth/login")
111
+ async def api_login(req: Request, email: str = Form(...), password: str = Form(...)):
112
+ auth_limit(req); global_limit(req)
113
+ user, err = login_user(email, password)
114
+ if err: raise HTTPException(401, detail=err)
115
+ token = create_user_session(user["id"])
116
+ resp = JSONResponse({"status":"ok","user":_safe_user(user)})
117
+ set_session_cookie(resp, token)
118
+ return resp
119
+
120
+ @app.post("/api/auth/logout")
121
+ async def api_logout(req: Request):
122
+ from database import delete_session as ds
123
+ t = req.cookies.get("op_sid")
124
+ if t: ds(t)
125
+ resp = JSONResponse({"status":"ok"})
126
+ clear_session_cookie(resp)
127
+ return resp
128
+
129
+ @app.get("/api/auth/me")
130
+ async def api_me(u: dict = Depends(require_auth)):
131
+ return _safe_user(u)
132
+
133
+ # ═══════════════════════════════════════════════════════════════════════════
134
+ # API Routes — Invoices
135
+ # ═══════════════════════════════════════════════════════════════════════════
136
+ @app.post("/api/invoices/upload")
137
+ async def api_upload(req: Request, files: list[UploadFile] = File(...), u: dict = Depends(require_auth)):
138
+ upload_limit(req); global_limit(req)
139
+ validate_file_count(len(files))
140
+ plan = u.get("plan","free")
141
+ limit = PLAN_LIMITS.get(plan, 20)
142
+ used = count_invoices_this_month(u["id"])
143
+ results, skipped, errs, processed = [], 0, [], 0
144
+
145
+ for f in files:
146
+ try: contents = await f.read()
147
+ except Exception as e: errs.append(f"Read error {f.filename}: {e}"); continue
148
+ validate_file_size(len(contents))
149
+ cn = validate_filename(f.filename or "upload")
150
+ ext = cn.lower().split(".")[-1] if "." in cn else ""
151
+ if ext not in ("pdf","jpg","jpeg","png","tiff","tif"):
152
+ errs.append(f"{cn}: Unsupported format."); continue
153
+ if limit != float("inf") and (used + processed) >= limit: skipped += 1; continue
154
+ tmp = os.path.join(tempfile.gettempdir(), f"op_{uuid.uuid4().hex}_{cn}")
155
+ try:
156
+ with open(tmp,"wb") as tf: tf.write(contents)
157
+ data = _process_file(tmp, u, cn)
158
+ data = sanitize_invoice_data(data); data["filename"] = cn
159
+ if plan in ("pro","enterprise") and check_duplicate(u["id"], data.get("vendor"), data.get("total")):
160
+ data["status"] = "duplicate"; data["is_duplicate"] = True
161
+ saved = insert_invoice(u["id"], data); results.append(_serialize(saved)); processed += 1
162
+ except Exception as e: errs.append(f"{cn}: {e}")
163
+ finally:
164
+ try: os.unlink(tmp)
165
+ except OSError: pass
166
+
167
+ nu = count_invoices_this_month(u["id"])
168
+ return {"status":"ok","processed":processed,"skipped":skipped,"errors":errs,
169
+ "results":results,"usage":{"used":nu,"limit":None if limit==float("inf") else int(limit)}}
170
+
171
+ @app.get("/api/invoices")
172
+ async def api_list(req: Request, filter: str = "all", u: dict = Depends(require_auth)):
173
+ global_limit(req)
174
+ invs = get_invoices(u["id"])
175
+ out = []
176
+ for i in invs:
177
+ s = i.get("status","done")
178
+ if filter == "done" and s != "done": continue
179
+ if filter == "review" and s != "review": continue
180
+ if filter == "duplicates" and not i.get("is_duplicate"): continue
181
+ out.append(_serialize(i))
182
+ return {"status":"ok","invoices":out,"count":len(out)}
183
+
184
+ @app.delete("/api/invoices/{inv_id}")
185
+ async def api_delete(inv_id: int, req: Request, u: dict = Depends(require_auth)):
186
+ global_limit(req)
187
+ delete_invoice(u["id"], inv_id)
188
+ return {"status":"ok","deleted":inv_id}
189
+
190
+ # ═══════════════════════════════════════════════════════════════════════════
191
+ # API Routes — Export
192
+ # ═══════════════════════════════════════════════════════════════════════════
193
+ @app.get("/api/export/csv")
194
+ async def api_csv(req: Request, u: dict = Depends(require_auth)):
195
+ global_limit(req)
196
+ invs = get_invoices(u["id"])
197
+ out = io.StringIO(); w = csv.writer(out)
198
+ w.writerow(["ID","Filename","Vendor","Invoice#","Date","Due Date","Amount","VAT","Total","Currency","Status"])
199
+ for i in invs:
200
+ w.writerow([i.get("id"),i.get("filename"),i.get("vendor"),i.get("inv_number"),
201
+ i.get("inv_date"),i.get("due_date"),i.get("amount"),i.get("vat_amount"),
202
+ i.get("total"),i.get("currency"),i.get("status")])
203
+ out.seek(0)
204
+ return StreamingResponse(iter([out.getvalue()]), media_type="text/csv",
205
+ headers={"Content-Disposition":"attachment; filename=omniparse_export.csv"})
206
+
207
+ @app.get("/api/export/json")
208
+ async def api_json(req: Request, u: dict = Depends(require_auth)):
209
+ global_limit(req)
210
+ if u.get("plan") == "free": raise HTTPException(403, detail="JSON export requires Basic+ plan.")
211
+ j = json.dumps(get_invoices(u["id"]), default=str, ensure_ascii=False, indent=2)
212
+ return StreamingResponse(iter([j]), media_type="application/json",
213
+ headers={"Content-Disposition":"attachment; filename=omniparse_export.json"})
214
+
215
+ # ═══════════════════════════════════════════════════════════════════════════
216
+ # API Routes — Stripe
217
+ # ═══════════════════════════════════════════════════════════════════════════
218
+ @app.post("/api/upgrade/checkout")
219
+ async def api_checkout(req: Request, plan: str = Form(...), u: dict = Depends(require_auth)):
220
+ global_limit(req)
221
+ s = _get_stripe()
222
+ if not s: raise HTTPException(503, detail="Payments not configured.")
223
+ pid = PLAN_PRICE_IDS.get(plan)
224
+ if not pid: raise HTTPException(400, detail="Unknown plan.")
225
+ try:
226
+ session = s.checkout.Session.create(
227
+ mode="subscription", payment_method_types=["card"],
228
+ line_items=[{"price":pid,"quantity":1}], customer_email=u["email"],
229
+ success_url=f"{APP_URL}?checkout=success&session_id={{CHECKOUT_SESSION_ID}}",
230
+ cancel_url=f"{APP_URL}?checkout=cancel",
231
+ metadata={"plan":plan,"user_id":str(u["id"])})
232
+ return {"status":"ok","url":session.url}
233
+ except Exception as e: raise HTTPException(500, detail=f"Checkout error: {e}")
234
+
235
+ @app.post("/api/upgrade/verify")
236
+ async def api_verify(req: Request, session_id: str = Form(...), u: dict = Depends(require_auth)):
237
+ global_limit(req)
238
+ s = _get_stripe()
239
+ if not s: raise HTTPException(503, detail="Payments not configured.")
240
+ try:
241
+ session = s.checkout.Session.retrieve(session_id)
242
+ if session.payment_status == "paid":
243
+ plan = session.metadata.get("plan","basic"); cid = session.customer
244
+ update_user(u["id"], {"plan":plan,"stripe_cid":cid})
245
+ return {"status":"ok","plan":plan,"plan_label":PLAN_LABELS.get(plan,plan),
246
+ "message":f"Upgraded to {PLAN_LABELS.get(plan,plan)}!"}
247
+ return {"status":"pending","message":"Payment not yet confirmed."}
248
+ except Exception as e: raise HTTPException(500, detail=f"Verification error: {e}")
249
+
250
+ # ═══════════════════════════════════════════════════════════════════════════
251
+ # API Routes — Stripe Webhook (signature verified)
252
+ # ═══════════════════════════════════════════════════════════════════════════
253
+ @app.post("/api/webhooks/stripe")
254
+ async def api_stripe_webhook(req: Request):
255
+ from middleware import verify_stripe_webhook
256
+ body = await req.body()
257
+ if not await verify_stripe_webhook(req, body):
258
+ raise HTTPException(400, detail="Invalid webhook signature.")
259
+ try:
260
+ event = json.loads(body)
261
+ t = event.get("type","")
262
+ if t == "checkout.session.completed":
263
+ obj = event["data"]["object"]
264
+ plan = obj.get("metadata",{}).get("plan","basic")
265
+ uid = int(obj.get("metadata",{}).get("user_id","0"))
266
+ if uid: update_user(uid, {"plan":plan,"stripe_cid":obj.get("customer")})
267
+ elif t == "customer.subscription.deleted":
268
+ obj = event["data"]["object"]
269
+ cid = obj.get("customer")
270
+ if cid:
271
+ from database import _supabase
272
+ sb = _supabase()
273
+ if sb:
274
+ u = sb.table("users").select("id").eq("stripe_cid",cid).execute()
275
+ if u.data: update_user(u.data[0]["id"],{"plan":"free"})
276
+ except Exception as e: print(f"[WARN] webhook: {e}")
277
+ return {"status":"ok"}
278
+
279
+ # ═══════════════════════════════════════════════════════════════════════════
280
+ # API Routes — AI Chat (Pro+)
281
+ # ═══════════════════════════════════════════════════════════════════════════
282
+ @app.post("/api/chat")
283
+ async def api_chat(req: Request, message: str = Form(...), u: dict = Depends(require_auth)):
284
+ global_limit(req)
285
+ plan = u.get("plan","free")
286
+ if plan not in ("pro","enterprise"):
287
+ raise HTTPException(403, detail="AI Chat requires Pro or Enterprise plan.")
288
+ msg = message.strip()[:2000]
289
+ invs = get_invoices(u["id"])
290
+ ctx = json.dumps(invs[:100], default=str, ensure_ascii=False)[:6000]
291
+ from config import GROQ_API_KEY
292
+ answer = "AI is currently unavailable."
293
+ if GROQ_API_KEY:
294
+ try:
295
+ from groq import Groq
296
+ gc = Groq(api_key=GROQ_API_KEY)
297
+ r = gc.chat.completions.create(
298
+ model="llama-3.1-8b-instant",
299
+ messages=[{"role":"system","content":f"You answer questions about invoices. Data: {ctx}. Concise, data only. Never reveal raw JSON."},
300
+ {"role":"user","content":msg}],
301
+ max_tokens=400, temperature=0.2, timeout=15)
302
+ answer = r.choices[0].message.content
303
+ except Exception as e: answer = f"AI unavailable: {e}"
304
+ return {"status":"ok","answer":answer}
305
+
306
+ # ═══════════════════════════════════════════════════════════════════════════
307
+ # API Routes — Profile
308
+ # ═══════════════════════════════════════════════════════════════════════════
309
+ @app.post("/api/profile/password")
310
+ async def api_chpw(req: Request, current_password: str = Form(...), new_password: str = Form(...), u: dict = Depends(require_auth)):
311
+ global_limit(req); auth_limit(req)
312
+ if not verify_password(current_password, u["password_hash"]):
313
+ raise HTTPException(400, detail="Current password is incorrect.")
314
+ valid, errs = check_password_strength(new_password)
315
+ if not valid: raise HTTPException(400, detail=" ".join(errs))
316
+ update_user(u["id"], {"password_hash": hash_password(new_password)})
317
+ return {"status":"ok","message":"Password changed."}
318
+
319
+ @app.delete("/api/profile/account")
320
+ async def api_delete_account(req: Request, u: dict = Depends(require_auth)):
321
+ global_limit(req); auth_limit(req)
322
+ delete_user(u["id"])
323
+ resp = JSONResponse({"status":"ok","message":"Account deleted."})
324
+ clear_session_cookie(resp)
325
+ return resp
326
+
327
+ # ═══════════════════════════════════════════════════════════════════════════
328
+ # Health
329
+ # ═══════════════════════════════════════════════════════════════════════════
330
+ @app.get("/api/health")
331
+ async def health():
332
+ return {"status":"healthy","timestamp":datetime.now(timezone.utc).isoformat(),"version":"2.0.0"}
333
+
334
+ # ═══════════════════════════════════════════════════════════════════════════
335
+ # Internals
336
+ # ═══════════════════════════════════════════════════════════════════════════
337
+ def _process_file(filepath: str, user: dict, filename: str) -> dict:
338
+ ext = filename.lower().split(".")[-1] if "." in filename else ""
339
+ images = []
340
+ if ext == "pdf": images = pdf_to_images(filepath)
341
+ elif ext in ("jpg","jpeg","png","tiff","tif"):
342
+ img = load_image(filepath)
343
+ if img: images = [img]
344
+ ocr_text = ""
345
+ for img in images: ocr_text += run_ocr(img) + "\n"
346
+ if ocr_text.strip():
347
+ data = ai_extract_groq(ocr_text) or ai_extract_hf(ocr_text) or regex_extract(ocr_text)
348
+ else:
349
+ data = {"vendor":"Demo Vendor Inc.","invoice_number":f"DEMO-{uuid.uuid4().hex[:6].upper()}",
350
+ "invoice_date":datetime.now().strftime("%Y-%m-%d"),"due_date":None,
351
+ "amount":100.0,"vat_amount":21.0,"total":121.0,"currency":"USD","line_items":[]}
352
+ data["filename"] = filename; data["confidence"] = 0.95 if ocr_text.strip() else 0.3
353
+ warnings = validate_invoice(data); data["warnings"] = warnings
354
+ data["status"] = "review" if warnings else "done"
355
+ return data
356
+
357
+ def _serialize(inv: dict) -> dict:
358
+ labels = {"done":"Done","review":"Needs Review","duplicate":"Duplicate","processing":"Processing"}
359
+ return {"id":inv.get("id"),"filename":inv.get("filename"),"vendor":inv.get("vendor"),
360
+ "invoice_number":inv.get("inv_number"),"invoice_date":inv.get("inv_date"),
361
+ "due_date":inv.get("due_date"),"amount":inv.get("amount"),
362
+ "vat_amount":inv.get("vat_amount"),"total":inv.get("total"),
363
+ "currency":inv.get("currency","USD"),"status":inv.get("status","done"),
364
+ "status_label":labels.get(inv.get("status"),inv.get("status")),
365
+ "is_duplicate":bool(inv.get("is_duplicate")),
366
+ "confidence":inv.get("confidence"),"created_at":inv.get("created_at")}
367
+
368
+ def _safe_user(u: dict) -> dict:
369
+ return {"id":u["id"],"email":u["email"],"name":u["name"],"plan":u["plan"],
370
+ "plan_label":PLAN_LABELS.get(u["plan"],u["plan"]),
371
+ "api_key":u.get("api_key"),"created_at":u.get("created_at"),
372
+ "is_admin":bool(u.get("is_admin"))}
373
+
374
+ # ═══════════════════════════════════════════════════════════════════════════
375
+ # Frontend — serve SPA
376
+ # ═══════════════════════════════════════════════════════════════════════════
377
+ @app.get("/", response_class=HTMLResponse)
378
+ async def serve_frontend():
379
+ p = os.path.join(STATIC, "index.html")
380
+ if not os.path.exists(p): return HTMLResponse("<h1>Frontend not found.</h1>", 404)
381
+ with open(p, "r", encoding="utf-8") as f: return HTMLResponse(f.read())
382
+
383
+ @app.get("/{path:path}")
384
+ async def catch_all(path: str):
385
+ p = os.path.join(STATIC, "index.html")
386
+ if os.path.exists(p):
387
+ with open(p, "r", encoding="utf-8") as f: return HTMLResponse(f.read())
388
+ return HTMLResponse("<h1>Not Found</h1>", 404)
389
+
390
+ # ═══════════════════════════════════════════════════════════════════════════
391
+ # Launch
392
+ # ═══════════════════════════════════════════════════════════════════════════
393
+ if __name__ == "__main__":
394
+ app.launch(show_error=True, server_name="0.0.0.0", server_port=7860)