segovia commited on
Commit
db9f03c
·
1 Parent(s): 23282ad

Move the scanning to a job

Browse files
.github/workflows/orchestrator-call.yml ADDED
@@ -0,0 +1,129 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ name: Call Orchestrator Endpoint
2
+
3
+ on:
4
+ workflow_dispatch:
5
+ inputs:
6
+ model:
7
+ description: OpenAI model name
8
+ required: false
9
+ default: gpt-4o
10
+ include_subfolders:
11
+ description: Include subfolders in Drive scan
12
+ required: false
13
+ default: "true"
14
+ send_to_sheet:
15
+ description: Send rows to Google Sheets
16
+ required: false
17
+ default: "false"
18
+ schedule:
19
+ - cron: '*/5 * * * *'
20
+ jobs:
21
+ call-orchestrator:
22
+ runs-on: ubuntu-latest
23
+ timeout-minutes: 1
24
+ env:
25
+ ORCHESTRATOR_BASE_URL: ${{ secrets.ORCHESTRATOR_BASE_URL }}
26
+ ORCHESTRATOR_API_KEY: ${{ secrets.ORCHESTRATOR_API_KEY }}
27
+ MODEL_INPUT: ${{ github.event.inputs.model || 'gpt-4o' }}
28
+ INCLUDE_SUBFOLDERS_INPUT: ${{ github.event.inputs.include_subfolders || 'true' }}
29
+ SEND_TO_SHEET_INPUT: ${{ github.event.inputs.send_to_sheet || 'false' }}
30
+
31
+ steps:
32
+ - name: Validate required secrets
33
+ run: |
34
+ set -euo pipefail
35
+ if [ -z "${ORCHESTRATOR_BASE_URL}" ]; then
36
+ echo "Missing secret: ORCHESTRATOR_BASE_URL"
37
+ exit 1
38
+ fi
39
+ if [ -z "${ORCHESTRATOR_API_KEY}" ]; then
40
+ echo "Missing secret: ORCHESTRATOR_API_KEY"
41
+ exit 1
42
+ fi
43
+
44
+ - name: Start orchestrator job (POST)
45
+ id: start
46
+ shell: bash
47
+ run: |
48
+ set -euo pipefail
49
+ python - <<'PY'
50
+ import json
51
+ import os
52
+ import urllib.request
53
+
54
+ base_url = os.environ["ORCHESTRATOR_BASE_URL"].rstrip("/")
55
+ payload = {
56
+ "data": [
57
+ os.environ["MODEL_INPUT"],
58
+ os.environ["INCLUDE_SUBFOLDERS_INPUT"].lower() == "true",
59
+ os.environ["SEND_TO_SHEET_INPUT"].lower() == "true",
60
+ os.environ["ORCHESTRATOR_API_KEY"],
61
+ ]
62
+ }
63
+
64
+ req = urllib.request.Request(
65
+ url=f"{base_url}/gradio_api/call/orchestrate_drive_documents",
66
+ data=json.dumps(payload).encode("utf-8"),
67
+ headers={"Content-Type": "application/json"},
68
+ method="POST",
69
+ )
70
+
71
+ with urllib.request.urlopen(req, timeout=60) as resp:
72
+ body = resp.read().decode("utf-8")
73
+
74
+ parsed = json.loads(body)
75
+ event_id = parsed.get("event_id")
76
+ if not event_id:
77
+ raise SystemExit(f"POST succeeded but event_id missing. Response: {body}")
78
+
79
+ with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as fh:
80
+ fh.write(f"event_id={event_id}\n")
81
+
82
+ print("Orchestrator job started.")
83
+ PY
84
+
85
+ - name: Read orchestrator stream result
86
+ shell: bash
87
+ run: |
88
+ set -euo pipefail
89
+ STREAM_URL="${ORCHESTRATOR_BASE_URL%/}/gradio_api/call/orchestrate_drive_documents/${{ steps.start.outputs.event_id }}"
90
+ curl -sS --max-time 900 -N "$STREAM_URL" > stream.txt
91
+ echo "Stream captured in stream.txt"
92
+ id: stream
93
+
94
+ - name: Validate orchestrator response
95
+ shell: bash
96
+ run: |
97
+ set -euo pipefail
98
+ python - <<'PY'
99
+ import json
100
+
101
+ with open("stream.txt", "r", encoding="utf-8") as fh:
102
+ lines = [line.rstrip("\n") for line in fh]
103
+
104
+ data_lines = [line[6:] for line in lines if line.startswith("data: ")]
105
+ non_null = [line for line in data_lines if line and line != "null"]
106
+ if not non_null:
107
+ raise SystemExit("No non-null data payload found in stream.")
108
+
109
+ payload = json.loads(non_null[-1])
110
+ if not isinstance(payload, list) or not payload:
111
+ raise SystemExit(f"Unexpected payload shape: {payload}")
112
+
113
+ result = payload[0]
114
+ ok = bool(result.get("ok")) if isinstance(result, dict) else False
115
+ if not ok:
116
+ raise SystemExit(f"Orchestrator failed: {json.dumps(result, ensure_ascii=False)}")
117
+
118
+ summary = {
119
+ "ok": result.get("ok"),
120
+ "scanned": result.get("scanned"),
121
+ "to_process": result.get("to_process"),
122
+ "skipped": result.get("skipped"),
123
+ "parsed": result.get("parsed"),
124
+ "modified": result.get("modified"),
125
+ "sent": result.get("sent"),
126
+ "errors_count": len(result.get("errors", [])) if isinstance(result.get("errors"), list) else None,
127
+ }
128
+ print(json.dumps(summary, ensure_ascii=False))
129
+ PY
README.md CHANGED
@@ -11,3 +11,83 @@ pinned: false
11
  ---
12
 
13
  Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
11
  ---
12
 
13
  Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
14
+
15
+ ## Drive Folder Scan (Unprocessed Files)
16
+
17
+ The app supports scanning one or more Google Drive folders and parsing only files that were not processed before.
18
+
19
+ ### Required environment variables
20
+
21
+ - `GOOGLE_SERVICE_ACCOUNT_JSON` or `GOOGLE_SERVICE_ACCOUNT_FILE`
22
+ - `GOOGLE_DRIVE_FOLDER_IDS`: comma-separated Drive folder IDs
23
+
24
+ Optional:
25
+
26
+ - `FIREBASE_TRACK_PROCESSED`: defaults to `true`
27
+ - `FIREBASE_PROCESSED_COLLECTION`: defaults to `processed_files`
28
+ - `FIREBASE_DOCUMENTS_COLLECTION`: defaults to `processed_documents`
29
+
30
+ ### How it works
31
+
32
+ 1. Click **Scan Drive folders** in the UI.
33
+ 2. The app recursively scans configured Drive folders.
34
+ 3. Supported file types are downloaded to a temporary local folder (`.txt`, `.pdf`, `.docx`, `.jpg`, `.jpeg`, `.png`).
35
+ 4. Files already registered in Firebase are skipped.
36
+ 5. New files are parsed and appended to the same grid used by uploads.
37
+
38
+ ## Drive Orchestration (Status + modificationDate)
39
+
40
+ The orchestration script scans Drive, checks if a document was already processed, parses only new/modified documents, and saves parsed payload + lifecycle status in Firebase.
41
+
42
+ ### Status rules
43
+
44
+ - `Parsed`: document was never processed before.
45
+ - `Modified`: document exists in Firebase but Drive `modifiedTime` changed.
46
+ - `Sent`: document was parsed and then sent to Google Sheets.
47
+
48
+ ### Run orchestration
49
+
50
+ ```bash
51
+ python document_orchester.py --model gpt-4o
52
+ ```
53
+
54
+ Send parsed rows to Google Sheets and mark them as `Sent`:
55
+
56
+ ```bash
57
+ python document_orchester.py --model gpt-4o --send
58
+ ```
59
+
60
+ Optional flags:
61
+
62
+ - `--no-subfolders`: scan only direct files in configured folders.
63
+ - `--output <path>`: save orchestration summary JSON to a file.
64
+
65
+ ## Public API Endpoint For Orchestrator
66
+
67
+ The Gradio app exposes a public API endpoint that internally calls the orchestrator flow.
68
+
69
+ Security:
70
+
71
+ - Set `ORCHESTRATOR_API_KEY` in the server environment.
72
+ - Every endpoint call must include that key in the payload.
73
+
74
+ Endpoint name:
75
+
76
+ - `orchestrate_drive_documents`
77
+
78
+ Local URL example:
79
+
80
+ ```bash
81
+ curl -X POST "http://localhost:7860/gradio_api/call/orchestrate_drive_documents" \
82
+ -H "Content-Type: application/json" \
83
+ -d '{"data": ["gpt-4o", true, false, "YOUR_API_KEY"]}'
84
+ ```
85
+
86
+ `data` argument order:
87
+
88
+ 1. `model` (string)
89
+ 2. `include_subfolders` (boolean)
90
+ 3. `send_to_sheet` (boolean)
91
+ 4. `api_key` (string)
92
+
93
+ Response payload is the same summary produced by `orchestrate_drive_documents` in `document_orchester.py`.
app.py CHANGED
@@ -1,20 +1,15 @@
1
  from __future__ import annotations
2
 
3
  import os
4
- import csv
5
- import tempfile
6
- from pathlib import Path
7
 
8
  # Avoid the Node.js SSR proxy in hosted environments where it can terminate unexpectedly.
9
  os.environ["GRADIO_SSR_MODE"] = "false"
10
 
11
  import gradio as gr
12
- from dotenv import load_dotenv
13
- from openai import OpenAI
14
 
15
- from document_processing import extract_text, is_image_document, to_data_uri
16
- from receipt_ai import extract_receipt_json, extract_receipt_json_from_image
17
- from receipt_results import build_empty_result
18
  from google_sheets_service import GoogleSheetsConfigError, append_row_to_google_sheet
19
 
20
  try:
@@ -36,71 +31,138 @@ RESULT_COLUMNS = [
36
  "total",
37
  "confidence",
38
  ]
39
- SELECT_COLUMN = "sel"
 
 
 
 
 
40
  custom_css = """
41
  .cell-menu, .cell-menu-button {
42
  display: none !important;
43
  }
44
  """
45
 
 
46
  def gpu_decorator(fn):
47
  if spaces is None:
48
  return fn
49
  return spaces.GPU(fn)
50
 
51
 
52
- load_dotenv()
53
 
54
 
55
- def create_openai_client() -> OpenAI | None:
56
- api_key = (os.getenv("OPENAI_API_KEY") or "").strip()
57
- if not api_key:
58
- return None
59
- return OpenAI(api_key=api_key)
60
 
 
 
61
 
62
- client = create_openai_client()
63
 
 
 
 
 
 
 
64
 
65
- def parse_uploaded_file(client, model: str, file_path: str) -> dict:
66
- path = Path(file_path)
67
- if is_image_document(path):
68
- result = extract_receipt_json_from_image(client, model, path, to_data_uri(path))
69
- else:
70
- text = extract_text(path)
71
- if not text.strip():
72
- return build_empty_result(path.name)
73
- result = extract_receipt_json(client, model, path, text)
74
 
75
- result["source_file"] = path.name
76
- return result
 
 
 
 
 
77
 
78
 
79
  def rows_to_table(rows: list[dict]) -> list[list[object]]:
80
- return [[False] + [row.get(column) for column in RESULT_COLUMNS] for row in rows]
 
 
 
 
 
 
 
 
 
 
81
 
82
 
83
  def table_to_rows(table: object) -> list[dict]:
84
  if table is None:
85
  return []
86
 
87
- if hasattr(table, "to_dict"):
88
  try:
89
- records = table.to_dict(orient="records")
90
- return [{column: record.get(column) for column in RESULT_COLUMNS} for record in records]
91
- except TypeError:
 
 
 
 
 
 
 
 
 
 
92
  pass
93
 
94
  rows: list[dict] = []
95
  for raw_row in table: # type: ignore[assignment]
96
  values = list(raw_row)
97
- # Dataframe rows include a leading checkbox value that is not part of the result payload.
98
- if len(values) > len(RESULT_COLUMNS):
99
- values = values[-len(RESULT_COLUMNS):]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
100
  row = {}
101
- for index, column in enumerate(RESULT_COLUMNS):
102
  row[column] = values[index] if index < len(values) else None
103
  rows.append(row)
 
104
  return rows
105
 
106
 
@@ -113,7 +175,8 @@ def get_selected_row_indexes(grid_value: object) -> list[int]:
113
  if hasattr(grid_value, "iterrows"):
114
  try:
115
  for index, row in grid_value.iterrows():
116
- if bool(row.get(SELECT_COLUMN, False)):
 
117
  selected_indexes.append(int(index))
118
  return selected_indexes
119
  except Exception:
@@ -121,146 +184,242 @@ def get_selected_row_indexes(grid_value: object) -> list[int]:
121
 
122
  for index, raw_row in enumerate(grid_value): # type: ignore[assignment]
123
  values = list(raw_row)
124
- if values and bool(values[0]):
125
  selected_indexes.append(index)
126
 
127
  return selected_indexes
128
 
129
 
130
- def build_csv_download(rows: list[dict]) -> str:
131
- temp_file = tempfile.NamedTemporaryFile(delete=False, suffix=".csv", mode="w", newline="", encoding="utf-8")
132
- with temp_file as csv_file:
133
- writer = csv.DictWriter(csv_file, fieldnames=RESULT_COLUMNS)
134
- writer.writeheader()
135
- for row in rows:
136
- writer.writerow({column: row.get(column) for column in RESULT_COLUMNS})
137
 
138
- return temp_file.name
 
 
 
 
139
 
 
 
140
 
141
- @gpu_decorator
142
- def process_files(files: list[str] | None, existing_rows: list[dict] | None, model: str):
143
- rows = list(existing_rows or [])
144
- if not files:
145
- return rows, gr.update(value=rows_to_table(rows)), "No files selected."
146
-
147
- if client is None:
148
- return (
149
- rows,
150
- gr.update(value=rows_to_table(rows)),
151
- "Missing OPENAI_API_KEY. Add it as a Hugging Face Space Secret or define it in your local environment.",
152
- )
153
 
154
- for file_path in files:
155
- rows.append(parse_uploaded_file(client, model, file_path))
156
 
157
- return rows, gr.update(value=rows_to_table(rows)), f"Parsed {len(files)} file(s). Total rows: {len(rows)}."
 
 
 
 
158
 
 
 
 
 
 
159
 
160
- def clear_results():
161
- return [], gr.update(value=[]), "Results cleared."
 
162
 
163
 
164
- def sync_grid_to_state(grid_value: object):
165
- rows = table_to_rows(grid_value)
166
- return rows, "Grid updated."
167
 
168
 
169
- def select_grid_row(evt: gr.SelectData):
170
- index = evt.index
171
- if isinstance(index, (tuple, list)) and len(index) == 2:
172
- row_index = index[0]
173
- elif isinstance(index, int):
174
- row_index = index
175
  else:
176
- return None, "Select a row first."
177
 
178
- try:
179
- row_index = int(row_index)
180
- except (TypeError, ValueError):
181
- return None, "Select a row first."
182
 
183
- if row_index < 0:
184
- return None, "Select a row first."
 
 
 
 
 
 
 
 
 
 
185
 
186
- return row_index, f"Selected row {row_index + 1}."
187
 
188
 
189
  def delete_selected_rows(existing_rows: list[dict] | None, grid_value: object):
190
  rows = list(existing_rows or [])
191
  selected_indexes = get_selected_row_indexes(grid_value)
192
  if not selected_indexes:
193
- return rows, gr.update(value=rows_to_table(rows)), "Select at least one row with the checkbox first."
194
 
195
  for index in selected_indexes:
196
  if index < 0 or index >= len(rows):
197
- return rows, gr.update(value=rows_to_table(rows)), "One or more selected rows are out of range."
198
 
199
  selected_set = set(selected_indexes)
200
  remaining_rows = [row for index, row in enumerate(rows) if index not in selected_set]
 
201
  return (
202
  remaining_rows,
203
  gr.update(value=rows_to_table(remaining_rows)),
204
- f"Deleted {len(selected_indexes)} row(s).",
205
  )
206
 
207
 
 
 
 
 
208
  def send_selected_rows(existing_rows: list[dict] | None, grid_value: object):
209
  rows = list(existing_rows or [])
210
- selected_indexes = get_selected_row_indexes(grid_value)
 
211
  if not selected_indexes:
212
- return rows, gr.update(value=rows_to_table(rows)), "Select at least one row with the checkbox first."
213
 
 
 
 
 
 
 
 
214
  try:
215
  for index in selected_indexes:
216
- if index < 0 or index >= len(rows):
217
- return rows, gr.update(value=rows_to_table(rows)), "One or more selected rows are out of range."
218
- append_row_to_google_sheet(rows[index], RESULT_COLUMNS)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
219
  except GoogleSheetsConfigError as exc:
220
- return rows, gr.update(value=rows_to_table(rows)), f"Google Sheets configuration error: {exc}"
 
 
 
 
 
 
 
 
 
221
  except Exception as exc:
222
- return rows, gr.update(value=rows_to_table(rows)), f"Failed to insert row into Google Sheets: {exc}"
 
 
 
 
 
 
 
 
 
223
 
224
  selected_set = set(selected_indexes)
225
  remaining_rows = [row for index, row in enumerate(rows) if index not in selected_set]
226
  return (
227
  remaining_rows,
228
  gr.update(value=rows_to_table(remaining_rows)),
229
- f"Sent {len(selected_indexes)} row(s) to Google Sheets and removed them from the grid.",
230
  )
231
 
232
- def build_app() -> gr.Blocks:
233
- default_model = os.getenv("OPENAI_MODEL", "gpt-4o")
234
 
235
- with gr.Blocks(css= custom_css,title="Document Parser") as demo:
236
- gr.Markdown("# Purchase Receipt Parser\nUpload one or more files and append the extracted receipt data into the grid.")
 
237
 
238
- with gr.Row():
239
- files = gr.File(label="Upload files", file_count="multiple", type="filepath")
240
- model = gr.Textbox(label="Model", value=default_model)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
241
 
242
- status = gr.Markdown("Ready.")
243
- results_state = gr.State([])
244
- selected_row_state = gr.State(None)
 
 
245
 
246
  with gr.Row():
247
- parse_button = gr.Button("Parse documents", variant="primary")
248
- clear_button = gr.Button("Clear grid")
249
- send_row_button = gr.Button("Send", variant="secondary")
250
- delete_row_button = gr.Button("Delete selected", variant="stop")
251
- download_button = gr.DownloadButton(label="Download CSV", variant="secondary", value=build_csv_download, inputs=[results_state])
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
252
 
253
  results_grid = gr.Dataframe(
254
- label="Parsed documents",
255
- headers=[SELECT_COLUMN] + RESULT_COLUMNS,
256
- value=rows_to_table([]),
257
  interactive=True,
258
  wrap=True,
259
  )
260
 
261
- parse_button.click(
262
- process_files,
263
- inputs=[files, results_state, model],
264
  outputs=[results_state, results_grid, status],
265
  )
266
 
@@ -272,16 +431,10 @@ def build_app() -> gr.Blocks:
272
 
273
  results_grid.change(
274
  sync_grid_to_state,
275
- inputs=[results_grid],
276
  outputs=[results_state, status],
277
  )
278
 
279
- results_grid.select(
280
- select_grid_row,
281
- inputs=[],
282
- outputs=[selected_row_state, status],
283
- )
284
-
285
  delete_row_button.click(
286
  delete_selected_rows,
287
  inputs=[results_state, results_grid],
@@ -294,6 +447,13 @@ def build_app() -> gr.Blocks:
294
  outputs=[results_state, results_grid, status],
295
  )
296
 
 
 
 
 
 
 
 
297
  return demo
298
 
299
 
@@ -306,12 +466,12 @@ if __name__ == "__main__":
306
  # Spaces manages networking itself; local runs keep explicit host/port.
307
  if os.getenv("SPACE_ID"):
308
  try:
309
- demo.launch(ssr_mode=False, **launch_kwargs)
310
  except TypeError:
311
  # Backward-compatible fallback if this Gradio build does not accept ssr_mode.
312
- demo.launch(**launch_kwargs)
313
  else:
314
  try:
315
- demo.launch(server_name="0.0.0.0", server_port=7860, ssr_mode=False, **launch_kwargs)
316
  except TypeError:
317
- demo.launch(server_name="0.0.0.0", server_port=7860, **launch_kwargs)
 
1
  from __future__ import annotations
2
 
3
  import os
4
+ from datetime import datetime
 
 
5
 
6
  # Avoid the Node.js SSR proxy in hosted environments where it can terminate unexpectedly.
7
  os.environ["GRADIO_SSR_MODE"] = "false"
8
 
9
  import gradio as gr
 
 
10
 
11
+ from document_orchester import orchestrate_drive_documents
12
+ from firebase_processed_files import build_firebase_tracker
 
13
  from google_sheets_service import GoogleSheetsConfigError, append_row_to_google_sheet
14
 
15
  try:
 
31
  "total",
32
  "confidence",
33
  ]
34
+ DISPLAY_DESCRIPTION_COLUMN = "display_description"
35
+ GRID_COLUMNS = ["confidence", "fecha", "source_file", DISPLAY_DESCRIPTION_COLUMN, "total"]
36
+ GRID_EDITABLE_COLUMNS = ["confidence", "fecha", "source_file", "total"]
37
+ GRID_HEADER_LABELS = ["Confianza", "Fecha", "Nombre del archivo", "Descripcion", "Total", "Seleccionar"]
38
+ SHEET_COLUMNS = ["fecha", "source_file", DISPLAY_DESCRIPTION_COLUMN, "total"]
39
+ PENDING_STATUSES = ["Parsed", "Modified", "parsed", "modified"]
40
  custom_css = """
41
  .cell-menu, .cell-menu-button {
42
  display: none !important;
43
  }
44
  """
45
 
46
+
47
  def gpu_decorator(fn):
48
  if spaces is None:
49
  return fn
50
  return spaces.GPU(fn)
51
 
52
 
53
+ firebase_tracker = build_firebase_tracker()
54
 
55
 
56
+ def build_display_description(row: dict) -> str:
57
+ provider_name = str(row.get("description_proveedor") or "").strip()
58
+ provider_cuit = str(row.get("cuit_proveedor") or "").strip()
59
+ city = str(row.get("ciudad") or row.get("city") or "").strip()
 
60
 
61
+ parts = [part for part in [provider_name, provider_cuit, city] if part]
62
+ return " - ".join(parts)
63
 
 
64
 
65
+ def sort_rows_by_fecha(rows: list[dict]) -> list[dict]:
66
+ def sort_key(row: dict):
67
+ raw = row.get("fecha")
68
+ text = "" if raw is None else str(raw).strip()
69
+ if not text:
70
+ return (1, float("inf"), "")
71
 
72
+ for fmt in ("%Y-%m-%d", "%d/%m/%Y", "%d-%m-%Y", "%Y/%m/%d"):
73
+ try:
74
+ parsed = datetime.strptime(text, fmt)
75
+ return (0, parsed.timestamp(), "")
76
+ except ValueError:
77
+ continue
 
 
 
78
 
79
+ try:
80
+ parsed = datetime.fromisoformat(text)
81
+ return (0, parsed.timestamp(), "")
82
+ except ValueError:
83
+ return (0, float("inf"), text.lower())
84
+
85
+ return sorted(rows, key=sort_key)
86
 
87
 
88
  def rows_to_table(rows: list[dict]) -> list[list[object]]:
89
+ table_rows: list[list[object]] = []
90
+ for row in rows:
91
+ values: list[object] = []
92
+ for column in GRID_COLUMNS:
93
+ if column == DISPLAY_DESCRIPTION_COLUMN:
94
+ values.append(build_display_description(row))
95
+ else:
96
+ values.append(row.get(column))
97
+ table_rows.append(values + [False])
98
+
99
+ return table_rows
100
 
101
 
102
  def table_to_rows(table: object) -> list[dict]:
103
  if table is None:
104
  return []
105
 
106
+ if hasattr(table, "iterrows"):
107
  try:
108
+ rows: list[dict] = []
109
+ for _, record in table.iterrows():
110
+ values = list(record)
111
+ if len(values) > len(GRID_COLUMNS):
112
+ values = values[: len(GRID_COLUMNS)]
113
+ row = {}
114
+ for index, column in enumerate(GRID_COLUMNS):
115
+ if column not in GRID_EDITABLE_COLUMNS:
116
+ continue
117
+ row[column] = values[index] if index < len(values) else None
118
+ rows.append(row)
119
+ return rows
120
+ except Exception:
121
  pass
122
 
123
  rows: list[dict] = []
124
  for raw_row in table: # type: ignore[assignment]
125
  values = list(raw_row)
126
+ if len(values) > len(GRID_COLUMNS):
127
+ values = values[: len(GRID_COLUMNS)]
128
+ row = {}
129
+ for index, column in enumerate(GRID_COLUMNS):
130
+ if column not in GRID_EDITABLE_COLUMNS:
131
+ continue
132
+ row[column] = values[index] if index < len(values) else None
133
+ rows.append(row)
134
+ return rows
135
+
136
+
137
+ def table_to_display_rows(table: object) -> list[dict]:
138
+ if table is None:
139
+ return []
140
+
141
+ rows: list[dict] = []
142
+
143
+ if hasattr(table, "iterrows"):
144
+ try:
145
+ for _, record in table.iterrows():
146
+ values = list(record)
147
+ if len(values) > len(GRID_COLUMNS):
148
+ values = values[: len(GRID_COLUMNS)]
149
+ row = {}
150
+ for index, column in enumerate(GRID_COLUMNS):
151
+ row[column] = values[index] if index < len(values) else None
152
+ rows.append(row)
153
+ return rows
154
+ except Exception:
155
+ pass
156
+
157
+ for raw_row in table: # type: ignore[assignment]
158
+ values = list(raw_row)
159
+ if len(values) > len(GRID_COLUMNS):
160
+ values = values[: len(GRID_COLUMNS)]
161
  row = {}
162
+ for index, column in enumerate(GRID_COLUMNS):
163
  row[column] = values[index] if index < len(values) else None
164
  rows.append(row)
165
+
166
  return rows
167
 
168
 
 
175
  if hasattr(grid_value, "iterrows"):
176
  try:
177
  for index, row in grid_value.iterrows():
178
+ values = list(row)
179
+ if values and bool(values[-1]):
180
  selected_indexes.append(int(index))
181
  return selected_indexes
182
  except Exception:
 
184
 
185
  for index, raw_row in enumerate(grid_value): # type: ignore[assignment]
186
  values = list(raw_row)
187
+ if values and bool(values[-1]):
188
  selected_indexes.append(index)
189
 
190
  return selected_indexes
191
 
192
 
193
+ def _build_row_from_document(document: dict) -> dict:
194
+ parsed_data = document.get("parsed_data")
 
 
 
 
 
195
 
196
+ row: dict = {column: None for column in RESULT_COLUMNS}
197
+ if isinstance(parsed_data, dict):
198
+ for column in RESULT_COLUMNS:
199
+ if column in parsed_data:
200
+ row[column] = parsed_data.get(column)
201
 
202
+ if document.get("source_file"):
203
+ row["source_file"] = document.get("source_file")
204
 
205
+ row["_document_id"] = str(document.get("document_id") or "").strip()
206
+ row["_modificationDate"] = document.get("modificationDate")
207
+ row["_status"] = document.get("status")
208
+ return row
 
 
 
 
 
 
 
 
209
 
 
 
210
 
211
+ @gpu_decorator
212
+ def load_pending_documents():
213
+ if firebase_tracker is None:
214
+ rows: list[dict] = []
215
+ return rows, gr.update(value=rows_to_table(rows)), "Firebase tracking no esta habilitado."
216
 
217
+ try:
218
+ documents = firebase_tracker.list_documents_by_statuses(PENDING_STATUSES)
219
+ except Exception as exc:
220
+ rows = []
221
+ return rows, gr.update(value=rows_to_table(rows)), f"No se pudieron cargar documentos pendientes: {exc}"
222
 
223
+ rows = [_build_row_from_document(document) for document in documents]
224
+ rows = sort_rows_by_fecha(rows)
225
+ return rows, gr.update(value=rows_to_table(rows)), f"Se cargaron {len(rows)} documento(s) con estado Parsed/Modified."
226
 
227
 
228
+ def clear_results():
229
+ return [], gr.update(value=[]), "Grilla limpiada."
 
230
 
231
 
232
+ def sync_grid_to_state(*args: object):
233
+ if len(args) == 1:
234
+ existing_rows = None
235
+ grid_value = args[0]
236
+ elif len(args) == 2:
237
+ existing_rows, grid_value = args
238
  else:
239
+ return [], "Grilla actualizada."
240
 
241
+ rows = table_to_rows(grid_value)
242
+ previous_rows = list(existing_rows or []) if isinstance(existing_rows, list) else []
 
 
243
 
244
+ merged_rows: list[dict] = []
245
+ for index, row in enumerate(rows):
246
+ merged_row: dict = {column: None for column in RESULT_COLUMNS}
247
+ if index < len(previous_rows):
248
+ previous_row = previous_rows[index]
249
+ if isinstance(previous_row, dict):
250
+ merged_row.update(previous_row)
251
+ for key, value in previous_row.items():
252
+ if key not in RESULT_COLUMNS:
253
+ merged_row[key] = value
254
+ merged_row.update(row)
255
+ merged_rows.append(merged_row)
256
 
257
+ return merged_rows, "Grilla actualizada."
258
 
259
 
260
  def delete_selected_rows(existing_rows: list[dict] | None, grid_value: object):
261
  rows = list(existing_rows or [])
262
  selected_indexes = get_selected_row_indexes(grid_value)
263
  if not selected_indexes:
264
+ return rows, gr.update(value=rows_to_table(rows)), "Selecciona al menos una fila con el checkbox."
265
 
266
  for index in selected_indexes:
267
  if index < 0 or index >= len(rows):
268
+ return rows, gr.update(value=rows_to_table(rows)), "Una o mas filas seleccionadas estan fuera de rango."
269
 
270
  selected_set = set(selected_indexes)
271
  remaining_rows = [row for index, row in enumerate(rows) if index not in selected_set]
272
+
273
  return (
274
  remaining_rows,
275
  gr.update(value=rows_to_table(remaining_rows)),
276
+ f"Se eliminaron {len(selected_indexes)} fila(s).",
277
  )
278
 
279
 
280
+ def _build_updated_parsed_data(row: dict) -> dict:
281
+ return {column: row.get(column) for column in RESULT_COLUMNS}
282
+
283
+
284
  def send_selected_rows(existing_rows: list[dict] | None, grid_value: object):
285
  rows = list(existing_rows or [])
286
+ selected_indexes = sorted(set(get_selected_row_indexes(grid_value)))
287
+ display_rows = table_to_display_rows(grid_value)
288
  if not selected_indexes:
289
+ return rows, gr.update(value=rows_to_table(rows)), "Selecciona al menos una fila con el checkbox."
290
 
291
+ for index in selected_indexes:
292
+ if index < 0 or index >= len(rows):
293
+ return rows, gr.update(value=rows_to_table(rows)), "Una o mas filas seleccionadas estan fuera de rango."
294
+ if index >= len(display_rows):
295
+ return rows, gr.update(value=rows_to_table(rows)), "No se pudo leer una o mas filas seleccionadas de la grilla."
296
+
297
+ sent_indexes: list[int] = []
298
  try:
299
  for index in selected_indexes:
300
+ row = rows[index]
301
+ row_for_sheet = {column: display_rows[index].get(column) for column in SHEET_COLUMNS}
302
+ append_row_to_google_sheet(row_for_sheet, SHEET_COLUMNS)
303
+
304
+ if firebase_tracker is not None:
305
+ document_id = str(row.get("_document_id") or "").strip()
306
+ source_file = str(row.get("source_file") or "").strip()
307
+ modification_date = row.get("_modificationDate")
308
+ parsed_data = _build_updated_parsed_data(row)
309
+ if document_id:
310
+ firebase_tracker.save_document_record(
311
+ document_id=document_id,
312
+ source_file=source_file,
313
+ modification_date=modification_date,
314
+ status="sent",
315
+ parsed_data=parsed_data,
316
+ )
317
+
318
+ sent_indexes.append(index)
319
  except GoogleSheetsConfigError as exc:
320
+ if not sent_indexes:
321
+ return rows, gr.update(value=rows_to_table(rows)), f"Error de configuracion de Google Sheets: {exc}"
322
+
323
+ sent_set = set(sent_indexes)
324
+ remaining_rows = [row for index, row in enumerate(rows) if index not in sent_set]
325
+ return (
326
+ remaining_rows,
327
+ gr.update(value=rows_to_table(remaining_rows)),
328
+ f"Se enviaron {len(sent_indexes)} fila(s) antes de un error de configuracion de Google Sheets: {exc}",
329
+ )
330
  except Exception as exc:
331
+ if not sent_indexes:
332
+ return rows, gr.update(value=rows_to_table(rows)), f"Error al insertar filas en Google Sheets: {exc}"
333
+
334
+ sent_set = set(sent_indexes)
335
+ remaining_rows = [row for index, row in enumerate(rows) if index not in sent_set]
336
+ return (
337
+ remaining_rows,
338
+ gr.update(value=rows_to_table(remaining_rows)),
339
+ f"Se enviaron {len(sent_indexes)} fila(s) antes del fallo: {exc}",
340
+ )
341
 
342
  selected_set = set(selected_indexes)
343
  remaining_rows = [row for index, row in enumerate(rows) if index not in selected_set]
344
  return (
345
  remaining_rows,
346
  gr.update(value=rows_to_table(remaining_rows)),
347
+ f"Se enviaron {len(selected_indexes)} fila(s) a Google Sheets y se actualizaron en Firebase con estado sent.",
348
  )
349
 
 
 
350
 
351
+ def run_orchestrator_endpoint(model: str, include_subfolders: bool, send_to_sheet: bool, api_key: str) -> dict:
352
+ expected_api_key = str(os.getenv("ORCHESTRATOR_API_KEY") or "").strip()
353
+ provided_api_key = str(api_key or "").strip()
354
 
355
+ if provided_api_key != expected_api_key:
356
+ return {
357
+ "ok": False,
358
+ "error": "Unauthorized: invalid API key.",
359
+ }
360
+
361
+ model_name = str(model or "").strip() or os.getenv("OPENAI_MODEL", "gpt-4o")
362
+ try:
363
+ return orchestrate_drive_documents(
364
+ model=model_name,
365
+ include_subfolders=bool(include_subfolders),
366
+ send_to_sheet=bool(send_to_sheet),
367
+ )
368
+ except Exception as exc:
369
+ return {
370
+ "ok": False,
371
+ "error": str(exc),
372
+ }
373
+
374
+
375
+ def build_app() -> gr.Blocks:
376
+ initial_rows, _, initial_status = load_pending_documents()
377
 
378
+ with gr.Blocks(title="Procesador de Documentos") as demo:
379
+ gr.Markdown("# Procesador de Comprobantes de Compra\nRevisa documentos Parsed/Modified y envia los seleccionados.")
380
+
381
+ status = gr.Markdown(initial_status)
382
+ results_state = gr.State(initial_rows)
383
 
384
  with gr.Row():
385
+ refresh_button = gr.Button("Recargar pendientes", variant="primary")
386
+ clear_button = gr.Button("Limpiar grilla")
387
+ send_row_button = gr.Button("Enviar", variant="secondary")
388
+ delete_row_button = gr.Button("Eliminar seleccionados", variant="stop")
389
+
390
+ with gr.Accordion("Orquestador API", open=False):
391
+ gr.Markdown("Endpoint publico para ejecutar escaneo + parseo + persistencia del orquestador.")
392
+ orchestrator_model = gr.Textbox(
393
+ label="Modelo OpenAI",
394
+ value=os.getenv("OPENAI_MODEL", "gpt-4o"),
395
+ )
396
+ orchestrator_subfolders = gr.Checkbox(
397
+ label="Incluir subcarpetas",
398
+ value=True,
399
+ )
400
+ orchestrator_send = gr.Checkbox(
401
+ label="Enviar a Google Sheets",
402
+ value=False,
403
+ )
404
+ orchestrator_api_key = gr.Textbox(
405
+ label="API Key",
406
+ type="password",
407
+ value="",
408
+ )
409
+ orchestrator_button = gr.Button("Ejecutar orquestador", variant="primary")
410
+ orchestrator_output = gr.JSON(label="Resultado orquestacion")
411
 
412
  results_grid = gr.Dataframe(
413
+ label="Documentos procesados",
414
+ headers=GRID_HEADER_LABELS,
415
+ value=rows_to_table(initial_rows),
416
  interactive=True,
417
  wrap=True,
418
  )
419
 
420
+ refresh_button.click(
421
+ load_pending_documents,
422
+ inputs=[],
423
  outputs=[results_state, results_grid, status],
424
  )
425
 
 
431
 
432
  results_grid.change(
433
  sync_grid_to_state,
434
+ inputs=[results_state, results_grid],
435
  outputs=[results_state, status],
436
  )
437
 
 
 
 
 
 
 
438
  delete_row_button.click(
439
  delete_selected_rows,
440
  inputs=[results_state, results_grid],
 
447
  outputs=[results_state, results_grid, status],
448
  )
449
 
450
+ orchestrator_button.click(
451
+ run_orchestrator_endpoint,
452
+ inputs=[orchestrator_model, orchestrator_subfolders, orchestrator_send, orchestrator_api_key],
453
+ outputs=[orchestrator_output],
454
+ api_name="orchestrate_drive_documents",
455
+ )
456
+
457
  return demo
458
 
459
 
 
466
  # Spaces manages networking itself; local runs keep explicit host/port.
467
  if os.getenv("SPACE_ID"):
468
  try:
469
+ demo.launch(css=custom_css, ssr_mode=False, **launch_kwargs)
470
  except TypeError:
471
  # Backward-compatible fallback if this Gradio build does not accept ssr_mode.
472
+ demo.launch(css=custom_css, **launch_kwargs)
473
  else:
474
  try:
475
+ demo.launch(css=custom_css, server_name="0.0.0.0", server_port=7860, ssr_mode=False, **launch_kwargs)
476
  except TypeError:
477
+ demo.launch(css=custom_css, server_name="0.0.0.0", server_port=7860, **launch_kwargs)
document_orchester.py ADDED
@@ -0,0 +1,221 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import argparse
4
+ import json
5
+ import os
6
+ import shutil
7
+ from pathlib import Path
8
+ from typing import Any
9
+
10
+ from dotenv import load_dotenv
11
+ from openai import OpenAI
12
+
13
+ from document_processing import extract_text, is_image_document, to_data_uri
14
+ from firebase_processed_files import (
15
+ build_firebase_tracker,
16
+ check_drive_documents_to_process,
17
+ )
18
+ from google_drive_scanner import (
19
+ GoogleDriveConfigError,
20
+ scan_google_drive_supported_documents,
21
+ )
22
+ from google_sheets_service import GoogleSheetsConfigError, append_row_to_google_sheet
23
+ from receipt_ai import extract_receipt_json, extract_receipt_json_from_image
24
+ from receipt_results import build_empty_result
25
+
26
+
27
+ load_dotenv()
28
+
29
+
30
+ SHEET_COLUMNS = ["fecha", "source_file", "display_description", "total"]
31
+
32
+
33
+ def _build_display_description(parsed: dict[str, Any]) -> str:
34
+ provider_name = str(parsed.get("description_proveedor") or "").strip()
35
+ provider_cuit = str(parsed.get("cuit_proveedor") or "").strip()
36
+ city = str(parsed.get("ciudad") or parsed.get("city") or "").strip()
37
+ parts = [part for part in [provider_name, provider_cuit, city] if part]
38
+ return " - ".join(parts)
39
+
40
+
41
+ def _build_sheet_row(parsed: dict[str, Any]) -> dict[str, Any]:
42
+ return {
43
+ "fecha": parsed.get("fecha"),
44
+ "source_file": parsed.get("source_file"),
45
+ "display_description": _build_display_description(parsed),
46
+ "total": parsed.get("total"),
47
+ }
48
+
49
+
50
+ def _create_openai_client() -> OpenAI:
51
+ api_key = (os.getenv("OPENAI_API_KEY") or "").strip()
52
+ if not api_key:
53
+ raise RuntimeError("Missing OPENAI_API_KEY.")
54
+ return OpenAI(api_key=api_key)
55
+
56
+
57
+ def _parse_document(client: OpenAI, model: str, local_path: str, source_file: str) -> dict[str, Any]:
58
+ path = Path(local_path)
59
+ if is_image_document(path):
60
+ parsed = extract_receipt_json_from_image(client, model, path, to_data_uri(path))
61
+ else:
62
+ text = extract_text(path)
63
+ if not text.strip():
64
+ parsed = build_empty_result(source_file)
65
+ else:
66
+ parsed = extract_receipt_json(client, model, path, text)
67
+
68
+ parsed["source_file"] = source_file
69
+ return parsed
70
+
71
+
72
+ def orchestrate_drive_documents(
73
+ model: str,
74
+ include_subfolders: bool = True,
75
+ send_to_sheet: bool = False,
76
+ ) -> dict[str, Any]:
77
+ tracker = build_firebase_tracker()
78
+ client = _create_openai_client()
79
+
80
+ scan_temp_dir: str | None = None
81
+ try:
82
+ drive_documents, scan_temp_dir = scan_google_drive_supported_documents(include_subfolders=include_subfolders)
83
+ except GoogleDriveConfigError as exc:
84
+ return {
85
+ "ok": False,
86
+ "error": f"Google Drive configuration error: {exc}",
87
+ }
88
+
89
+ docs_payload = [
90
+ {
91
+ "document_id": doc.document_id,
92
+ "source_file": doc.source_file,
93
+ "modificationDate": doc.modificationDate,
94
+ "local_path": doc.local_path,
95
+ }
96
+ for doc in drive_documents
97
+ ]
98
+
99
+ to_process, skipped, tracking_warning = check_drive_documents_to_process(tracker, docs_payload)
100
+
101
+ parsed_count = 0
102
+ modified_count = 0
103
+ sent_count = 0
104
+ error_items: list[dict[str, str]] = []
105
+ processed_items: list[dict[str, Any]] = []
106
+
107
+ for document in to_process:
108
+ document_id = str(document.get("document_id") or "").strip()
109
+ source_file = str(document.get("source_file") or "").strip()
110
+ local_path = str(document.get("local_path") or "").strip()
111
+ modification_date = document.get("modificationDate")
112
+ status = str(document.get("target_status") or "Parsed")
113
+
114
+ try:
115
+ parsed = _parse_document(client, model, local_path, source_file)
116
+
117
+ if tracker is not None:
118
+ tracker.save_document_record(
119
+ document_id=document_id,
120
+ source_file=source_file,
121
+ modification_date=modification_date,
122
+ status=status,
123
+ parsed_data=parsed,
124
+ )
125
+
126
+ if status == "Modified":
127
+ modified_count += 1
128
+ else:
129
+ parsed_count += 1
130
+
131
+ if send_to_sheet:
132
+ append_row_to_google_sheet(_build_sheet_row(parsed), SHEET_COLUMNS)
133
+ if tracker is not None:
134
+ tracker.mark_document_sent(document_id)
135
+ sent_count += 1
136
+
137
+ processed_items.append(
138
+ {
139
+ "document_id": document_id,
140
+ "source_file": source_file,
141
+ "modificationDate": modification_date,
142
+ "status": "Sent" if send_to_sheet else status,
143
+ }
144
+ )
145
+ except GoogleSheetsConfigError as exc:
146
+ error_items.append(
147
+ {
148
+ "document_id": document_id,
149
+ "source_file": source_file,
150
+ "error": f"Google Sheets config error: {exc}",
151
+ }
152
+ )
153
+ except Exception as exc:
154
+ error_items.append(
155
+ {
156
+ "document_id": document_id,
157
+ "source_file": source_file,
158
+ "error": str(exc),
159
+ }
160
+ )
161
+
162
+ if scan_temp_dir:
163
+ shutil.rmtree(scan_temp_dir, ignore_errors=True)
164
+
165
+ return {
166
+ "ok": True,
167
+ "scanned": len(drive_documents),
168
+ "to_process": len(to_process),
169
+ "skipped": len(skipped),
170
+ "parsed": parsed_count,
171
+ "modified": modified_count,
172
+ "sent": sent_count,
173
+ "tracking_warning": tracking_warning,
174
+ "processed_items": processed_items,
175
+ "skipped_items": skipped,
176
+ "errors": error_items,
177
+ }
178
+
179
+
180
+ def parse_args() -> argparse.Namespace:
181
+ parser = argparse.ArgumentParser(description="Orchestrate Drive documents: scan, parse, and persist status.")
182
+ parser.add_argument("--model", default=os.getenv("OPENAI_MODEL", "gpt-4o"), help="OpenAI model name.")
183
+ parser.add_argument(
184
+ "--no-subfolders",
185
+ action="store_true",
186
+ help="Only scan direct files in configured Drive folders.",
187
+ )
188
+ parser.add_argument(
189
+ "--send",
190
+ action="store_true",
191
+ help="After parsing, append each row to Google Sheets and mark status as Sent.",
192
+ )
193
+ parser.add_argument(
194
+ "--output",
195
+ default="",
196
+ help="Optional output JSON path for the orchestration summary.",
197
+ )
198
+ return parser.parse_args()
199
+
200
+
201
+ def main() -> int:
202
+ args = parse_args()
203
+ summary = orchestrate_drive_documents(
204
+ model=args.model,
205
+ include_subfolders=not args.no_subfolders,
206
+ send_to_sheet=args.send,
207
+ )
208
+
209
+ rendered = json.dumps(summary, indent=2, ensure_ascii=False)
210
+ if args.output:
211
+ output_path = Path(args.output)
212
+ output_path.parent.mkdir(parents=True, exist_ok=True)
213
+ output_path.write_text(rendered, encoding="utf-8")
214
+ else:
215
+ print(rendered)
216
+
217
+ return 0 if summary.get("ok") else 1
218
+
219
+
220
+ if __name__ == "__main__":
221
+ raise SystemExit(main())
firebase_processed_files.py ADDED
@@ -0,0 +1,229 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import hashlib
4
+ import json
5
+ import os
6
+ from datetime import datetime, timezone
7
+ from pathlib import Path
8
+ from typing import Any
9
+
10
+ from dotenv import load_dotenv
11
+
12
+
13
+ load_dotenv()
14
+
15
+
16
+ class FirebaseConfigError(RuntimeError):
17
+ """Raised when Firebase configuration is missing or invalid."""
18
+
19
+
20
+ class FirebaseProcessedFilesTracker:
21
+ def __init__(self, collection_name: str | None = None):
22
+ self.collection_name = (collection_name or os.getenv("FIREBASE_PROCESSED_COLLECTION") or "processed_files").strip()
23
+ self.documents_collection_name = (
24
+ os.getenv("FIREBASE_DOCUMENTS_COLLECTION") or "processed_documents"
25
+ ).strip()
26
+ self._db = None
27
+
28
+ def _build_credentials(self):
29
+ from firebase_admin import credentials
30
+
31
+ service_account_json = (os.getenv("FIREBASE_SERVICE_ACCOUNT_JSON") or "").strip()
32
+ service_account_file = (os.getenv("FIREBASE_SERVICE_ACCOUNT_FILE") or "").strip()
33
+
34
+ # Reuse Google service account settings when dedicated Firebase keys are not set.
35
+ if not service_account_json:
36
+ service_account_json = (os.getenv("GOOGLE_SERVICE_ACCOUNT_JSON") or "").strip()
37
+ if not service_account_file:
38
+ service_account_file = (os.getenv("GOOGLE_SERVICE_ACCOUNT_FILE") or "").strip()
39
+
40
+ if service_account_json:
41
+ try:
42
+ info = json.loads(service_account_json)
43
+ except json.JSONDecodeError as exc:
44
+ raise FirebaseConfigError("FIREBASE_SERVICE_ACCOUNT_JSON is not valid JSON.") from exc
45
+ return credentials.Certificate(info)
46
+
47
+ if service_account_file:
48
+ return credentials.Certificate(service_account_file)
49
+
50
+ raise FirebaseConfigError(
51
+ "Missing Firebase credentials. Set FIREBASE_SERVICE_ACCOUNT_JSON or FIREBASE_SERVICE_ACCOUNT_FILE "
52
+ "(or reuse GOOGLE_SERVICE_ACCOUNT_JSON/GOOGLE_SERVICE_ACCOUNT_FILE)."
53
+ )
54
+
55
+ def _get_db(self):
56
+ if self._db is not None:
57
+ return self._db
58
+
59
+ import firebase_admin
60
+ from firebase_admin import firestore
61
+
62
+ if not firebase_admin._apps:
63
+ cred = self._build_credentials()
64
+ firebase_admin.initialize_app(cred)
65
+
66
+ self._db = firestore.client()
67
+ return self._db
68
+
69
+ def compute_file_hash(self, file_path: Path) -> str:
70
+ sha256 = hashlib.sha256()
71
+ with file_path.open("rb") as stream:
72
+ for chunk in iter(lambda: stream.read(1024 * 1024), b""):
73
+ sha256.update(chunk)
74
+ return sha256.hexdigest()
75
+
76
+ def is_processed(self, file_hash: str) -> bool:
77
+ doc = self._get_db().collection(self.collection_name).document(file_hash).get()
78
+ return bool(doc.exists)
79
+
80
+ def mark_processed(self, file_hash: str, source_file: str) -> None:
81
+ payload = {
82
+ "file_hash": file_hash,
83
+ "source_file": source_file,
84
+ "processed_at": datetime.now(timezone.utc).isoformat(),
85
+ }
86
+ self._get_db().collection(self.collection_name).document(file_hash).set(payload)
87
+
88
+ def get_document_record(self, document_id: str) -> dict[str, Any] | None:
89
+ doc = self._get_db().collection(self.documents_collection_name).document(document_id).get()
90
+ if not doc.exists:
91
+ return None
92
+ return doc.to_dict() or {}
93
+
94
+ def save_document_record(
95
+ self,
96
+ document_id: str,
97
+ source_file: str,
98
+ modification_date: str | None,
99
+ status: str,
100
+ parsed_data: dict[str, Any] | None,
101
+ ) -> None:
102
+ now = datetime.now(timezone.utc).isoformat()
103
+ payload: dict[str, Any] = {
104
+ "document_id": document_id,
105
+ "source_file": source_file,
106
+ "modificationDate": modification_date,
107
+ "status": status,
108
+ "updated_at": now,
109
+ }
110
+
111
+ if parsed_data is not None:
112
+ payload["parsed_data"] = parsed_data
113
+
114
+ existing = self.get_document_record(document_id)
115
+ if existing is None:
116
+ payload["created_at"] = now
117
+
118
+ self._get_db().collection(self.documents_collection_name).document(document_id).set(payload, merge=True)
119
+
120
+ def mark_document_sent(self, document_id: str) -> None:
121
+ now = datetime.now(timezone.utc).isoformat()
122
+ payload = {
123
+ "status": "Sent",
124
+ "sent_at": now,
125
+ "updated_at": now,
126
+ }
127
+ self._get_db().collection(self.documents_collection_name).document(document_id).set(payload, merge=True)
128
+
129
+ def list_documents_by_statuses(self, statuses: list[str]) -> list[dict[str, Any]]:
130
+ unique_statuses = [status for status in dict.fromkeys(statuses) if str(status).strip()]
131
+ if not unique_statuses:
132
+ return []
133
+
134
+ collection = self._get_db().collection(self.documents_collection_name)
135
+ documents: list[dict[str, Any]] = []
136
+
137
+ try:
138
+ query = collection.where("status", "in", unique_statuses)
139
+ for doc in query.stream():
140
+ payload = doc.to_dict() or {}
141
+ payload.setdefault("document_id", doc.id)
142
+ documents.append(payload)
143
+ return documents
144
+ except Exception:
145
+ # Fallback for environments where the 'in' operator is unavailable or restricted.
146
+ for doc in collection.stream():
147
+ payload = doc.to_dict() or {}
148
+ status = str(payload.get("status") or "")
149
+ if status in unique_statuses:
150
+ payload.setdefault("document_id", doc.id)
151
+ documents.append(payload)
152
+
153
+ return documents
154
+
155
+
156
+ def build_firebase_tracker() -> FirebaseProcessedFilesTracker | None:
157
+ enabled = (os.getenv("FIREBASE_TRACK_PROCESSED") or "true").strip().lower()
158
+ if enabled in {"0", "false", "no", "off"}:
159
+ return None
160
+
161
+ return FirebaseProcessedFilesTracker()
162
+
163
+
164
+ def check_files_already_processed(
165
+ tracker: FirebaseProcessedFilesTracker | None, file_paths: list[str]
166
+ ) -> tuple[list[str], list[tuple[str, str]], str | None]:
167
+ if tracker is None:
168
+ return file_paths, [], None
169
+
170
+ new_files: list[str] = []
171
+ skipped_files: list[tuple[str, str]] = []
172
+
173
+ try:
174
+ for file_path in file_paths:
175
+ path = Path(file_path)
176
+ file_hash = tracker.compute_file_hash(path)
177
+ if tracker.is_processed(file_hash):
178
+ skipped_files.append((path.name, file_hash))
179
+ else:
180
+ new_files.append(file_path)
181
+ except Exception as exc:
182
+ return file_paths, [], f"Firebase tracking unavailable. Processing all files. Details: {exc}"
183
+
184
+ return new_files, skipped_files, None
185
+
186
+
187
+ def check_drive_documents_to_process(
188
+ tracker: FirebaseProcessedFilesTracker | None,
189
+ documents: list[dict[str, Any]],
190
+ ) -> tuple[list[dict[str, Any]], list[dict[str, Any]], str | None]:
191
+ """
192
+ Returns (documents_to_process, skipped_documents, warning).
193
+
194
+ - New document (no record) -> status "Parsed"
195
+ - Existing document with modified date changed -> status "Modified"
196
+ - Existing document with same modified date -> skipped
197
+ """
198
+ if tracker is None:
199
+ docs = [dict(doc, target_status="Parsed") for doc in documents]
200
+ return docs, [], None
201
+
202
+ to_process: list[dict[str, Any]] = []
203
+ skipped: list[dict[str, Any]] = []
204
+
205
+ try:
206
+ for document in documents:
207
+ document_id = str(document.get("document_id") or "").strip()
208
+ modified_time = str(document.get("modificationDate") or "").strip() or None
209
+
210
+ if not document_id:
211
+ skipped.append({**document, "skip_reason": "missing_document_id"})
212
+ continue
213
+
214
+ existing = tracker.get_document_record(document_id)
215
+ if existing is None:
216
+ to_process.append({**document, "target_status": "Parsed"})
217
+ continue
218
+
219
+ existing_modified = str(existing.get("modificationDate") or "").strip() or None
220
+ if existing_modified != modified_time:
221
+ to_process.append({**document, "target_status": "Modified"})
222
+ else:
223
+ skipped.append({**document, "skip_reason": "already_processed_same_modification"})
224
+ except Exception as exc:
225
+ docs = [dict(doc, target_status="Parsed") for doc in documents]
226
+ return docs, [], f"Firebase tracking unavailable. Processing all Drive files. Details: {exc}"
227
+
228
+ return to_process, skipped, None
229
+
google_drive_scanner.py ADDED
@@ -0,0 +1,297 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import io
4
+ import json
5
+ import os
6
+ import re
7
+ import shutil
8
+ import tempfile
9
+ from dataclasses import dataclass
10
+ from pathlib import Path
11
+ from typing import Any
12
+
13
+ from dotenv import load_dotenv
14
+ from google.oauth2.service_account import Credentials
15
+
16
+ from document_processing import SUPPORTED_EXTENSIONS
17
+ from firebase_processed_files import FirebaseProcessedFilesTracker, check_files_already_processed
18
+
19
+
20
+ load_dotenv()
21
+
22
+
23
+ DRIVE_READONLY_SCOPE = "https://www.googleapis.com/auth/drive.readonly"
24
+ FOLDER_MIME_TYPE = "application/vnd.google-apps.folder"
25
+
26
+ # Google Workspace files require export, while binary files can be downloaded directly.
27
+ SUPPORTED_EXPORT_MIME_MAP = {
28
+ "application/vnd.google-apps.document": ("application/vnd.openxmlformats-officedocument.wordprocessingml.document", ".docx"),
29
+ }
30
+
31
+ SUPPORTED_BINARY_MIME_TO_SUFFIX = {
32
+ "application/pdf": ".pdf",
33
+ "text/plain": ".txt",
34
+ "application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx",
35
+ "image/jpeg": ".jpg",
36
+ "image/png": ".png",
37
+ }
38
+
39
+
40
+ class GoogleDriveConfigError(RuntimeError):
41
+ """Raised when required Google Drive environment configuration is missing or invalid."""
42
+
43
+
44
+ @dataclass
45
+ class DriveScanResult:
46
+ files_to_process: list[str]
47
+ skipped_files: list[tuple[str, str]]
48
+ discovered_count: int
49
+ temp_dir: str
50
+ warning: str | None = None
51
+
52
+
53
+ @dataclass
54
+ class DriveDocument:
55
+ document_id: str
56
+ source_file: str
57
+ mime_type: str
58
+ modificationDate: str | None
59
+ local_path: str
60
+
61
+
62
+ def _build_credentials() -> Credentials:
63
+ service_account_json = (os.getenv("GOOGLE_SERVICE_ACCOUNT_JSON") or "").strip()
64
+ service_account_file = (os.getenv("GOOGLE_SERVICE_ACCOUNT_FILE") or "").strip()
65
+
66
+ if service_account_json:
67
+ try:
68
+ info = json.loads(service_account_json)
69
+ except json.JSONDecodeError as exc:
70
+ raise GoogleDriveConfigError("GOOGLE_SERVICE_ACCOUNT_JSON is not valid JSON.") from exc
71
+ return Credentials.from_service_account_info(info, scopes=[DRIVE_READONLY_SCOPE])
72
+
73
+ if service_account_file:
74
+ return Credentials.from_service_account_file(service_account_file, scopes=[DRIVE_READONLY_SCOPE])
75
+
76
+ raise GoogleDriveConfigError(
77
+ "Missing Google credentials. Set GOOGLE_SERVICE_ACCOUNT_JSON or GOOGLE_SERVICE_ACCOUNT_FILE."
78
+ )
79
+
80
+
81
+ def _build_drive_service():
82
+ try:
83
+ from googleapiclient.discovery import build
84
+ except ImportError as exc:
85
+ raise GoogleDriveConfigError(
86
+ "google-api-python-client is required for Drive scanning. Add it to requirements and install dependencies."
87
+ ) from exc
88
+
89
+ credentials = _build_credentials()
90
+ return build("drive", "v3", credentials=credentials, cache_discovery=False)
91
+
92
+
93
+ def _load_folder_ids_from_env() -> list[str]:
94
+ raw = (os.getenv("GOOGLE_DRIVE_FOLDER_IDS") or "").strip()
95
+ if not raw:
96
+ raise GoogleDriveConfigError("Missing GOOGLE_DRIVE_FOLDER_IDS. Set one or more folder IDs separated by commas.")
97
+
98
+ folder_ids = [item.strip() for item in raw.split(",") if item.strip()]
99
+ if not folder_ids:
100
+ raise GoogleDriveConfigError("GOOGLE_DRIVE_FOLDER_IDS is empty after parsing.")
101
+
102
+ return folder_ids
103
+
104
+
105
+ def _sanitize_file_name(name: str) -> str:
106
+ cleaned = re.sub(r'[<>:"/\\|?*\x00-\x1F]', "_", name).strip().rstrip(".")
107
+ return cleaned or "document"
108
+
109
+
110
+ def _ensure_unique_destination(path: Path) -> Path:
111
+ if not path.exists():
112
+ return path
113
+
114
+ counter = 1
115
+ while True:
116
+ candidate = path.with_name(f"{path.stem}_{counter}{path.suffix}")
117
+ if not candidate.exists():
118
+ return candidate
119
+ counter += 1
120
+
121
+
122
+ def _list_files_from_folder(service, folder_id: str, include_subfolders: bool) -> list[dict[str, Any]]:
123
+ folders_to_scan = [folder_id]
124
+ discovered: list[dict[str, Any]] = []
125
+
126
+ while folders_to_scan:
127
+ current_folder = folders_to_scan.pop(0)
128
+ page_token = None
129
+
130
+ while True:
131
+ response = (
132
+ service.files()
133
+ .list(
134
+ q=f"'{current_folder}' in parents and trashed = false",
135
+ fields="nextPageToken, files(id, name, mimeType, modifiedTime)",
136
+ supportsAllDrives=True,
137
+ includeItemsFromAllDrives=True,
138
+ pageSize=1000,
139
+ pageToken=page_token,
140
+ )
141
+ .execute()
142
+ )
143
+
144
+ for file_meta in response.get("files", []):
145
+ mime_type = file_meta.get("mimeType")
146
+ if mime_type == FOLDER_MIME_TYPE:
147
+ if include_subfolders:
148
+ child_folder_id = file_meta.get("id")
149
+ if child_folder_id:
150
+ folders_to_scan.append(child_folder_id)
151
+ continue
152
+
153
+ discovered.append(file_meta)
154
+
155
+ page_token = response.get("nextPageToken")
156
+ if not page_token:
157
+ break
158
+
159
+ return discovered
160
+
161
+
162
+ def _is_supported_drive_file(file_meta: dict[str, Any]) -> bool:
163
+ mime_type = (file_meta.get("mimeType") or "").strip().lower()
164
+ if mime_type in SUPPORTED_BINARY_MIME_TO_SUFFIX:
165
+ return True
166
+ if mime_type in SUPPORTED_EXPORT_MIME_MAP:
167
+ return True
168
+
169
+ name = (file_meta.get("name") or "").strip()
170
+ suffix = Path(name).suffix.lower()
171
+ return suffix in SUPPORTED_EXTENSIONS
172
+
173
+
174
+ def _download_drive_file(service, file_meta: dict[str, Any], destination_dir: Path) -> str:
175
+ try:
176
+ from googleapiclient.http import MediaIoBaseDownload
177
+ except ImportError as exc:
178
+ raise GoogleDriveConfigError(
179
+ "google-api-python-client is required for Drive scanning. Add it to requirements and install dependencies."
180
+ ) from exc
181
+
182
+ file_id = (file_meta.get("id") or "").strip()
183
+ file_name = _sanitize_file_name((file_meta.get("name") or "document").strip())
184
+ mime_type = (file_meta.get("mimeType") or "").strip().lower()
185
+
186
+ suffix = Path(file_name).suffix.lower()
187
+ request = None
188
+
189
+ if mime_type in SUPPORTED_EXPORT_MIME_MAP:
190
+ export_mime, export_suffix = SUPPORTED_EXPORT_MIME_MAP[mime_type]
191
+ if suffix != export_suffix:
192
+ file_name = f"{Path(file_name).stem}{export_suffix}"
193
+ suffix = export_suffix
194
+ request = service.files().export_media(fileId=file_id, mimeType=export_mime)
195
+ else:
196
+ if not suffix and mime_type in SUPPORTED_BINARY_MIME_TO_SUFFIX:
197
+ file_name = f"{file_name}{SUPPORTED_BINARY_MIME_TO_SUFFIX[mime_type]}"
198
+ suffix = Path(file_name).suffix.lower()
199
+ request = service.files().get_media(fileId=file_id)
200
+
201
+ if suffix not in SUPPORTED_EXTENSIONS:
202
+ raise ValueError(f"Unsupported downloaded file type: {suffix or '[no extension]'}")
203
+
204
+ destination = _ensure_unique_destination(destination_dir / file_name)
205
+
206
+ with destination.open("wb") as stream:
207
+ downloader = MediaIoBaseDownload(stream, request)
208
+ done = False
209
+ while not done:
210
+ _, done = downloader.next_chunk()
211
+
212
+ return str(destination)
213
+
214
+
215
+ def scan_google_drive_unprocessed_files(
216
+ tracker: FirebaseProcessedFilesTracker | None,
217
+ include_subfolders: bool = True,
218
+ ) -> DriveScanResult:
219
+ folder_ids = _load_folder_ids_from_env()
220
+ service = _build_drive_service()
221
+
222
+ discovered_files: list[dict[str, Any]] = []
223
+ for folder_id in folder_ids:
224
+ discovered_files.extend(_list_files_from_folder(service, folder_id, include_subfolders))
225
+
226
+ supported_files = [file_meta for file_meta in discovered_files if _is_supported_drive_file(file_meta)]
227
+ temp_dir = tempfile.mkdtemp(prefix="drive_docs_")
228
+ destination_dir = Path(temp_dir)
229
+
230
+ downloaded_paths: list[str] = []
231
+ for file_meta in supported_files:
232
+ try:
233
+ downloaded_paths.append(_download_drive_file(service, file_meta, destination_dir))
234
+ except Exception:
235
+ # Keep scan resilient if one file is malformed or inaccessible.
236
+ continue
237
+
238
+ files_to_process, skipped_files, warning = check_files_already_processed(tracker, downloaded_paths)
239
+
240
+ skipped_set = set(path for path, _ in skipped_files)
241
+ for downloaded in downloaded_paths:
242
+ path = Path(downloaded)
243
+ if path.name in skipped_set:
244
+ try:
245
+ path.unlink(missing_ok=True)
246
+ except Exception:
247
+ pass
248
+
249
+ return DriveScanResult(
250
+ files_to_process=files_to_process,
251
+ skipped_files=skipped_files,
252
+ discovered_count=len(supported_files),
253
+ temp_dir=temp_dir,
254
+ warning=warning,
255
+ )
256
+
257
+
258
+ def scan_google_drive_supported_documents(include_subfolders: bool = True) -> tuple[list[DriveDocument], str]:
259
+ """Scans configured Drive folders and downloads supported documents with metadata."""
260
+ folder_ids = _load_folder_ids_from_env()
261
+ service = _build_drive_service()
262
+
263
+ discovered_files: list[dict[str, Any]] = []
264
+ for folder_id in folder_ids:
265
+ discovered_files.extend(_list_files_from_folder(service, folder_id, include_subfolders))
266
+
267
+ supported_files = [file_meta for file_meta in discovered_files if _is_supported_drive_file(file_meta)]
268
+ temp_dir = tempfile.mkdtemp(prefix="drive_docs_")
269
+ destination_dir = Path(temp_dir)
270
+
271
+ documents: list[DriveDocument] = []
272
+ for file_meta in supported_files:
273
+ try:
274
+ local_path = _download_drive_file(service, file_meta, destination_dir)
275
+ except Exception:
276
+ # Keep scan resilient if one file is malformed or inaccessible.
277
+ continue
278
+
279
+ documents.append(
280
+ DriveDocument(
281
+ document_id=str(file_meta.get("id") or "").strip(),
282
+ source_file=str(file_meta.get("name") or "").strip(),
283
+ mime_type=str(file_meta.get("mimeType") or "").strip(),
284
+ modificationDate=str(file_meta.get("modifiedTime") or "").strip() or None,
285
+ local_path=local_path,
286
+ )
287
+ )
288
+
289
+ return documents, temp_dir
290
+
291
+
292
+ def cleanup_drive_scan_result(scan_result: DriveScanResult | None) -> None:
293
+ if scan_result is None:
294
+ return
295
+
296
+ if scan_result.temp_dir:
297
+ shutil.rmtree(scan_result.temp_dir, ignore_errors=True)
google_sheets_service.py CHANGED
@@ -57,9 +57,27 @@ def _normalize_cell_value(value: Any) -> Any:
57
  if value is None:
58
  return ""
59
 
 
 
 
 
 
 
 
 
 
 
 
60
  if isinstance(value, float) and not math.isfinite(value):
61
  return ""
62
 
 
 
 
 
 
 
 
63
  return value
64
 
65
 
 
57
  if value is None:
58
  return ""
59
 
60
+ # Pandas uses a dedicated NA sentinel that is not JSON serializable.
61
+ if type(value).__name__ == "NAType":
62
+ return ""
63
+
64
+ # Convert numpy scalar wrappers into plain Python values when possible.
65
+ if hasattr(value, "item"):
66
+ try:
67
+ value = value.item()
68
+ except Exception:
69
+ pass
70
+
71
  if isinstance(value, float) and not math.isfinite(value):
72
  return ""
73
 
74
+ # Fallback for NaN-like values that are not plain floats.
75
+ try:
76
+ if value != value:
77
+ return ""
78
+ except Exception:
79
+ pass
80
+
81
  return value
82
 
83
 
receipt_parser.py CHANGED
@@ -7,6 +7,7 @@ from pathlib import Path
7
  from typing import Any
8
 
9
  from document_processing import extract_text, is_image_document, load_documents, to_data_uri
 
10
  from receipt_ai import extract_receipt_json, extract_receipt_json_from_image
11
  from receipt_results import build_empty_result
12
 
@@ -30,9 +31,24 @@ def main() -> int:
30
  input_folder = Path(args.input)
31
  files = load_documents(input_folder)
32
  results: list[dict[str, Any]] = []
 
 
 
33
 
34
  for file_path in files:
35
  relative_name = file_path.relative_to(input_folder).as_posix()
 
 
 
 
 
 
 
 
 
 
 
 
36
  if is_image_document(file_path):
37
  image_data_uri = to_data_uri(file_path)
38
  result = extract_receipt_json_from_image(client, args.model, file_path, image_data_uri)
@@ -47,7 +63,18 @@ def main() -> int:
47
  result["source_file"] = relative_name
48
  results.append(result)
49
 
50
- payload = {"input_folder": str(input_folder), "results": results}
 
 
 
 
 
 
 
 
 
 
 
51
  rendered = json.dumps(payload, indent=2, ensure_ascii=False)
52
 
53
  if args.output:
 
7
  from typing import Any
8
 
9
  from document_processing import extract_text, is_image_document, load_documents, to_data_uri
10
+ from firebase_processed_files import build_firebase_tracker
11
  from receipt_ai import extract_receipt_json, extract_receipt_json_from_image
12
  from receipt_results import build_empty_result
13
 
 
31
  input_folder = Path(args.input)
32
  files = load_documents(input_folder)
33
  results: list[dict[str, Any]] = []
34
+ skipped_files: list[str] = []
35
+ firebase_warning: str | None = None
36
+ firebase_tracker = build_firebase_tracker()
37
 
38
  for file_path in files:
39
  relative_name = file_path.relative_to(input_folder).as_posix()
40
+ file_hash: str | None = None
41
+
42
+ if firebase_tracker is not None:
43
+ try:
44
+ file_hash = firebase_tracker.compute_file_hash(file_path)
45
+ if firebase_tracker.is_processed(file_hash):
46
+ skipped_files.append(relative_name)
47
+ continue
48
+ except Exception as exc:
49
+ firebase_warning = f"Firebase tracking unavailable. Processing remaining files without deduplication. Details: {exc}"
50
+ firebase_tracker = None
51
+
52
  if is_image_document(file_path):
53
  image_data_uri = to_data_uri(file_path)
54
  result = extract_receipt_json_from_image(client, args.model, file_path, image_data_uri)
 
63
  result["source_file"] = relative_name
64
  results.append(result)
65
 
66
+ if firebase_tracker is not None and file_hash is not None:
67
+ try:
68
+ firebase_tracker.mark_processed(file_hash, relative_name)
69
+ except Exception:
70
+ pass
71
+
72
+ payload: dict[str, Any] = {"input_folder": str(input_folder), "results": results}
73
+ if skipped_files:
74
+ payload["skipped_already_processed"] = skipped_files
75
+ if firebase_warning:
76
+ payload["firebase_warning"] = firebase_warning
77
+
78
  rendered = json.dumps(payload, indent=2, ensure_ascii=False)
79
 
80
  if args.output:
requirements.txt CHANGED
@@ -5,4 +5,6 @@ python-dotenv>=1.0.1
5
  gradio>=4.0.0
6
  spaces>=0.31.1
7
  gspread>=6.0.0
8
- google-auth>=2.30.0
 
 
 
5
  gradio>=4.0.0
6
  spaces>=0.31.1
7
  gspread>=6.0.0
8
+ google-auth>=2.30.0
9
+ firebase-admin>=6.5.0
10
+ google-api-python-client>=2.140.0