segovia commited on
Commit
207d271
·
1 Parent(s): db9f03c

log information

Browse files
Files changed (3) hide show
  1. README.md +2 -0
  2. document_orchester.py +70 -4
  3. google_sheets_service.py +17 -5
README.md CHANGED
@@ -91,3 +91,5 @@ curl -X POST "http://localhost:7860/gradio_api/call/orchestrate_drive_documents"
91
  4. `api_key` (string)
92
 
93
  Response payload is the same summary produced by `orchestrate_drive_documents` in `document_orchester.py`.
 
 
 
91
  4. `api_key` (string)
92
 
93
  Response payload is the same summary produced by `orchestrate_drive_documents` in `document_orchester.py`.
94
+
95
+ When a row is missing required sheet data, it is written to a worksheet named `corrupted data` instead of the main sheet.
document_orchester.py CHANGED
@@ -2,6 +2,7 @@ from __future__ import annotations
2
 
3
  import argparse
4
  import json
 
5
  import os
6
  import shutil
7
  from pathlib import Path
@@ -27,7 +28,17 @@ from receipt_results import build_empty_result
27
  load_dotenv()
28
 
29
 
 
 
 
 
 
 
 
 
 
30
  SHEET_COLUMNS = ["fecha", "source_file", "display_description", "total"]
 
31
 
32
 
33
  def _build_display_description(parsed: dict[str, Any]) -> str:
@@ -47,6 +58,20 @@ def _build_sheet_row(parsed: dict[str, Any]) -> dict[str, Any]:
47
  }
48
 
49
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
50
  def _create_openai_client() -> OpenAI:
51
  api_key = (os.getenv("OPENAI_API_KEY") or "").strip()
52
  if not api_key:
@@ -74,6 +99,12 @@ def orchestrate_drive_documents(
74
  include_subfolders: bool = True,
75
  send_to_sheet: bool = False,
76
  ) -> dict[str, Any]:
 
 
 
 
 
 
77
  tracker = build_firebase_tracker()
78
  client = _create_openai_client()
79
 
@@ -81,11 +112,14 @@ def orchestrate_drive_documents(
81
  try:
82
  drive_documents, scan_temp_dir = scan_google_drive_supported_documents(include_subfolders=include_subfolders)
83
  except GoogleDriveConfigError as exc:
 
84
  return {
85
  "ok": False,
86
  "error": f"Google Drive configuration error: {exc}",
87
  }
88
 
 
 
89
  docs_payload = [
90
  {
91
  "document_id": doc.document_id,
@@ -97,10 +131,14 @@ def orchestrate_drive_documents(
97
  ]
98
 
99
  to_process, skipped, tracking_warning = check_drive_documents_to_process(tracker, docs_payload)
 
 
 
100
 
101
  parsed_count = 0
102
  modified_count = 0
103
  sent_count = 0
 
104
  error_items: list[dict[str, str]] = []
105
  processed_items: list[dict[str, Any]] = []
106
 
@@ -110,9 +148,11 @@ def orchestrate_drive_documents(
110
  local_path = str(document.get("local_path") or "").strip()
111
  modification_date = document.get("modificationDate")
112
  status = str(document.get("target_status") or "Parsed")
 
113
 
114
  try:
115
  parsed = _parse_document(client, model, local_path, source_file)
 
116
 
117
  if tracker is not None:
118
  tracker.save_document_record(
@@ -128,21 +168,33 @@ def orchestrate_drive_documents(
128
  else:
129
  parsed_count += 1
130
 
 
131
  if send_to_sheet:
132
- append_row_to_google_sheet(_build_sheet_row(parsed), SHEET_COLUMNS)
133
- if tracker is not None:
 
 
 
 
 
 
 
 
134
  tracker.mark_document_sent(document_id)
135
  sent_count += 1
 
 
136
 
137
  processed_items.append(
138
  {
139
  "document_id": document_id,
140
  "source_file": source_file,
141
  "modificationDate": modification_date,
142
- "status": "Sent" if send_to_sheet else status,
143
  }
144
  )
145
  except GoogleSheetsConfigError as exc:
 
146
  error_items.append(
147
  {
148
  "document_id": document_id,
@@ -151,6 +203,7 @@ def orchestrate_drive_documents(
151
  }
152
  )
153
  except Exception as exc:
 
154
  error_items.append(
155
  {
156
  "document_id": document_id,
@@ -161,6 +214,16 @@ def orchestrate_drive_documents(
161
 
162
  if scan_temp_dir:
163
  shutil.rmtree(scan_temp_dir, ignore_errors=True)
 
 
 
 
 
 
 
 
 
 
164
 
165
  return {
166
  "ok": True,
@@ -170,6 +233,7 @@ def orchestrate_drive_documents(
170
  "parsed": parsed_count,
171
  "modified": modified_count,
172
  "sent": sent_count,
 
173
  "tracking_warning": tracking_warning,
174
  "processed_items": processed_items,
175
  "skipped_items": skipped,
@@ -200,6 +264,7 @@ def parse_args() -> argparse.Namespace:
200
 
201
  def main() -> int:
202
  args = parse_args()
 
203
  summary = orchestrate_drive_documents(
204
  model=args.model,
205
  include_subfolders=not args.no_subfolders,
@@ -211,8 +276,9 @@ def main() -> int:
211
  output_path = Path(args.output)
212
  output_path.parent.mkdir(parents=True, exist_ok=True)
213
  output_path.write_text(rendered, encoding="utf-8")
 
214
  else:
215
- print(rendered)
216
 
217
  return 0 if summary.get("ok") else 1
218
 
 
2
 
3
  import argparse
4
  import json
5
+ import logging
6
  import os
7
  import shutil
8
  from pathlib import Path
 
28
  load_dotenv()
29
 
30
 
31
+ logger = logging.getLogger(__name__)
32
+ if not logger.handlers:
33
+ handler = logging.StreamHandler()
34
+ handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(name)s: %(message)s"))
35
+ logger.addHandler(handler)
36
+ logger.setLevel(logging.INFO)
37
+ logger.propagate = False
38
+
39
+
40
  SHEET_COLUMNS = ["fecha", "source_file", "display_description", "total"]
41
+ CORRUPTED_SHEET_NAME = "corrupted data"
42
 
43
 
44
  def _build_display_description(parsed: dict[str, Any]) -> str:
 
58
  }
59
 
60
 
61
+ def _is_blank_sheet_value(value: Any) -> bool:
62
+ if value is None:
63
+ return True
64
+
65
+ if isinstance(value, str):
66
+ return not value.strip()
67
+
68
+ return False
69
+
70
+
71
+ def _is_complete_sheet_row(row: dict[str, Any]) -> bool:
72
+ return all(not _is_blank_sheet_value(row.get(column)) for column in SHEET_COLUMNS)
73
+
74
+
75
  def _create_openai_client() -> OpenAI:
76
  api_key = (os.getenv("OPENAI_API_KEY") or "").strip()
77
  if not api_key:
 
99
  include_subfolders: bool = True,
100
  send_to_sheet: bool = False,
101
  ) -> dict[str, Any]:
102
+ logger.info(
103
+ "Starting orchestration model=%s include_subfolders=%s send_to_sheet=%s",
104
+ model,
105
+ include_subfolders,
106
+ send_to_sheet,
107
+ )
108
  tracker = build_firebase_tracker()
109
  client = _create_openai_client()
110
 
 
112
  try:
113
  drive_documents, scan_temp_dir = scan_google_drive_supported_documents(include_subfolders=include_subfolders)
114
  except GoogleDriveConfigError as exc:
115
+ logger.error("Google Drive configuration error: %s", exc)
116
  return {
117
  "ok": False,
118
  "error": f"Google Drive configuration error: {exc}",
119
  }
120
 
121
+ logger.info("Scan completed scanned_documents=%s", len(drive_documents))
122
+
123
  docs_payload = [
124
  {
125
  "document_id": doc.document_id,
 
131
  ]
132
 
133
  to_process, skipped, tracking_warning = check_drive_documents_to_process(tracker, docs_payload)
134
+ logger.info("Documents to process=%s skipped=%s", len(to_process), len(skipped))
135
+ if tracking_warning:
136
+ logger.warning("Tracking warning: %s", tracking_warning)
137
 
138
  parsed_count = 0
139
  modified_count = 0
140
  sent_count = 0
141
+ corrupted_count = 0
142
  error_items: list[dict[str, str]] = []
143
  processed_items: list[dict[str, Any]] = []
144
 
 
148
  local_path = str(document.get("local_path") or "").strip()
149
  modification_date = document.get("modificationDate")
150
  status = str(document.get("target_status") or "Parsed")
151
+ logger.info("Processing document_id=%s source_file=%s status=%s", document_id, source_file, status)
152
 
153
  try:
154
  parsed = _parse_document(client, model, local_path, source_file)
155
+ logger.info("Parsed document_id=%s source_file=%s", document_id, source_file)
156
 
157
  if tracker is not None:
158
  tracker.save_document_record(
 
168
  else:
169
  parsed_count += 1
170
 
171
+ worksheet_name: str | None = None
172
  if send_to_sheet:
173
+ row_for_sheet = _build_sheet_row(parsed)
174
+ worksheet_name = None if _is_complete_sheet_row(row_for_sheet) else CORRUPTED_SHEET_NAME
175
+ append_row_to_google_sheet(row_for_sheet, SHEET_COLUMNS, worksheet_name=worksheet_name)
176
+ logger.info(
177
+ "Sent document_id=%s source_file=%s worksheet=%s",
178
+ document_id,
179
+ source_file,
180
+ worksheet_name or "main",
181
+ )
182
+ if tracker is not None and worksheet_name is None:
183
  tracker.mark_document_sent(document_id)
184
  sent_count += 1
185
+ if worksheet_name is not None:
186
+ corrupted_count += 1
187
 
188
  processed_items.append(
189
  {
190
  "document_id": document_id,
191
  "source_file": source_file,
192
  "modificationDate": modification_date,
193
+ "status": "Corrupted" if worksheet_name is not None else ("Sent" if send_to_sheet else status),
194
  }
195
  )
196
  except GoogleSheetsConfigError as exc:
197
+ logger.error("Google Sheets config error for document_id=%s source_file=%s: %s", document_id, source_file, exc)
198
  error_items.append(
199
  {
200
  "document_id": document_id,
 
203
  }
204
  )
205
  except Exception as exc:
206
+ logger.exception("Failed processing document_id=%s source_file=%s", document_id, source_file)
207
  error_items.append(
208
  {
209
  "document_id": document_id,
 
214
 
215
  if scan_temp_dir:
216
  shutil.rmtree(scan_temp_dir, ignore_errors=True)
217
+ logger.info("Cleaned up temporary scan directory")
218
+
219
+ logger.info(
220
+ "Orchestration finished parsed=%s modified=%s sent=%s corrupted=%s errors=%s",
221
+ parsed_count,
222
+ modified_count,
223
+ sent_count,
224
+ corrupted_count,
225
+ len(error_items),
226
+ )
227
 
228
  return {
229
  "ok": True,
 
233
  "parsed": parsed_count,
234
  "modified": modified_count,
235
  "sent": sent_count,
236
+ "corrupted": corrupted_count,
237
  "tracking_warning": tracking_warning,
238
  "processed_items": processed_items,
239
  "skipped_items": skipped,
 
264
 
265
  def main() -> int:
266
  args = parse_args()
267
+ logger.info("Running document orchestration CLI output=%s", args.output or "stdout")
268
  summary = orchestrate_drive_documents(
269
  model=args.model,
270
  include_subfolders=not args.no_subfolders,
 
276
  output_path = Path(args.output)
277
  output_path.parent.mkdir(parents=True, exist_ok=True)
278
  output_path.write_text(rendered, encoding="utf-8")
279
+ logger.info("Wrote orchestration summary to %s", output_path)
280
  else:
281
+ logger.info("Orchestration summary:\n%s", rendered)
282
 
283
  return 0 if summary.get("ok") else 1
284
 
google_sheets_service.py CHANGED
@@ -7,6 +7,7 @@ from typing import Any
7
 
8
  import gspread
9
  from dotenv import load_dotenv
 
10
  from google.oauth2.service_account import Credentials
11
 
12
 
@@ -42,15 +43,22 @@ def _build_credentials() -> Credentials:
42
  )
43
 
44
 
45
- def _open_worksheet(client: gspread.Client):
46
  spreadsheet_id = (os.getenv("GOOGLE_SHEETS_SPREADSHEET_ID") or "").strip()
47
- worksheet_name = (os.getenv("GOOGLE_SHEETS_WORKSHEET") or "Sheet1").strip()
48
 
49
  if not spreadsheet_id:
50
  raise GoogleSheetsConfigError("Missing GOOGLE_SHEETS_SPREADSHEET_ID.")
51
 
52
  spreadsheet = client.open_by_key(spreadsheet_id)
53
- return spreadsheet.worksheet(worksheet_name)
 
 
 
 
 
 
 
54
 
55
 
56
  def _normalize_cell_value(value: Any) -> Any:
@@ -92,10 +100,14 @@ def _column_index_to_letter(index: int) -> str:
92
  return "".join(reversed(letters))
93
 
94
 
95
- def append_row_to_google_sheet(row: dict[str, Any], ordered_columns: list[str]) -> None:
 
 
 
 
96
  credentials = _build_credentials()
97
  client = gspread.authorize(credentials)
98
- worksheet = _open_worksheet(client)
99
 
100
  values = [_normalize_cell_value(row.get(column, "")) for column in ordered_columns]
101
 
 
7
 
8
  import gspread
9
  from dotenv import load_dotenv
10
+ from gspread.exceptions import WorksheetNotFound
11
  from google.oauth2.service_account import Credentials
12
 
13
 
 
43
  )
44
 
45
 
46
+ def _open_worksheet(client: gspread.Client, worksheet_name: str | None = None, column_count: int = 0):
47
  spreadsheet_id = (os.getenv("GOOGLE_SHEETS_SPREADSHEET_ID") or "").strip()
48
+ resolved_worksheet_name = (worksheet_name or os.getenv("GOOGLE_SHEETS_WORKSHEET") or "Sheet1").strip()
49
 
50
  if not spreadsheet_id:
51
  raise GoogleSheetsConfigError("Missing GOOGLE_SHEETS_SPREADSHEET_ID.")
52
 
53
  spreadsheet = client.open_by_key(spreadsheet_id)
54
+ try:
55
+ return spreadsheet.worksheet(resolved_worksheet_name)
56
+ except WorksheetNotFound:
57
+ return spreadsheet.add_worksheet(
58
+ title=resolved_worksheet_name,
59
+ rows=1000,
60
+ cols=max(column_count, 1),
61
+ )
62
 
63
 
64
  def _normalize_cell_value(value: Any) -> Any:
 
100
  return "".join(reversed(letters))
101
 
102
 
103
+ def append_row_to_google_sheet(
104
+ row: dict[str, Any],
105
+ ordered_columns: list[str],
106
+ worksheet_name: str | None = None,
107
+ ) -> None:
108
  credentials = _build_credentials()
109
  client = gspread.authorize(credentials)
110
+ worksheet = _open_worksheet(client, worksheet_name=worksheet_name, column_count=len(ordered_columns))
111
 
112
  values = [_normalize_cell_value(row.get(column, "")) for column in ordered_columns]
113