Spaces:
Sleeping
Sleeping
segovia commited on
Commit ·
207d271
1
Parent(s): db9f03c
log information
Browse files- README.md +2 -0
- document_orchester.py +70 -4
- google_sheets_service.py +17 -5
README.md
CHANGED
|
@@ -91,3 +91,5 @@ curl -X POST "http://localhost:7860/gradio_api/call/orchestrate_drive_documents"
|
|
| 91 |
4. `api_key` (string)
|
| 92 |
|
| 93 |
Response payload is the same summary produced by `orchestrate_drive_documents` in `document_orchester.py`.
|
|
|
|
|
|
|
|
|
| 91 |
4. `api_key` (string)
|
| 92 |
|
| 93 |
Response payload is the same summary produced by `orchestrate_drive_documents` in `document_orchester.py`.
|
| 94 |
+
|
| 95 |
+
When a row is missing required sheet data, it is written to a worksheet named `corrupted data` instead of the main sheet.
|
document_orchester.py
CHANGED
|
@@ -2,6 +2,7 @@ from __future__ import annotations
|
|
| 2 |
|
| 3 |
import argparse
|
| 4 |
import json
|
|
|
|
| 5 |
import os
|
| 6 |
import shutil
|
| 7 |
from pathlib import Path
|
|
@@ -27,7 +28,17 @@ from receipt_results import build_empty_result
|
|
| 27 |
load_dotenv()
|
| 28 |
|
| 29 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 30 |
SHEET_COLUMNS = ["fecha", "source_file", "display_description", "total"]
|
|
|
|
| 31 |
|
| 32 |
|
| 33 |
def _build_display_description(parsed: dict[str, Any]) -> str:
|
|
@@ -47,6 +58,20 @@ def _build_sheet_row(parsed: dict[str, Any]) -> dict[str, Any]:
|
|
| 47 |
}
|
| 48 |
|
| 49 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 50 |
def _create_openai_client() -> OpenAI:
|
| 51 |
api_key = (os.getenv("OPENAI_API_KEY") or "").strip()
|
| 52 |
if not api_key:
|
|
@@ -74,6 +99,12 @@ def orchestrate_drive_documents(
|
|
| 74 |
include_subfolders: bool = True,
|
| 75 |
send_to_sheet: bool = False,
|
| 76 |
) -> dict[str, Any]:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 77 |
tracker = build_firebase_tracker()
|
| 78 |
client = _create_openai_client()
|
| 79 |
|
|
@@ -81,11 +112,14 @@ def orchestrate_drive_documents(
|
|
| 81 |
try:
|
| 82 |
drive_documents, scan_temp_dir = scan_google_drive_supported_documents(include_subfolders=include_subfolders)
|
| 83 |
except GoogleDriveConfigError as exc:
|
|
|
|
| 84 |
return {
|
| 85 |
"ok": False,
|
| 86 |
"error": f"Google Drive configuration error: {exc}",
|
| 87 |
}
|
| 88 |
|
|
|
|
|
|
|
| 89 |
docs_payload = [
|
| 90 |
{
|
| 91 |
"document_id": doc.document_id,
|
|
@@ -97,10 +131,14 @@ def orchestrate_drive_documents(
|
|
| 97 |
]
|
| 98 |
|
| 99 |
to_process, skipped, tracking_warning = check_drive_documents_to_process(tracker, docs_payload)
|
|
|
|
|
|
|
|
|
|
| 100 |
|
| 101 |
parsed_count = 0
|
| 102 |
modified_count = 0
|
| 103 |
sent_count = 0
|
|
|
|
| 104 |
error_items: list[dict[str, str]] = []
|
| 105 |
processed_items: list[dict[str, Any]] = []
|
| 106 |
|
|
@@ -110,9 +148,11 @@ def orchestrate_drive_documents(
|
|
| 110 |
local_path = str(document.get("local_path") or "").strip()
|
| 111 |
modification_date = document.get("modificationDate")
|
| 112 |
status = str(document.get("target_status") or "Parsed")
|
|
|
|
| 113 |
|
| 114 |
try:
|
| 115 |
parsed = _parse_document(client, model, local_path, source_file)
|
|
|
|
| 116 |
|
| 117 |
if tracker is not None:
|
| 118 |
tracker.save_document_record(
|
|
@@ -128,21 +168,33 @@ def orchestrate_drive_documents(
|
|
| 128 |
else:
|
| 129 |
parsed_count += 1
|
| 130 |
|
|
|
|
| 131 |
if send_to_sheet:
|
| 132 |
-
|
| 133 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 134 |
tracker.mark_document_sent(document_id)
|
| 135 |
sent_count += 1
|
|
|
|
|
|
|
| 136 |
|
| 137 |
processed_items.append(
|
| 138 |
{
|
| 139 |
"document_id": document_id,
|
| 140 |
"source_file": source_file,
|
| 141 |
"modificationDate": modification_date,
|
| 142 |
-
"status": "Sent" if send_to_sheet else status,
|
| 143 |
}
|
| 144 |
)
|
| 145 |
except GoogleSheetsConfigError as exc:
|
|
|
|
| 146 |
error_items.append(
|
| 147 |
{
|
| 148 |
"document_id": document_id,
|
|
@@ -151,6 +203,7 @@ def orchestrate_drive_documents(
|
|
| 151 |
}
|
| 152 |
)
|
| 153 |
except Exception as exc:
|
|
|
|
| 154 |
error_items.append(
|
| 155 |
{
|
| 156 |
"document_id": document_id,
|
|
@@ -161,6 +214,16 @@ def orchestrate_drive_documents(
|
|
| 161 |
|
| 162 |
if scan_temp_dir:
|
| 163 |
shutil.rmtree(scan_temp_dir, ignore_errors=True)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 164 |
|
| 165 |
return {
|
| 166 |
"ok": True,
|
|
@@ -170,6 +233,7 @@ def orchestrate_drive_documents(
|
|
| 170 |
"parsed": parsed_count,
|
| 171 |
"modified": modified_count,
|
| 172 |
"sent": sent_count,
|
|
|
|
| 173 |
"tracking_warning": tracking_warning,
|
| 174 |
"processed_items": processed_items,
|
| 175 |
"skipped_items": skipped,
|
|
@@ -200,6 +264,7 @@ def parse_args() -> argparse.Namespace:
|
|
| 200 |
|
| 201 |
def main() -> int:
|
| 202 |
args = parse_args()
|
|
|
|
| 203 |
summary = orchestrate_drive_documents(
|
| 204 |
model=args.model,
|
| 205 |
include_subfolders=not args.no_subfolders,
|
|
@@ -211,8 +276,9 @@ def main() -> int:
|
|
| 211 |
output_path = Path(args.output)
|
| 212 |
output_path.parent.mkdir(parents=True, exist_ok=True)
|
| 213 |
output_path.write_text(rendered, encoding="utf-8")
|
|
|
|
| 214 |
else:
|
| 215 |
-
|
| 216 |
|
| 217 |
return 0 if summary.get("ok") else 1
|
| 218 |
|
|
|
|
| 2 |
|
| 3 |
import argparse
|
| 4 |
import json
|
| 5 |
+
import logging
|
| 6 |
import os
|
| 7 |
import shutil
|
| 8 |
from pathlib import Path
|
|
|
|
| 28 |
load_dotenv()
|
| 29 |
|
| 30 |
|
| 31 |
+
logger = logging.getLogger(__name__)
|
| 32 |
+
if not logger.handlers:
|
| 33 |
+
handler = logging.StreamHandler()
|
| 34 |
+
handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(name)s: %(message)s"))
|
| 35 |
+
logger.addHandler(handler)
|
| 36 |
+
logger.setLevel(logging.INFO)
|
| 37 |
+
logger.propagate = False
|
| 38 |
+
|
| 39 |
+
|
| 40 |
SHEET_COLUMNS = ["fecha", "source_file", "display_description", "total"]
|
| 41 |
+
CORRUPTED_SHEET_NAME = "corrupted data"
|
| 42 |
|
| 43 |
|
| 44 |
def _build_display_description(parsed: dict[str, Any]) -> str:
|
|
|
|
| 58 |
}
|
| 59 |
|
| 60 |
|
| 61 |
+
def _is_blank_sheet_value(value: Any) -> bool:
|
| 62 |
+
if value is None:
|
| 63 |
+
return True
|
| 64 |
+
|
| 65 |
+
if isinstance(value, str):
|
| 66 |
+
return not value.strip()
|
| 67 |
+
|
| 68 |
+
return False
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def _is_complete_sheet_row(row: dict[str, Any]) -> bool:
|
| 72 |
+
return all(not _is_blank_sheet_value(row.get(column)) for column in SHEET_COLUMNS)
|
| 73 |
+
|
| 74 |
+
|
| 75 |
def _create_openai_client() -> OpenAI:
|
| 76 |
api_key = (os.getenv("OPENAI_API_KEY") or "").strip()
|
| 77 |
if not api_key:
|
|
|
|
| 99 |
include_subfolders: bool = True,
|
| 100 |
send_to_sheet: bool = False,
|
| 101 |
) -> dict[str, Any]:
|
| 102 |
+
logger.info(
|
| 103 |
+
"Starting orchestration model=%s include_subfolders=%s send_to_sheet=%s",
|
| 104 |
+
model,
|
| 105 |
+
include_subfolders,
|
| 106 |
+
send_to_sheet,
|
| 107 |
+
)
|
| 108 |
tracker = build_firebase_tracker()
|
| 109 |
client = _create_openai_client()
|
| 110 |
|
|
|
|
| 112 |
try:
|
| 113 |
drive_documents, scan_temp_dir = scan_google_drive_supported_documents(include_subfolders=include_subfolders)
|
| 114 |
except GoogleDriveConfigError as exc:
|
| 115 |
+
logger.error("Google Drive configuration error: %s", exc)
|
| 116 |
return {
|
| 117 |
"ok": False,
|
| 118 |
"error": f"Google Drive configuration error: {exc}",
|
| 119 |
}
|
| 120 |
|
| 121 |
+
logger.info("Scan completed scanned_documents=%s", len(drive_documents))
|
| 122 |
+
|
| 123 |
docs_payload = [
|
| 124 |
{
|
| 125 |
"document_id": doc.document_id,
|
|
|
|
| 131 |
]
|
| 132 |
|
| 133 |
to_process, skipped, tracking_warning = check_drive_documents_to_process(tracker, docs_payload)
|
| 134 |
+
logger.info("Documents to process=%s skipped=%s", len(to_process), len(skipped))
|
| 135 |
+
if tracking_warning:
|
| 136 |
+
logger.warning("Tracking warning: %s", tracking_warning)
|
| 137 |
|
| 138 |
parsed_count = 0
|
| 139 |
modified_count = 0
|
| 140 |
sent_count = 0
|
| 141 |
+
corrupted_count = 0
|
| 142 |
error_items: list[dict[str, str]] = []
|
| 143 |
processed_items: list[dict[str, Any]] = []
|
| 144 |
|
|
|
|
| 148 |
local_path = str(document.get("local_path") or "").strip()
|
| 149 |
modification_date = document.get("modificationDate")
|
| 150 |
status = str(document.get("target_status") or "Parsed")
|
| 151 |
+
logger.info("Processing document_id=%s source_file=%s status=%s", document_id, source_file, status)
|
| 152 |
|
| 153 |
try:
|
| 154 |
parsed = _parse_document(client, model, local_path, source_file)
|
| 155 |
+
logger.info("Parsed document_id=%s source_file=%s", document_id, source_file)
|
| 156 |
|
| 157 |
if tracker is not None:
|
| 158 |
tracker.save_document_record(
|
|
|
|
| 168 |
else:
|
| 169 |
parsed_count += 1
|
| 170 |
|
| 171 |
+
worksheet_name: str | None = None
|
| 172 |
if send_to_sheet:
|
| 173 |
+
row_for_sheet = _build_sheet_row(parsed)
|
| 174 |
+
worksheet_name = None if _is_complete_sheet_row(row_for_sheet) else CORRUPTED_SHEET_NAME
|
| 175 |
+
append_row_to_google_sheet(row_for_sheet, SHEET_COLUMNS, worksheet_name=worksheet_name)
|
| 176 |
+
logger.info(
|
| 177 |
+
"Sent document_id=%s source_file=%s worksheet=%s",
|
| 178 |
+
document_id,
|
| 179 |
+
source_file,
|
| 180 |
+
worksheet_name or "main",
|
| 181 |
+
)
|
| 182 |
+
if tracker is not None and worksheet_name is None:
|
| 183 |
tracker.mark_document_sent(document_id)
|
| 184 |
sent_count += 1
|
| 185 |
+
if worksheet_name is not None:
|
| 186 |
+
corrupted_count += 1
|
| 187 |
|
| 188 |
processed_items.append(
|
| 189 |
{
|
| 190 |
"document_id": document_id,
|
| 191 |
"source_file": source_file,
|
| 192 |
"modificationDate": modification_date,
|
| 193 |
+
"status": "Corrupted" if worksheet_name is not None else ("Sent" if send_to_sheet else status),
|
| 194 |
}
|
| 195 |
)
|
| 196 |
except GoogleSheetsConfigError as exc:
|
| 197 |
+
logger.error("Google Sheets config error for document_id=%s source_file=%s: %s", document_id, source_file, exc)
|
| 198 |
error_items.append(
|
| 199 |
{
|
| 200 |
"document_id": document_id,
|
|
|
|
| 203 |
}
|
| 204 |
)
|
| 205 |
except Exception as exc:
|
| 206 |
+
logger.exception("Failed processing document_id=%s source_file=%s", document_id, source_file)
|
| 207 |
error_items.append(
|
| 208 |
{
|
| 209 |
"document_id": document_id,
|
|
|
|
| 214 |
|
| 215 |
if scan_temp_dir:
|
| 216 |
shutil.rmtree(scan_temp_dir, ignore_errors=True)
|
| 217 |
+
logger.info("Cleaned up temporary scan directory")
|
| 218 |
+
|
| 219 |
+
logger.info(
|
| 220 |
+
"Orchestration finished parsed=%s modified=%s sent=%s corrupted=%s errors=%s",
|
| 221 |
+
parsed_count,
|
| 222 |
+
modified_count,
|
| 223 |
+
sent_count,
|
| 224 |
+
corrupted_count,
|
| 225 |
+
len(error_items),
|
| 226 |
+
)
|
| 227 |
|
| 228 |
return {
|
| 229 |
"ok": True,
|
|
|
|
| 233 |
"parsed": parsed_count,
|
| 234 |
"modified": modified_count,
|
| 235 |
"sent": sent_count,
|
| 236 |
+
"corrupted": corrupted_count,
|
| 237 |
"tracking_warning": tracking_warning,
|
| 238 |
"processed_items": processed_items,
|
| 239 |
"skipped_items": skipped,
|
|
|
|
| 264 |
|
| 265 |
def main() -> int:
|
| 266 |
args = parse_args()
|
| 267 |
+
logger.info("Running document orchestration CLI output=%s", args.output or "stdout")
|
| 268 |
summary = orchestrate_drive_documents(
|
| 269 |
model=args.model,
|
| 270 |
include_subfolders=not args.no_subfolders,
|
|
|
|
| 276 |
output_path = Path(args.output)
|
| 277 |
output_path.parent.mkdir(parents=True, exist_ok=True)
|
| 278 |
output_path.write_text(rendered, encoding="utf-8")
|
| 279 |
+
logger.info("Wrote orchestration summary to %s", output_path)
|
| 280 |
else:
|
| 281 |
+
logger.info("Orchestration summary:\n%s", rendered)
|
| 282 |
|
| 283 |
return 0 if summary.get("ok") else 1
|
| 284 |
|
google_sheets_service.py
CHANGED
|
@@ -7,6 +7,7 @@ from typing import Any
|
|
| 7 |
|
| 8 |
import gspread
|
| 9 |
from dotenv import load_dotenv
|
|
|
|
| 10 |
from google.oauth2.service_account import Credentials
|
| 11 |
|
| 12 |
|
|
@@ -42,15 +43,22 @@ def _build_credentials() -> Credentials:
|
|
| 42 |
)
|
| 43 |
|
| 44 |
|
| 45 |
-
def _open_worksheet(client: gspread.Client):
|
| 46 |
spreadsheet_id = (os.getenv("GOOGLE_SHEETS_SPREADSHEET_ID") or "").strip()
|
| 47 |
-
|
| 48 |
|
| 49 |
if not spreadsheet_id:
|
| 50 |
raise GoogleSheetsConfigError("Missing GOOGLE_SHEETS_SPREADSHEET_ID.")
|
| 51 |
|
| 52 |
spreadsheet = client.open_by_key(spreadsheet_id)
|
| 53 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 54 |
|
| 55 |
|
| 56 |
def _normalize_cell_value(value: Any) -> Any:
|
|
@@ -92,10 +100,14 @@ def _column_index_to_letter(index: int) -> str:
|
|
| 92 |
return "".join(reversed(letters))
|
| 93 |
|
| 94 |
|
| 95 |
-
def append_row_to_google_sheet(
|
|
|
|
|
|
|
|
|
|
|
|
|
| 96 |
credentials = _build_credentials()
|
| 97 |
client = gspread.authorize(credentials)
|
| 98 |
-
worksheet = _open_worksheet(client)
|
| 99 |
|
| 100 |
values = [_normalize_cell_value(row.get(column, "")) for column in ordered_columns]
|
| 101 |
|
|
|
|
| 7 |
|
| 8 |
import gspread
|
| 9 |
from dotenv import load_dotenv
|
| 10 |
+
from gspread.exceptions import WorksheetNotFound
|
| 11 |
from google.oauth2.service_account import Credentials
|
| 12 |
|
| 13 |
|
|
|
|
| 43 |
)
|
| 44 |
|
| 45 |
|
| 46 |
+
def _open_worksheet(client: gspread.Client, worksheet_name: str | None = None, column_count: int = 0):
|
| 47 |
spreadsheet_id = (os.getenv("GOOGLE_SHEETS_SPREADSHEET_ID") or "").strip()
|
| 48 |
+
resolved_worksheet_name = (worksheet_name or os.getenv("GOOGLE_SHEETS_WORKSHEET") or "Sheet1").strip()
|
| 49 |
|
| 50 |
if not spreadsheet_id:
|
| 51 |
raise GoogleSheetsConfigError("Missing GOOGLE_SHEETS_SPREADSHEET_ID.")
|
| 52 |
|
| 53 |
spreadsheet = client.open_by_key(spreadsheet_id)
|
| 54 |
+
try:
|
| 55 |
+
return spreadsheet.worksheet(resolved_worksheet_name)
|
| 56 |
+
except WorksheetNotFound:
|
| 57 |
+
return spreadsheet.add_worksheet(
|
| 58 |
+
title=resolved_worksheet_name,
|
| 59 |
+
rows=1000,
|
| 60 |
+
cols=max(column_count, 1),
|
| 61 |
+
)
|
| 62 |
|
| 63 |
|
| 64 |
def _normalize_cell_value(value: Any) -> Any:
|
|
|
|
| 100 |
return "".join(reversed(letters))
|
| 101 |
|
| 102 |
|
| 103 |
+
def append_row_to_google_sheet(
|
| 104 |
+
row: dict[str, Any],
|
| 105 |
+
ordered_columns: list[str],
|
| 106 |
+
worksheet_name: str | None = None,
|
| 107 |
+
) -> None:
|
| 108 |
credentials = _build_credentials()
|
| 109 |
client = gspread.authorize(credentials)
|
| 110 |
+
worksheet = _open_worksheet(client, worksheet_name=worksheet_name, column_count=len(ordered_columns))
|
| 111 |
|
| 112 |
values = [_normalize_cell_value(row.get(column, "")) for column in ordered_columns]
|
| 113 |
|