Spaces:
Sleeping
Sleeping
segovia commited on
Commit ·
81ecf5d
1
Parent(s): 119c0cc
google sheet integration
Browse files* add select column
* add send button
* add delete rows
* remove context menu
- app.py +77 -31
- google_sheets_service.py +72 -0
- requirements.txt +3 -1
app.py
CHANGED
|
@@ -2,7 +2,6 @@ from __future__ import annotations
|
|
| 2 |
|
| 3 |
import os
|
| 4 |
import csv
|
| 5 |
-
import json
|
| 6 |
import tempfile
|
| 7 |
from pathlib import Path
|
| 8 |
|
|
@@ -16,6 +15,7 @@ from openai import OpenAI
|
|
| 16 |
from document_processing import extract_text, is_image_document, to_data_uri
|
| 17 |
from receipt_ai import extract_receipt_json, extract_receipt_json_from_image
|
| 18 |
from receipt_results import build_empty_result
|
|
|
|
| 19 |
|
| 20 |
try:
|
| 21 |
import spaces # type: ignore[import-not-found]
|
|
@@ -36,6 +36,12 @@ RESULT_COLUMNS = [
|
|
| 36 |
"total",
|
| 37 |
"confidence",
|
| 38 |
]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 39 |
|
| 40 |
def gpu_decorator(fn):
|
| 41 |
if spaces is None:
|
|
@@ -71,7 +77,7 @@ def parse_uploaded_file(client, model: str, file_path: str) -> dict:
|
|
| 71 |
|
| 72 |
|
| 73 |
def rows_to_table(rows: list[dict]) -> list[list[object]]:
|
| 74 |
-
return [[row.get(column) for column in RESULT_COLUMNS] for row in rows]
|
| 75 |
|
| 76 |
|
| 77 |
def table_to_rows(table: object) -> list[dict]:
|
|
@@ -88,6 +94,9 @@ def table_to_rows(table: object) -> list[dict]:
|
|
| 88 |
rows: list[dict] = []
|
| 89 |
for raw_row in table: # type: ignore[assignment]
|
| 90 |
values = list(raw_row)
|
|
|
|
|
|
|
|
|
|
| 91 |
row = {}
|
| 92 |
for index, column in enumerate(RESULT_COLUMNS):
|
| 93 |
row[column] = values[index] if index < len(values) else None
|
|
@@ -95,6 +104,29 @@ def table_to_rows(table: object) -> list[dict]:
|
|
| 95 |
return rows
|
| 96 |
|
| 97 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 98 |
def build_csv_download(rows: list[dict]) -> str:
|
| 99 |
temp_file = tempfile.NamedTemporaryFile(delete=False, suffix=".csv", mode="w", newline="", encoding="utf-8")
|
| 100 |
with temp_file as csv_file:
|
|
@@ -154,33 +186,53 @@ def select_grid_row(evt: gr.SelectData):
|
|
| 154 |
return row_index, f"Selected row {row_index + 1}."
|
| 155 |
|
| 156 |
|
| 157 |
-
def
|
| 158 |
rows = list(existing_rows or [])
|
| 159 |
-
|
| 160 |
-
|
|
|
|
| 161 |
|
| 162 |
-
|
| 163 |
-
|
|
|
|
| 164 |
|
| 165 |
-
|
| 166 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 167 |
|
| 168 |
|
| 169 |
-
def
|
| 170 |
rows = list(existing_rows or [])
|
| 171 |
-
|
| 172 |
-
|
|
|
|
| 173 |
|
| 174 |
-
|
| 175 |
-
|
| 176 |
-
|
| 177 |
-
|
| 178 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 179 |
|
| 180 |
def build_app() -> gr.Blocks:
|
| 181 |
default_model = os.getenv("OPENAI_MODEL", "gpt-4o")
|
| 182 |
|
| 183 |
-
with gr.Blocks(title="Document Parser") as demo:
|
| 184 |
gr.Markdown("# Purchase Receipt Parser\nUpload one or more files and append the extracted receipt data into the grid.")
|
| 185 |
|
| 186 |
with gr.Row():
|
|
@@ -190,18 +242,17 @@ def build_app() -> gr.Blocks:
|
|
| 190 |
status = gr.Markdown("Ready.")
|
| 191 |
results_state = gr.State([])
|
| 192 |
selected_row_state = gr.State(None)
|
| 193 |
-
send_payload = gr.Textbox(visible=False)
|
| 194 |
|
| 195 |
with gr.Row():
|
| 196 |
parse_button = gr.Button("Parse documents", variant="primary")
|
| 197 |
clear_button = gr.Button("Clear grid")
|
| 198 |
send_row_button = gr.Button("Send", variant="secondary")
|
| 199 |
-
delete_row_button = gr.Button("Delete
|
| 200 |
download_button = gr.DownloadButton(label="Download CSV", variant="secondary", value=build_csv_download, inputs=[results_state])
|
| 201 |
|
| 202 |
results_grid = gr.Dataframe(
|
| 203 |
label="Parsed documents",
|
| 204 |
-
headers=RESULT_COLUMNS,
|
| 205 |
value=rows_to_table([]),
|
| 206 |
interactive=True,
|
| 207 |
wrap=True,
|
|
@@ -232,20 +283,15 @@ def build_app() -> gr.Blocks:
|
|
| 232 |
)
|
| 233 |
|
| 234 |
delete_row_button.click(
|
| 235 |
-
|
| 236 |
-
inputs=[results_state,
|
| 237 |
outputs=[results_state, results_grid, status],
|
| 238 |
)
|
| 239 |
|
| 240 |
send_row_button.click(
|
| 241 |
-
|
| 242 |
-
inputs=[results_state,
|
| 243 |
-
outputs=[
|
| 244 |
-
).then(
|
| 245 |
-
fn=None,
|
| 246 |
-
inputs=[send_payload],
|
| 247 |
-
outputs=[],
|
| 248 |
-
js="(payload) => { if (payload) { alert(payload); } }",
|
| 249 |
)
|
| 250 |
|
| 251 |
return demo
|
|
|
|
| 2 |
|
| 3 |
import os
|
| 4 |
import csv
|
|
|
|
| 5 |
import tempfile
|
| 6 |
from pathlib import Path
|
| 7 |
|
|
|
|
| 15 |
from document_processing import extract_text, is_image_document, to_data_uri
|
| 16 |
from receipt_ai import extract_receipt_json, extract_receipt_json_from_image
|
| 17 |
from receipt_results import build_empty_result
|
| 18 |
+
from google_sheets_service import GoogleSheetsConfigError, append_row_to_google_sheet
|
| 19 |
|
| 20 |
try:
|
| 21 |
import spaces # type: ignore[import-not-found]
|
|
|
|
| 36 |
"total",
|
| 37 |
"confidence",
|
| 38 |
]
|
| 39 |
+
SELECT_COLUMN = "sel"
|
| 40 |
+
custom_css = """
|
| 41 |
+
.cell-menu, .cell-menu-button {
|
| 42 |
+
display: none !important;
|
| 43 |
+
}
|
| 44 |
+
"""
|
| 45 |
|
| 46 |
def gpu_decorator(fn):
|
| 47 |
if spaces is None:
|
|
|
|
| 77 |
|
| 78 |
|
| 79 |
def rows_to_table(rows: list[dict]) -> list[list[object]]:
|
| 80 |
+
return [[False] + [row.get(column) for column in RESULT_COLUMNS] for row in rows]
|
| 81 |
|
| 82 |
|
| 83 |
def table_to_rows(table: object) -> list[dict]:
|
|
|
|
| 94 |
rows: list[dict] = []
|
| 95 |
for raw_row in table: # type: ignore[assignment]
|
| 96 |
values = list(raw_row)
|
| 97 |
+
# Dataframe rows include a leading checkbox value that is not part of the result payload.
|
| 98 |
+
if len(values) > len(RESULT_COLUMNS):
|
| 99 |
+
values = values[-len(RESULT_COLUMNS):]
|
| 100 |
row = {}
|
| 101 |
for index, column in enumerate(RESULT_COLUMNS):
|
| 102 |
row[column] = values[index] if index < len(values) else None
|
|
|
|
| 104 |
return rows
|
| 105 |
|
| 106 |
|
| 107 |
+
def get_selected_row_indexes(grid_value: object) -> list[int]:
|
| 108 |
+
if grid_value is None:
|
| 109 |
+
return []
|
| 110 |
+
|
| 111 |
+
selected_indexes: list[int] = []
|
| 112 |
+
|
| 113 |
+
if hasattr(grid_value, "iterrows"):
|
| 114 |
+
try:
|
| 115 |
+
for index, row in grid_value.iterrows():
|
| 116 |
+
if bool(row.get(SELECT_COLUMN, False)):
|
| 117 |
+
selected_indexes.append(int(index))
|
| 118 |
+
return selected_indexes
|
| 119 |
+
except Exception:
|
| 120 |
+
pass
|
| 121 |
+
|
| 122 |
+
for index, raw_row in enumerate(grid_value): # type: ignore[assignment]
|
| 123 |
+
values = list(raw_row)
|
| 124 |
+
if values and bool(values[0]):
|
| 125 |
+
selected_indexes.append(index)
|
| 126 |
+
|
| 127 |
+
return selected_indexes
|
| 128 |
+
|
| 129 |
+
|
| 130 |
def build_csv_download(rows: list[dict]) -> str:
|
| 131 |
temp_file = tempfile.NamedTemporaryFile(delete=False, suffix=".csv", mode="w", newline="", encoding="utf-8")
|
| 132 |
with temp_file as csv_file:
|
|
|
|
| 186 |
return row_index, f"Selected row {row_index + 1}."
|
| 187 |
|
| 188 |
|
| 189 |
+
def delete_selected_rows(existing_rows: list[dict] | None, grid_value: object):
|
| 190 |
rows = list(existing_rows or [])
|
| 191 |
+
selected_indexes = get_selected_row_indexes(grid_value)
|
| 192 |
+
if not selected_indexes:
|
| 193 |
+
return rows, gr.update(value=rows_to_table(rows)), "Select at least one row with the checkbox first."
|
| 194 |
|
| 195 |
+
for index in selected_indexes:
|
| 196 |
+
if index < 0 or index >= len(rows):
|
| 197 |
+
return rows, gr.update(value=rows_to_table(rows)), "One or more selected rows are out of range."
|
| 198 |
|
| 199 |
+
selected_set = set(selected_indexes)
|
| 200 |
+
remaining_rows = [row for index, row in enumerate(rows) if index not in selected_set]
|
| 201 |
+
return (
|
| 202 |
+
remaining_rows,
|
| 203 |
+
gr.update(value=rows_to_table(remaining_rows)),
|
| 204 |
+
f"Deleted {len(selected_indexes)} row(s).",
|
| 205 |
+
)
|
| 206 |
|
| 207 |
|
| 208 |
+
def send_selected_rows(existing_rows: list[dict] | None, grid_value: object):
|
| 209 |
rows = list(existing_rows or [])
|
| 210 |
+
selected_indexes = get_selected_row_indexes(grid_value)
|
| 211 |
+
if not selected_indexes:
|
| 212 |
+
return rows, gr.update(value=rows_to_table(rows)), "Select at least one row with the checkbox first."
|
| 213 |
|
| 214 |
+
try:
|
| 215 |
+
for index in selected_indexes:
|
| 216 |
+
if index < 0 or index >= len(rows):
|
| 217 |
+
return rows, gr.update(value=rows_to_table(rows)), "One or more selected rows are out of range."
|
| 218 |
+
append_row_to_google_sheet(rows[index], RESULT_COLUMNS)
|
| 219 |
+
except GoogleSheetsConfigError as exc:
|
| 220 |
+
return rows, gr.update(value=rows_to_table(rows)), f"Google Sheets configuration error: {exc}"
|
| 221 |
+
except Exception as exc:
|
| 222 |
+
return rows, gr.update(value=rows_to_table(rows)), f"Failed to insert row into Google Sheets: {exc}"
|
| 223 |
+
|
| 224 |
+
selected_set = set(selected_indexes)
|
| 225 |
+
remaining_rows = [row for index, row in enumerate(rows) if index not in selected_set]
|
| 226 |
+
return (
|
| 227 |
+
remaining_rows,
|
| 228 |
+
gr.update(value=rows_to_table(remaining_rows)),
|
| 229 |
+
f"Sent {len(selected_indexes)} row(s) to Google Sheets and removed them from the grid.",
|
| 230 |
+
)
|
| 231 |
|
| 232 |
def build_app() -> gr.Blocks:
|
| 233 |
default_model = os.getenv("OPENAI_MODEL", "gpt-4o")
|
| 234 |
|
| 235 |
+
with gr.Blocks(css= custom_css,title="Document Parser") as demo:
|
| 236 |
gr.Markdown("# Purchase Receipt Parser\nUpload one or more files and append the extracted receipt data into the grid.")
|
| 237 |
|
| 238 |
with gr.Row():
|
|
|
|
| 242 |
status = gr.Markdown("Ready.")
|
| 243 |
results_state = gr.State([])
|
| 244 |
selected_row_state = gr.State(None)
|
|
|
|
| 245 |
|
| 246 |
with gr.Row():
|
| 247 |
parse_button = gr.Button("Parse documents", variant="primary")
|
| 248 |
clear_button = gr.Button("Clear grid")
|
| 249 |
send_row_button = gr.Button("Send", variant="secondary")
|
| 250 |
+
delete_row_button = gr.Button("Delete selected", variant="stop")
|
| 251 |
download_button = gr.DownloadButton(label="Download CSV", variant="secondary", value=build_csv_download, inputs=[results_state])
|
| 252 |
|
| 253 |
results_grid = gr.Dataframe(
|
| 254 |
label="Parsed documents",
|
| 255 |
+
headers=[SELECT_COLUMN] + RESULT_COLUMNS,
|
| 256 |
value=rows_to_table([]),
|
| 257 |
interactive=True,
|
| 258 |
wrap=True,
|
|
|
|
| 283 |
)
|
| 284 |
|
| 285 |
delete_row_button.click(
|
| 286 |
+
delete_selected_rows,
|
| 287 |
+
inputs=[results_state, results_grid],
|
| 288 |
outputs=[results_state, results_grid, status],
|
| 289 |
)
|
| 290 |
|
| 291 |
send_row_button.click(
|
| 292 |
+
send_selected_rows,
|
| 293 |
+
inputs=[results_state, results_grid],
|
| 294 |
+
outputs=[results_state, results_grid, status],
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 295 |
)
|
| 296 |
|
| 297 |
return demo
|
google_sheets_service.py
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import json
|
| 4 |
+
import math
|
| 5 |
+
import os
|
| 6 |
+
from typing import Any
|
| 7 |
+
|
| 8 |
+
import gspread
|
| 9 |
+
from dotenv import load_dotenv
|
| 10 |
+
from google.oauth2.service_account import Credentials
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
SCOPES = [
|
| 14 |
+
"https://www.googleapis.com/auth/spreadsheets",
|
| 15 |
+
"https://www.googleapis.com/auth/drive",
|
| 16 |
+
]
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
load_dotenv()
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
class GoogleSheetsConfigError(RuntimeError):
|
| 23 |
+
"""Raised when required Google Sheets environment configuration is missing or invalid."""
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def _build_credentials() -> Credentials:
|
| 27 |
+
service_account_json = (os.getenv("GOOGLE_SERVICE_ACCOUNT_JSON") or "").strip()
|
| 28 |
+
service_account_file = (os.getenv("GOOGLE_SERVICE_ACCOUNT_FILE") or "").strip()
|
| 29 |
+
|
| 30 |
+
if service_account_json:
|
| 31 |
+
try:
|
| 32 |
+
info = json.loads(service_account_json)
|
| 33 |
+
except json.JSONDecodeError as exc:
|
| 34 |
+
raise GoogleSheetsConfigError("GOOGLE_SERVICE_ACCOUNT_JSON is not valid JSON.") from exc
|
| 35 |
+
return Credentials.from_service_account_info(info, scopes=SCOPES)
|
| 36 |
+
|
| 37 |
+
if service_account_file:
|
| 38 |
+
return Credentials.from_service_account_file(service_account_file, scopes=SCOPES)
|
| 39 |
+
|
| 40 |
+
raise GoogleSheetsConfigError(
|
| 41 |
+
"Missing Google credentials. Set GOOGLE_SERVICE_ACCOUNT_JSON or GOOGLE_SERVICE_ACCOUNT_FILE."
|
| 42 |
+
)
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def _open_worksheet(client: gspread.Client):
|
| 46 |
+
spreadsheet_id = (os.getenv("GOOGLE_SHEETS_SPREADSHEET_ID") or "").strip()
|
| 47 |
+
worksheet_name = (os.getenv("GOOGLE_SHEETS_WORKSHEET") or "Sheet1").strip()
|
| 48 |
+
|
| 49 |
+
if not spreadsheet_id:
|
| 50 |
+
raise GoogleSheetsConfigError("Missing GOOGLE_SHEETS_SPREADSHEET_ID.")
|
| 51 |
+
|
| 52 |
+
spreadsheet = client.open_by_key(spreadsheet_id)
|
| 53 |
+
return spreadsheet.worksheet(worksheet_name)
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def _normalize_cell_value(value: Any) -> Any:
|
| 57 |
+
if value is None:
|
| 58 |
+
return ""
|
| 59 |
+
|
| 60 |
+
if isinstance(value, float) and not math.isfinite(value):
|
| 61 |
+
return ""
|
| 62 |
+
|
| 63 |
+
return value
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def append_row_to_google_sheet(row: dict[str, Any], ordered_columns: list[str]) -> None:
|
| 67 |
+
credentials = _build_credentials()
|
| 68 |
+
client = gspread.authorize(credentials)
|
| 69 |
+
worksheet = _open_worksheet(client)
|
| 70 |
+
|
| 71 |
+
values = [_normalize_cell_value(row.get(column, "")) for column in ordered_columns]
|
| 72 |
+
worksheet.append_row(values, value_input_option="USER_ENTERED")
|
requirements.txt
CHANGED
|
@@ -3,4 +3,6 @@ pypdf>=4.2.0
|
|
| 3 |
python-docx>=1.1.2
|
| 4 |
python-dotenv>=1.0.1
|
| 5 |
gradio>=4.0.0
|
| 6 |
-
spaces>=0.31.1
|
|
|
|
|
|
|
|
|
| 3 |
python-docx>=1.1.2
|
| 4 |
python-dotenv>=1.0.1
|
| 5 |
gradio>=4.0.0
|
| 6 |
+
spaces>=0.31.1
|
| 7 |
+
gspread>=6.0.0
|
| 8 |
+
google-auth>=2.30.0
|