segovia commited on
Commit
cb04ca0
·
1 Parent(s): 207d271

Add drive service

Browse files

move files from the source folder to another one

document_orchester.py CHANGED
@@ -5,6 +5,7 @@ import json
5
  import logging
6
  import os
7
  import shutil
 
8
  from pathlib import Path
9
  from typing import Any
10
 
@@ -16,8 +17,13 @@ from firebase_processed_files import (
16
  build_firebase_tracker,
17
  check_drive_documents_to_process,
18
  )
19
- from google_drive_scanner import (
 
20
  GoogleDriveConfigError,
 
 
 
 
21
  scan_google_drive_supported_documents,
22
  )
23
  from google_sheets_service import GoogleSheetsConfigError, append_row_to_google_sheet
@@ -39,6 +45,14 @@ logger.propagate = False
39
 
40
  SHEET_COLUMNS = ["fecha", "source_file", "display_description", "total"]
41
  CORRUPTED_SHEET_NAME = "corrupted data"
 
 
 
 
 
 
 
 
42
 
43
 
44
  def _build_display_description(parsed: dict[str, Any]) -> str:
@@ -79,6 +93,60 @@ def _create_openai_client() -> OpenAI:
79
  return OpenAI(api_key=api_key)
80
 
81
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
82
  def _parse_document(client: OpenAI, model: str, local_path: str, source_file: str) -> dict[str, Any]:
83
  path = Path(local_path)
84
  if is_image_document(path):
@@ -105,11 +173,14 @@ def orchestrate_drive_documents(
105
  include_subfolders,
106
  send_to_sheet,
107
  )
 
108
  tracker = build_firebase_tracker()
109
  client = _create_openai_client()
 
110
 
111
  scan_temp_dir: str | None = None
112
  try:
 
113
  drive_documents, scan_temp_dir = scan_google_drive_supported_documents(include_subfolders=include_subfolders)
114
  except GoogleDriveConfigError as exc:
115
  logger.error("Google Drive configuration error: %s", exc)
@@ -138,6 +209,7 @@ def orchestrate_drive_documents(
138
  parsed_count = 0
139
  modified_count = 0
140
  sent_count = 0
 
141
  corrupted_count = 0
142
  error_items: list[dict[str, str]] = []
143
  processed_items: list[dict[str, Any]] = []
@@ -154,6 +226,23 @@ def orchestrate_drive_documents(
154
  parsed = _parse_document(client, model, local_path, source_file)
155
  logger.info("Parsed document_id=%s source_file=%s", document_id, source_file)
156
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
157
  if tracker is not None:
158
  tracker.save_document_record(
159
  document_id=document_id,
@@ -171,6 +260,10 @@ def orchestrate_drive_documents(
171
  worksheet_name: str | None = None
172
  if send_to_sheet:
173
  row_for_sheet = _build_sheet_row(parsed)
 
 
 
 
174
  worksheet_name = None if _is_complete_sheet_row(row_for_sheet) else CORRUPTED_SHEET_NAME
175
  append_row_to_google_sheet(row_for_sheet, SHEET_COLUMNS, worksheet_name=worksheet_name)
176
  logger.info(
@@ -182,15 +275,14 @@ def orchestrate_drive_documents(
182
  if tracker is not None and worksheet_name is None:
183
  tracker.mark_document_sent(document_id)
184
  sent_count += 1
185
- if worksheet_name is not None:
186
- corrupted_count += 1
187
 
188
  processed_items.append(
189
  {
190
  "document_id": document_id,
191
  "source_file": source_file,
192
  "modificationDate": modification_date,
193
- "status": "Corrupted" if worksheet_name is not None else ("Sent" if send_to_sheet else status),
 
194
  }
195
  )
196
  except GoogleSheetsConfigError as exc:
@@ -217,10 +309,11 @@ def orchestrate_drive_documents(
217
  logger.info("Cleaned up temporary scan directory")
218
 
219
  logger.info(
220
- "Orchestration finished parsed=%s modified=%s sent=%s corrupted=%s errors=%s",
221
  parsed_count,
222
  modified_count,
223
  sent_count,
 
224
  corrupted_count,
225
  len(error_items),
226
  )
@@ -233,6 +326,7 @@ def orchestrate_drive_documents(
233
  "parsed": parsed_count,
234
  "modified": modified_count,
235
  "sent": sent_count,
 
236
  "corrupted": corrupted_count,
237
  "tracking_warning": tracking_warning,
238
  "processed_items": processed_items,
 
5
  import logging
6
  import os
7
  import shutil
8
+ from datetime import datetime
9
  from pathlib import Path
10
  from typing import Any
11
 
 
17
  build_firebase_tracker,
18
  check_drive_documents_to_process,
19
  )
20
+ from google_drive_service import (
21
+ DRIVE_FULL_SCOPE,
22
  GoogleDriveConfigError,
23
+ build_drive_service,
24
+ move_file_to_path,
25
+ )
26
+ from google_drive_scanner import (
27
  scan_google_drive_supported_documents,
28
  )
29
  from google_sheets_service import GoogleSheetsConfigError, append_row_to_google_sheet
 
45
 
46
  SHEET_COLUMNS = ["fecha", "source_file", "display_description", "total"]
47
  CORRUPTED_SHEET_NAME = "corrupted data"
48
+ MANDATORY_ENV_VARS = [
49
+ "OPENAI_API_KEY",
50
+ "GOOGLE_SERVICE_ACCOUNT_JSON",
51
+ "GOOGLE_SHEETS_SPREADSHEET_ID",
52
+ "GOOGLE_DRIVE_FOLDER_IDS",
53
+ "GOOGLE_DRIVE_INVOICES_ROOT_FOLDER_ID",
54
+ "ORCHESTRATOR_API_KEY",
55
+ ]
56
 
57
 
58
  def _build_display_description(parsed: dict[str, Any]) -> str:
 
93
  return OpenAI(api_key=api_key)
94
 
95
 
96
+ def _require_env_var(name: str) -> str:
97
+ value = (os.getenv(name) or "").strip()
98
+ if not value:
99
+ raise RuntimeError(f"Missing {name}.")
100
+ return value
101
+
102
+
103
+ def _validate_mandatory_env_vars() -> None:
104
+ missing_vars = [name for name in MANDATORY_ENV_VARS if not (os.getenv(name) or "").strip()]
105
+ if missing_vars:
106
+ raise RuntimeError(f"Missing mandatory env vars: {', '.join(missing_vars)}")
107
+
108
+
109
+ def _parse_invoice_date(value: Any) -> datetime | None:
110
+ text = str(value or "").strip()
111
+ if not text:
112
+ return None
113
+
114
+ for fmt in ("%Y-%m-%d", "%d/%m/%Y", "%d-%m-%Y", "%Y/%m/%d"):
115
+ try:
116
+ return datetime.strptime(text, fmt)
117
+ except ValueError:
118
+ continue
119
+
120
+ try:
121
+ return datetime.fromisoformat(text.replace("Z", "+00:00"))
122
+ except ValueError:
123
+ return None
124
+
125
+
126
+ def _resolve_drive_destination_path(parsed: dict[str, Any]) -> tuple[str, bool]:
127
+ base_path = (os.getenv("GOOGLE_DRIVE_INVOICES_BASE_PATH") or "Facturas").strip().strip("/") or "Facturas"
128
+
129
+ row_for_sheet = _build_sheet_row(parsed)
130
+ if not _is_complete_sheet_row(row_for_sheet):
131
+ return f"Corrupted", True
132
+
133
+ invoice_date = _parse_invoice_date(parsed.get("fecha"))
134
+ if invoice_date is None:
135
+ return f"Corrupted", True
136
+
137
+ return f"{invoice_date.strftime('%Y%m')}", False
138
+
139
+
140
+ def _build_drive_file_url(file_id: str) -> str:
141
+ return f"https://drive.google.com/file/d/{file_id}/view"
142
+
143
+
144
+ def _build_hyperlink_formula(url: str, label: str) -> str:
145
+ safe_url = str(url).replace('"', '""')
146
+ safe_label = str(label).replace('"', '""')
147
+ return f'=HYPERLINK("{safe_url}","{safe_label}")'
148
+
149
+
150
  def _parse_document(client: OpenAI, model: str, local_path: str, source_file: str) -> dict[str, Any]:
151
  path = Path(local_path)
152
  if is_image_document(path):
 
173
  include_subfolders,
174
  send_to_sheet,
175
  )
176
+ _validate_mandatory_env_vars()
177
  tracker = build_firebase_tracker()
178
  client = _create_openai_client()
179
+ drive_root_folder_id = _require_env_var("GOOGLE_DRIVE_INVOICES_ROOT_FOLDER_ID")
180
 
181
  scan_temp_dir: str | None = None
182
  try:
183
+ drive_service = build_drive_service(scope=DRIVE_FULL_SCOPE)
184
  drive_documents, scan_temp_dir = scan_google_drive_supported_documents(include_subfolders=include_subfolders)
185
  except GoogleDriveConfigError as exc:
186
  logger.error("Google Drive configuration error: %s", exc)
 
209
  parsed_count = 0
210
  modified_count = 0
211
  sent_count = 0
212
+ moved_count = 0
213
  corrupted_count = 0
214
  error_items: list[dict[str, str]] = []
215
  processed_items: list[dict[str, Any]] = []
 
226
  parsed = _parse_document(client, model, local_path, source_file)
227
  logger.info("Parsed document_id=%s source_file=%s", document_id, source_file)
228
 
229
+ destination_path, is_corrupted = _resolve_drive_destination_path(parsed)
230
+ if is_corrupted:
231
+ corrupted_count += 1
232
+ move_file_to_path(
233
+ service=drive_service,
234
+ file_id=document_id,
235
+ destination_path=destination_path,
236
+ root_folder_id=drive_root_folder_id,
237
+ )
238
+ moved_count += 1
239
+ logger.info(
240
+ "Moved document_id=%s source_file=%s to drive_path=%s",
241
+ document_id,
242
+ source_file,
243
+ destination_path,
244
+ )
245
+
246
  if tracker is not None:
247
  tracker.save_document_record(
248
  document_id=document_id,
 
260
  worksheet_name: str | None = None
261
  if send_to_sheet:
262
  row_for_sheet = _build_sheet_row(parsed)
263
+ row_for_sheet["source_file"] = _build_hyperlink_formula(
264
+ _build_drive_file_url(document_id),
265
+ source_file,
266
+ )
267
  worksheet_name = None if _is_complete_sheet_row(row_for_sheet) else CORRUPTED_SHEET_NAME
268
  append_row_to_google_sheet(row_for_sheet, SHEET_COLUMNS, worksheet_name=worksheet_name)
269
  logger.info(
 
275
  if tracker is not None and worksheet_name is None:
276
  tracker.mark_document_sent(document_id)
277
  sent_count += 1
 
 
278
 
279
  processed_items.append(
280
  {
281
  "document_id": document_id,
282
  "source_file": source_file,
283
  "modificationDate": modification_date,
284
+ "status": "Corrupted" if is_corrupted else ("Sent" if send_to_sheet else status),
285
+ "destination_path": destination_path,
286
  }
287
  )
288
  except GoogleSheetsConfigError as exc:
 
309
  logger.info("Cleaned up temporary scan directory")
310
 
311
  logger.info(
312
+ "Orchestration finished parsed=%s modified=%s sent=%s moved=%s corrupted=%s errors=%s",
313
  parsed_count,
314
  modified_count,
315
  sent_count,
316
+ moved_count,
317
  corrupted_count,
318
  len(error_items),
319
  )
 
326
  "parsed": parsed_count,
327
  "modified": modified_count,
328
  "sent": sent_count,
329
+ "moved": moved_count,
330
  "corrupted": corrupted_count,
331
  "tracking_warning": tracking_warning,
332
  "processed_items": processed_items,
google_drive_scanner.py CHANGED
@@ -1,46 +1,22 @@
1
  from __future__ import annotations
2
 
3
- import io
4
- import json
5
- import os
6
- import re
7
  import shutil
8
- import tempfile
9
  from dataclasses import dataclass
10
  from pathlib import Path
11
- from typing import Any
12
 
13
  from dotenv import load_dotenv
14
- from google.oauth2.service_account import Credentials
15
-
16
- from document_processing import SUPPORTED_EXTENSIONS
17
  from firebase_processed_files import FirebaseProcessedFilesTracker, check_files_already_processed
 
 
 
 
 
 
18
 
19
 
20
  load_dotenv()
21
 
22
 
23
- DRIVE_READONLY_SCOPE = "https://www.googleapis.com/auth/drive.readonly"
24
- FOLDER_MIME_TYPE = "application/vnd.google-apps.folder"
25
-
26
- # Google Workspace files require export, while binary files can be downloaded directly.
27
- SUPPORTED_EXPORT_MIME_MAP = {
28
- "application/vnd.google-apps.document": ("application/vnd.openxmlformats-officedocument.wordprocessingml.document", ".docx"),
29
- }
30
-
31
- SUPPORTED_BINARY_MIME_TO_SUFFIX = {
32
- "application/pdf": ".pdf",
33
- "text/plain": ".txt",
34
- "application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx",
35
- "image/jpeg": ".jpg",
36
- "image/png": ".png",
37
- }
38
-
39
-
40
- class GoogleDriveConfigError(RuntimeError):
41
- """Raised when required Google Drive environment configuration is missing or invalid."""
42
-
43
-
44
  @dataclass
45
  class DriveScanResult:
46
  files_to_process: list[str]
@@ -50,190 +26,14 @@ class DriveScanResult:
50
  warning: str | None = None
51
 
52
 
53
- @dataclass
54
- class DriveDocument:
55
- document_id: str
56
- source_file: str
57
- mime_type: str
58
- modificationDate: str | None
59
- local_path: str
60
-
61
-
62
- def _build_credentials() -> Credentials:
63
- service_account_json = (os.getenv("GOOGLE_SERVICE_ACCOUNT_JSON") or "").strip()
64
- service_account_file = (os.getenv("GOOGLE_SERVICE_ACCOUNT_FILE") or "").strip()
65
-
66
- if service_account_json:
67
- try:
68
- info = json.loads(service_account_json)
69
- except json.JSONDecodeError as exc:
70
- raise GoogleDriveConfigError("GOOGLE_SERVICE_ACCOUNT_JSON is not valid JSON.") from exc
71
- return Credentials.from_service_account_info(info, scopes=[DRIVE_READONLY_SCOPE])
72
-
73
- if service_account_file:
74
- return Credentials.from_service_account_file(service_account_file, scopes=[DRIVE_READONLY_SCOPE])
75
-
76
- raise GoogleDriveConfigError(
77
- "Missing Google credentials. Set GOOGLE_SERVICE_ACCOUNT_JSON or GOOGLE_SERVICE_ACCOUNT_FILE."
78
- )
79
-
80
-
81
- def _build_drive_service():
82
- try:
83
- from googleapiclient.discovery import build
84
- except ImportError as exc:
85
- raise GoogleDriveConfigError(
86
- "google-api-python-client is required for Drive scanning. Add it to requirements and install dependencies."
87
- ) from exc
88
-
89
- credentials = _build_credentials()
90
- return build("drive", "v3", credentials=credentials, cache_discovery=False)
91
-
92
-
93
- def _load_folder_ids_from_env() -> list[str]:
94
- raw = (os.getenv("GOOGLE_DRIVE_FOLDER_IDS") or "").strip()
95
- if not raw:
96
- raise GoogleDriveConfigError("Missing GOOGLE_DRIVE_FOLDER_IDS. Set one or more folder IDs separated by commas.")
97
-
98
- folder_ids = [item.strip() for item in raw.split(",") if item.strip()]
99
- if not folder_ids:
100
- raise GoogleDriveConfigError("GOOGLE_DRIVE_FOLDER_IDS is empty after parsing.")
101
-
102
- return folder_ids
103
-
104
-
105
- def _sanitize_file_name(name: str) -> str:
106
- cleaned = re.sub(r'[<>:"/\\|?*\x00-\x1F]', "_", name).strip().rstrip(".")
107
- return cleaned or "document"
108
-
109
-
110
- def _ensure_unique_destination(path: Path) -> Path:
111
- if not path.exists():
112
- return path
113
-
114
- counter = 1
115
- while True:
116
- candidate = path.with_name(f"{path.stem}_{counter}{path.suffix}")
117
- if not candidate.exists():
118
- return candidate
119
- counter += 1
120
-
121
-
122
- def _list_files_from_folder(service, folder_id: str, include_subfolders: bool) -> list[dict[str, Any]]:
123
- folders_to_scan = [folder_id]
124
- discovered: list[dict[str, Any]] = []
125
-
126
- while folders_to_scan:
127
- current_folder = folders_to_scan.pop(0)
128
- page_token = None
129
-
130
- while True:
131
- response = (
132
- service.files()
133
- .list(
134
- q=f"'{current_folder}' in parents and trashed = false",
135
- fields="nextPageToken, files(id, name, mimeType, modifiedTime)",
136
- supportsAllDrives=True,
137
- includeItemsFromAllDrives=True,
138
- pageSize=1000,
139
- pageToken=page_token,
140
- )
141
- .execute()
142
- )
143
-
144
- for file_meta in response.get("files", []):
145
- mime_type = file_meta.get("mimeType")
146
- if mime_type == FOLDER_MIME_TYPE:
147
- if include_subfolders:
148
- child_folder_id = file_meta.get("id")
149
- if child_folder_id:
150
- folders_to_scan.append(child_folder_id)
151
- continue
152
-
153
- discovered.append(file_meta)
154
-
155
- page_token = response.get("nextPageToken")
156
- if not page_token:
157
- break
158
-
159
- return discovered
160
-
161
-
162
- def _is_supported_drive_file(file_meta: dict[str, Any]) -> bool:
163
- mime_type = (file_meta.get("mimeType") or "").strip().lower()
164
- if mime_type in SUPPORTED_BINARY_MIME_TO_SUFFIX:
165
- return True
166
- if mime_type in SUPPORTED_EXPORT_MIME_MAP:
167
- return True
168
-
169
- name = (file_meta.get("name") or "").strip()
170
- suffix = Path(name).suffix.lower()
171
- return suffix in SUPPORTED_EXTENSIONS
172
-
173
-
174
- def _download_drive_file(service, file_meta: dict[str, Any], destination_dir: Path) -> str:
175
- try:
176
- from googleapiclient.http import MediaIoBaseDownload
177
- except ImportError as exc:
178
- raise GoogleDriveConfigError(
179
- "google-api-python-client is required for Drive scanning. Add it to requirements and install dependencies."
180
- ) from exc
181
-
182
- file_id = (file_meta.get("id") or "").strip()
183
- file_name = _sanitize_file_name((file_meta.get("name") or "document").strip())
184
- mime_type = (file_meta.get("mimeType") or "").strip().lower()
185
-
186
- suffix = Path(file_name).suffix.lower()
187
- request = None
188
-
189
- if mime_type in SUPPORTED_EXPORT_MIME_MAP:
190
- export_mime, export_suffix = SUPPORTED_EXPORT_MIME_MAP[mime_type]
191
- if suffix != export_suffix:
192
- file_name = f"{Path(file_name).stem}{export_suffix}"
193
- suffix = export_suffix
194
- request = service.files().export_media(fileId=file_id, mimeType=export_mime)
195
- else:
196
- if not suffix and mime_type in SUPPORTED_BINARY_MIME_TO_SUFFIX:
197
- file_name = f"{file_name}{SUPPORTED_BINARY_MIME_TO_SUFFIX[mime_type]}"
198
- suffix = Path(file_name).suffix.lower()
199
- request = service.files().get_media(fileId=file_id)
200
-
201
- if suffix not in SUPPORTED_EXTENSIONS:
202
- raise ValueError(f"Unsupported downloaded file type: {suffix or '[no extension]'}")
203
-
204
- destination = _ensure_unique_destination(destination_dir / file_name)
205
-
206
- with destination.open("wb") as stream:
207
- downloader = MediaIoBaseDownload(stream, request)
208
- done = False
209
- while not done:
210
- _, done = downloader.next_chunk()
211
-
212
- return str(destination)
213
-
214
-
215
  def scan_google_drive_unprocessed_files(
216
  tracker: FirebaseProcessedFilesTracker | None,
217
  include_subfolders: bool = True,
218
  ) -> DriveScanResult:
219
- folder_ids = _load_folder_ids_from_env()
220
- service = _build_drive_service()
221
-
222
- discovered_files: list[dict[str, Any]] = []
223
- for folder_id in folder_ids:
224
- discovered_files.extend(_list_files_from_folder(service, folder_id, include_subfolders))
225
 
226
- supported_files = [file_meta for file_meta in discovered_files if _is_supported_drive_file(file_meta)]
227
- temp_dir = tempfile.mkdtemp(prefix="drive_docs_")
228
- destination_dir = Path(temp_dir)
229
-
230
- downloaded_paths: list[str] = []
231
- for file_meta in supported_files:
232
- try:
233
- downloaded_paths.append(_download_drive_file(service, file_meta, destination_dir))
234
- except Exception:
235
- # Keep scan resilient if one file is malformed or inaccessible.
236
- continue
237
 
238
  files_to_process, skipped_files, warning = check_files_already_processed(tracker, downloaded_paths)
239
 
@@ -249,44 +49,17 @@ def scan_google_drive_unprocessed_files(
249
  return DriveScanResult(
250
  files_to_process=files_to_process,
251
  skipped_files=skipped_files,
252
- discovered_count=len(supported_files),
253
- temp_dir=temp_dir,
254
  warning=warning,
255
  )
256
 
257
 
258
  def scan_google_drive_supported_documents(include_subfolders: bool = True) -> tuple[list[DriveDocument], str]:
259
  """Scans configured Drive folders and downloads supported documents with metadata."""
260
- folder_ids = _load_folder_ids_from_env()
261
- service = _build_drive_service()
262
-
263
- discovered_files: list[dict[str, Any]] = []
264
- for folder_id in folder_ids:
265
- discovered_files.extend(_list_files_from_folder(service, folder_id, include_subfolders))
266
-
267
- supported_files = [file_meta for file_meta in discovered_files if _is_supported_drive_file(file_meta)]
268
- temp_dir = tempfile.mkdtemp(prefix="drive_docs_")
269
- destination_dir = Path(temp_dir)
270
-
271
- documents: list[DriveDocument] = []
272
- for file_meta in supported_files:
273
- try:
274
- local_path = _download_drive_file(service, file_meta, destination_dir)
275
- except Exception:
276
- # Keep scan resilient if one file is malformed or inaccessible.
277
- continue
278
-
279
- documents.append(
280
- DriveDocument(
281
- document_id=str(file_meta.get("id") or "").strip(),
282
- source_file=str(file_meta.get("name") or "").strip(),
283
- mime_type=str(file_meta.get("mimeType") or "").strip(),
284
- modificationDate=str(file_meta.get("modifiedTime") or "").strip() or None,
285
- local_path=local_path,
286
- )
287
- )
288
-
289
- return documents, temp_dir
290
 
291
 
292
  def cleanup_drive_scan_result(scan_result: DriveScanResult | None) -> None:
 
1
  from __future__ import annotations
2
 
 
 
 
 
3
  import shutil
 
4
  from dataclasses import dataclass
5
  from pathlib import Path
 
6
 
7
  from dotenv import load_dotenv
 
 
 
8
  from firebase_processed_files import FirebaseProcessedFilesTracker, check_files_already_processed
9
+ from google_drive_service import (
10
+ DriveDocument,
11
+ GoogleDriveConfigError,
12
+ load_drive_folder_ids_from_env,
13
+ scan_drive_supported_documents,
14
+ )
15
 
16
 
17
  load_dotenv()
18
 
19
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
20
  @dataclass
21
  class DriveScanResult:
22
  files_to_process: list[str]
 
26
  warning: str | None = None
27
 
28
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
29
  def scan_google_drive_unprocessed_files(
30
  tracker: FirebaseProcessedFilesTracker | None,
31
  include_subfolders: bool = True,
32
  ) -> DriveScanResult:
33
+ folder_ids = load_drive_folder_ids_from_env()
34
+ scan_result = scan_drive_supported_documents(folder_ids=folder_ids, include_subfolders=include_subfolders)
 
 
 
 
35
 
36
+ downloaded_paths = [document.local_path for document in scan_result.documents]
 
 
 
 
 
 
 
 
 
 
37
 
38
  files_to_process, skipped_files, warning = check_files_already_processed(tracker, downloaded_paths)
39
 
 
49
  return DriveScanResult(
50
  files_to_process=files_to_process,
51
  skipped_files=skipped_files,
52
+ discovered_count=scan_result.discovered_count,
53
+ temp_dir=scan_result.temp_dir,
54
  warning=warning,
55
  )
56
 
57
 
58
  def scan_google_drive_supported_documents(include_subfolders: bool = True) -> tuple[list[DriveDocument], str]:
59
  """Scans configured Drive folders and downloads supported documents with metadata."""
60
+ folder_ids = load_drive_folder_ids_from_env()
61
+ scan_result = scan_drive_supported_documents(folder_ids=folder_ids, include_subfolders=include_subfolders)
62
+ return scan_result.documents, scan_result.temp_dir
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
63
 
64
 
65
  def cleanup_drive_scan_result(scan_result: DriveScanResult | None) -> None:
google_drive_service.py ADDED
@@ -0,0 +1,393 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import json
4
+ import logging
5
+ import os
6
+ import re
7
+ import tempfile
8
+ from dataclasses import dataclass
9
+ from pathlib import Path
10
+ from typing import Any
11
+
12
+ from dotenv import load_dotenv
13
+ from google.oauth2.service_account import Credentials
14
+
15
+ from document_processing import SUPPORTED_EXTENSIONS
16
+
17
+
18
+ load_dotenv()
19
+
20
+
21
+ logger = logging.getLogger(__name__)
22
+ if not logger.handlers:
23
+ handler = logging.StreamHandler()
24
+ handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(name)s: %(message)s"))
25
+ logger.addHandler(handler)
26
+ logger.setLevel(logging.INFO)
27
+ logger.propagate = False
28
+
29
+
30
+ DRIVE_READONLY_SCOPE = "https://www.googleapis.com/auth/drive.readonly"
31
+ DRIVE_FULL_SCOPE = "https://www.googleapis.com/auth/drive"
32
+ FOLDER_MIME_TYPE = "application/vnd.google-apps.folder"
33
+
34
+ # Google Workspace files require export, while binary files can be downloaded directly.
35
+ SUPPORTED_EXPORT_MIME_MAP = {
36
+ "application/vnd.google-apps.document": (
37
+ "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
38
+ ".docx",
39
+ ),
40
+ }
41
+
42
+ SUPPORTED_BINARY_MIME_TO_SUFFIX = {
43
+ "application/pdf": ".pdf",
44
+ "text/plain": ".txt",
45
+ "application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx",
46
+ "image/jpeg": ".jpg",
47
+ "image/png": ".png",
48
+ }
49
+
50
+
51
+ @dataclass
52
+ class DriveDocument:
53
+ document_id: str
54
+ source_file: str
55
+ mime_type: str
56
+ modificationDate: str | None
57
+ local_path: str
58
+
59
+
60
+ @dataclass
61
+ class DriveDownloadResult:
62
+ documents: list[DriveDocument]
63
+ discovered_count: int
64
+ temp_dir: str
65
+
66
+
67
+ class GoogleDriveConfigError(RuntimeError):
68
+ """Raised when required Google Drive environment configuration is missing or invalid."""
69
+
70
+
71
+ def load_drive_folder_ids_from_env(env_var: str = "GOOGLE_DRIVE_FOLDER_IDS") -> list[str]:
72
+ raw = (os.getenv(env_var) or "").strip()
73
+ if not raw:
74
+ raise GoogleDriveConfigError(f"Missing {env_var}. Set one or more folder IDs separated by commas.")
75
+
76
+ folder_ids = [item.strip() for item in raw.split(",") if item.strip()]
77
+ if not folder_ids:
78
+ raise GoogleDriveConfigError(f"{env_var} is empty after parsing.")
79
+
80
+ logger.info("Loaded %s folder id(s) from env var %s", len(folder_ids), env_var)
81
+
82
+ return folder_ids
83
+
84
+
85
+ def build_drive_credentials(scope: str = DRIVE_READONLY_SCOPE) -> Credentials:
86
+ service_account_json = (os.getenv("GOOGLE_SERVICE_ACCOUNT_JSON") or "").strip()
87
+ service_account_file = (os.getenv("GOOGLE_SERVICE_ACCOUNT_FILE") or "").strip()
88
+
89
+ if service_account_json:
90
+ try:
91
+ info: dict[str, Any] = json.loads(service_account_json)
92
+ except json.JSONDecodeError as exc:
93
+ raise GoogleDriveConfigError("GOOGLE_SERVICE_ACCOUNT_JSON is not valid JSON.") from exc
94
+ logger.info("Using GOOGLE_SERVICE_ACCOUNT_JSON for Drive credentials")
95
+ return Credentials.from_service_account_info(info, scopes=[scope])
96
+
97
+ if service_account_file:
98
+ logger.info("Using GOOGLE_SERVICE_ACCOUNT_FILE for Drive credentials")
99
+ return Credentials.from_service_account_file(service_account_file, scopes=[scope])
100
+
101
+ raise GoogleDriveConfigError(
102
+ "Missing Google credentials. Set GOOGLE_SERVICE_ACCOUNT_JSON or GOOGLE_SERVICE_ACCOUNT_FILE."
103
+ )
104
+
105
+
106
+ def build_drive_service(scope: str = DRIVE_READONLY_SCOPE):
107
+ try:
108
+ from googleapiclient.discovery import build
109
+ except ImportError as exc:
110
+ raise GoogleDriveConfigError(
111
+ "google-api-python-client is required for Drive scanning. Add it to requirements and install dependencies."
112
+ ) from exc
113
+
114
+ credentials = build_drive_credentials(scope=scope)
115
+ logger.info("Building Drive service with scope=%s", scope)
116
+ return build("drive", "v3", credentials=credentials, cache_discovery=False)
117
+
118
+
119
+ def _escape_drive_query_value(value: str) -> str:
120
+ return value.replace("'", "\\'")
121
+
122
+
123
+ def _sanitize_file_name(name: str) -> str:
124
+ cleaned = re.sub(r'[<>:"/\\|?*\x00-\x1F]', "_", name).strip().rstrip(".")
125
+ return cleaned or "document"
126
+
127
+
128
+ def _ensure_unique_destination(path: Path) -> Path:
129
+ if not path.exists():
130
+ return path
131
+
132
+ counter = 1
133
+ while True:
134
+ candidate = path.with_name(f"{path.stem}_{counter}{path.suffix}")
135
+ if not candidate.exists():
136
+ return candidate
137
+ counter += 1
138
+
139
+
140
+ def _list_files_from_folder(service, folder_id: str, include_subfolders: bool) -> list[dict[str, Any]]:
141
+ logger.info("Listing Drive folder id=%s include_subfolders=%s", folder_id, include_subfolders)
142
+ folders_to_scan = [folder_id]
143
+ discovered: list[dict[str, Any]] = []
144
+
145
+ while folders_to_scan:
146
+ current_folder = folders_to_scan.pop(0)
147
+ page_token = None
148
+
149
+ while True:
150
+ response = (
151
+ service.files()
152
+ .list(
153
+ q=f"'{current_folder}' in parents and trashed = false",
154
+ fields="nextPageToken, files(id, name, mimeType, modifiedTime)",
155
+ supportsAllDrives=True,
156
+ includeItemsFromAllDrives=True,
157
+ pageSize=1000,
158
+ pageToken=page_token,
159
+ )
160
+ .execute()
161
+ )
162
+
163
+ for file_meta in response.get("files", []):
164
+ mime_type = file_meta.get("mimeType")
165
+ if mime_type == FOLDER_MIME_TYPE:
166
+ if include_subfolders:
167
+ child_folder_id = file_meta.get("id")
168
+ if child_folder_id:
169
+ folders_to_scan.append(child_folder_id)
170
+ continue
171
+
172
+ discovered.append(file_meta)
173
+
174
+ page_token = response.get("nextPageToken")
175
+ if not page_token:
176
+ break
177
+
178
+ return discovered
179
+
180
+
181
+ def _is_supported_drive_file(file_meta: dict[str, Any]) -> bool:
182
+ mime_type = (file_meta.get("mimeType") or "").strip().lower()
183
+ if mime_type in SUPPORTED_BINARY_MIME_TO_SUFFIX:
184
+ return True
185
+ if mime_type in SUPPORTED_EXPORT_MIME_MAP:
186
+ return True
187
+
188
+ name = (file_meta.get("name") or "").strip()
189
+ suffix = Path(name).suffix.lower()
190
+ return suffix in SUPPORTED_EXTENSIONS
191
+
192
+
193
+ def _download_drive_file(service, file_meta: dict[str, Any], destination_dir: Path) -> str:
194
+ try:
195
+ from googleapiclient.http import MediaIoBaseDownload
196
+ except ImportError as exc:
197
+ raise GoogleDriveConfigError(
198
+ "google-api-python-client is required for Drive scanning. Add it to requirements and install dependencies."
199
+ ) from exc
200
+
201
+ file_id = (file_meta.get("id") or "").strip()
202
+ file_name = _sanitize_file_name((file_meta.get("name") or "document").strip())
203
+ mime_type = (file_meta.get("mimeType") or "").strip().lower()
204
+
205
+ suffix = Path(file_name).suffix.lower()
206
+ request = None
207
+
208
+ if mime_type in SUPPORTED_EXPORT_MIME_MAP:
209
+ export_mime, export_suffix = SUPPORTED_EXPORT_MIME_MAP[mime_type]
210
+ if suffix != export_suffix:
211
+ file_name = f"{Path(file_name).stem}{export_suffix}"
212
+ suffix = export_suffix
213
+ request = service.files().export_media(fileId=file_id, mimeType=export_mime)
214
+ else:
215
+ if not suffix and mime_type in SUPPORTED_BINARY_MIME_TO_SUFFIX:
216
+ file_name = f"{file_name}{SUPPORTED_BINARY_MIME_TO_SUFFIX[mime_type]}"
217
+ suffix = Path(file_name).suffix.lower()
218
+ request = service.files().get_media(fileId=file_id)
219
+
220
+ if suffix not in SUPPORTED_EXTENSIONS:
221
+ raise ValueError(f"Unsupported downloaded file type: {suffix or '[no extension]'}")
222
+
223
+ destination = _ensure_unique_destination(destination_dir / file_name)
224
+
225
+ with destination.open("wb") as stream:
226
+ downloader = MediaIoBaseDownload(stream, request)
227
+ done = False
228
+ while not done:
229
+ _, done = downloader.next_chunk()
230
+
231
+ return str(destination)
232
+
233
+
234
+ def scan_drive_supported_documents(folder_ids: list[str], include_subfolders: bool = True) -> DriveDownloadResult:
235
+ service = build_drive_service()
236
+ logger.info("Starting Drive scan for %s folder(s)", len(folder_ids))
237
+
238
+ discovered_files: list[dict[str, Any]] = []
239
+ for folder_id in folder_ids:
240
+ discovered_files.extend(_list_files_from_folder(service, folder_id, include_subfolders))
241
+
242
+ supported_files = [file_meta for file_meta in discovered_files if _is_supported_drive_file(file_meta)]
243
+ logger.info("Drive scan discovered=%s supported=%s", len(discovered_files), len(supported_files))
244
+ temp_dir = tempfile.mkdtemp(prefix="drive_docs_")
245
+ destination_dir = Path(temp_dir)
246
+
247
+ documents: list[DriveDocument] = []
248
+ for file_meta in supported_files:
249
+ try:
250
+ local_path = _download_drive_file(service, file_meta, destination_dir)
251
+ except Exception:
252
+ # Keep scan resilient if one file is malformed or inaccessible.
253
+ logger.exception("Skipping file during Drive download due to error file_id=%s", file_meta.get("id"))
254
+ continue
255
+
256
+ documents.append(
257
+ DriveDocument(
258
+ document_id=str(file_meta.get("id") or "").strip(),
259
+ source_file=str(file_meta.get("name") or "").strip(),
260
+ mime_type=str(file_meta.get("mimeType") or "").strip(),
261
+ modificationDate=str(file_meta.get("modifiedTime") or "").strip() or None,
262
+ local_path=local_path,
263
+ )
264
+ )
265
+
266
+ return DriveDownloadResult(
267
+ documents=documents,
268
+ discovered_count=len(supported_files),
269
+ temp_dir=temp_dir,
270
+ )
271
+
272
+
273
+ def _find_folder_in_parent(service, folder_name: str, parent_folder_id: str | None) -> str | None:
274
+ query_parts = [
275
+ "trashed = false",
276
+ f"mimeType = '{FOLDER_MIME_TYPE}'",
277
+ f"name = '{_escape_drive_query_value(folder_name)}'",
278
+ ]
279
+ if parent_folder_id:
280
+ query_parts.append(f"'{parent_folder_id}' in parents")
281
+
282
+ try:
283
+ response = (
284
+ service.files()
285
+ .list(
286
+ q=" and ".join(query_parts),
287
+ fields="files(id,name)",
288
+ supportsAllDrives=True,
289
+ includeItemsFromAllDrives=True,
290
+ pageSize=10,
291
+ )
292
+ .execute()
293
+ )
294
+ except Exception as exc:
295
+ raise GoogleDriveConfigError(f"Failed to query folder '{folder_name}': {exc}") from exc
296
+
297
+ files = response.get("files", [])
298
+ if not files:
299
+ logger.info("Drive folder not found name=%s parent=%s", folder_name, parent_folder_id or "root")
300
+ return None
301
+
302
+ logger.info("Found Drive folder name=%s parent=%s", folder_name, parent_folder_id or "root")
303
+ return str(files[0].get("id") or "").strip() or None
304
+
305
+
306
+ def _create_folder_in_parent(service, folder_name: str, parent_folder_id: str | None) -> str:
307
+ body: dict[str, Any] = {"name": folder_name, "mimeType": FOLDER_MIME_TYPE}
308
+ if parent_folder_id:
309
+ body["parents"] = [parent_folder_id]
310
+
311
+ try:
312
+ created = (
313
+ service.files()
314
+ .create(body=body, fields="id", supportsAllDrives=True)
315
+ .execute()
316
+ )
317
+ except Exception as exc:
318
+ raise GoogleDriveConfigError(
319
+ f"Failed to create folder '{folder_name}' under parent '{parent_folder_id or 'root'}': {exc}"
320
+ ) from exc
321
+
322
+ folder_id = str(created.get("id") or "").strip()
323
+ if not folder_id:
324
+ raise GoogleDriveConfigError(f"Folder '{folder_name}' was created but no id was returned.")
325
+
326
+ logger.info("Created Drive folder name=%s parent=%s id=%s", folder_name, parent_folder_id or "root", folder_id)
327
+
328
+ return folder_id
329
+
330
+
331
+ def ensure_drive_folder_path(service, folder_path: str, root_folder_id: str | None = None) -> str:
332
+ normalized = (folder_path or "").strip().strip("/")
333
+ if not normalized:
334
+ raise GoogleDriveConfigError("folder_path is required to resolve a Drive destination.")
335
+
336
+ current_parent = (root_folder_id or "").strip() or None
337
+ for part in [segment.strip() for segment in normalized.split("/") if segment.strip()]:
338
+ folder_id = _find_folder_in_parent(service, part, current_parent)
339
+ if folder_id is None:
340
+ folder_id = _create_folder_in_parent(service, part, current_parent)
341
+ current_parent = folder_id
342
+
343
+ if not current_parent:
344
+ raise GoogleDriveConfigError(f"Could not resolve Drive path '{folder_path}'.")
345
+
346
+ logger.info("Resolved Drive path=%s to folder_id=%s", folder_path, current_parent)
347
+
348
+ return current_parent
349
+
350
+
351
+ def move_file_to_path(service, file_id: str, destination_path: str, root_folder_id: str | None = None) -> str:
352
+ normalized_file_id = (file_id or "").strip()
353
+ if not normalized_file_id:
354
+ raise GoogleDriveConfigError("file_id is required to move a Drive file.")
355
+
356
+ destination_folder_id = ensure_drive_folder_path(service, destination_path, root_folder_id=root_folder_id)
357
+ logger.info("Moving Drive file_id=%s to destination_path=%s", normalized_file_id, destination_path)
358
+
359
+ try:
360
+ metadata = (
361
+ service.files()
362
+ .get(fileId=normalized_file_id, fields="parents", supportsAllDrives=True)
363
+ .execute()
364
+ )
365
+ except Exception as exc:
366
+ raise GoogleDriveConfigError(f"Failed to read current parents for file '{normalized_file_id}': {exc}") from exc
367
+
368
+ current_parents = [str(parent).strip() for parent in metadata.get("parents", []) if str(parent).strip()]
369
+
370
+ if destination_folder_id in current_parents and len(current_parents) == 1:
371
+ logger.info("Drive file already in destination file_id=%s folder_id=%s", normalized_file_id, destination_folder_id)
372
+ return destination_folder_id
373
+
374
+ request_kwargs: dict[str, Any] = {
375
+ "fileId": normalized_file_id,
376
+ "addParents": destination_folder_id,
377
+ "supportsAllDrives": True,
378
+ "fields": "id,parents",
379
+ }
380
+ remove_parents = ",".join(parent for parent in current_parents if parent != destination_folder_id)
381
+ if remove_parents:
382
+ request_kwargs["removeParents"] = remove_parents
383
+
384
+ try:
385
+ service.files().update(**request_kwargs).execute()
386
+ except Exception as exc:
387
+ raise GoogleDriveConfigError(
388
+ f"Failed to move file '{normalized_file_id}' to Drive path '{destination_path}': {exc}"
389
+ ) from exc
390
+
391
+ logger.info("Moved Drive file_id=%s to folder_id=%s", normalized_file_id, destination_folder_id)
392
+
393
+ return destination_folder_id