Spaces:
Sleeping
Sleeping
segovia commited on
Commit ·
cb04ca0
1
Parent(s): 207d271
Add drive service
Browse filesmove files from the source folder to another one
- document_orchester.py +99 -5
- google_drive_scanner.py +14 -241
- google_drive_service.py +393 -0
document_orchester.py
CHANGED
|
@@ -5,6 +5,7 @@ import json
|
|
| 5 |
import logging
|
| 6 |
import os
|
| 7 |
import shutil
|
|
|
|
| 8 |
from pathlib import Path
|
| 9 |
from typing import Any
|
| 10 |
|
|
@@ -16,8 +17,13 @@ from firebase_processed_files import (
|
|
| 16 |
build_firebase_tracker,
|
| 17 |
check_drive_documents_to_process,
|
| 18 |
)
|
| 19 |
-
from
|
|
|
|
| 20 |
GoogleDriveConfigError,
|
|
|
|
|
|
|
|
|
|
|
|
|
| 21 |
scan_google_drive_supported_documents,
|
| 22 |
)
|
| 23 |
from google_sheets_service import GoogleSheetsConfigError, append_row_to_google_sheet
|
|
@@ -39,6 +45,14 @@ logger.propagate = False
|
|
| 39 |
|
| 40 |
SHEET_COLUMNS = ["fecha", "source_file", "display_description", "total"]
|
| 41 |
CORRUPTED_SHEET_NAME = "corrupted data"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 42 |
|
| 43 |
|
| 44 |
def _build_display_description(parsed: dict[str, Any]) -> str:
|
|
@@ -79,6 +93,60 @@ def _create_openai_client() -> OpenAI:
|
|
| 79 |
return OpenAI(api_key=api_key)
|
| 80 |
|
| 81 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 82 |
def _parse_document(client: OpenAI, model: str, local_path: str, source_file: str) -> dict[str, Any]:
|
| 83 |
path = Path(local_path)
|
| 84 |
if is_image_document(path):
|
|
@@ -105,11 +173,14 @@ def orchestrate_drive_documents(
|
|
| 105 |
include_subfolders,
|
| 106 |
send_to_sheet,
|
| 107 |
)
|
|
|
|
| 108 |
tracker = build_firebase_tracker()
|
| 109 |
client = _create_openai_client()
|
|
|
|
| 110 |
|
| 111 |
scan_temp_dir: str | None = None
|
| 112 |
try:
|
|
|
|
| 113 |
drive_documents, scan_temp_dir = scan_google_drive_supported_documents(include_subfolders=include_subfolders)
|
| 114 |
except GoogleDriveConfigError as exc:
|
| 115 |
logger.error("Google Drive configuration error: %s", exc)
|
|
@@ -138,6 +209,7 @@ def orchestrate_drive_documents(
|
|
| 138 |
parsed_count = 0
|
| 139 |
modified_count = 0
|
| 140 |
sent_count = 0
|
|
|
|
| 141 |
corrupted_count = 0
|
| 142 |
error_items: list[dict[str, str]] = []
|
| 143 |
processed_items: list[dict[str, Any]] = []
|
|
@@ -154,6 +226,23 @@ def orchestrate_drive_documents(
|
|
| 154 |
parsed = _parse_document(client, model, local_path, source_file)
|
| 155 |
logger.info("Parsed document_id=%s source_file=%s", document_id, source_file)
|
| 156 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 157 |
if tracker is not None:
|
| 158 |
tracker.save_document_record(
|
| 159 |
document_id=document_id,
|
|
@@ -171,6 +260,10 @@ def orchestrate_drive_documents(
|
|
| 171 |
worksheet_name: str | None = None
|
| 172 |
if send_to_sheet:
|
| 173 |
row_for_sheet = _build_sheet_row(parsed)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 174 |
worksheet_name = None if _is_complete_sheet_row(row_for_sheet) else CORRUPTED_SHEET_NAME
|
| 175 |
append_row_to_google_sheet(row_for_sheet, SHEET_COLUMNS, worksheet_name=worksheet_name)
|
| 176 |
logger.info(
|
|
@@ -182,15 +275,14 @@ def orchestrate_drive_documents(
|
|
| 182 |
if tracker is not None and worksheet_name is None:
|
| 183 |
tracker.mark_document_sent(document_id)
|
| 184 |
sent_count += 1
|
| 185 |
-
if worksheet_name is not None:
|
| 186 |
-
corrupted_count += 1
|
| 187 |
|
| 188 |
processed_items.append(
|
| 189 |
{
|
| 190 |
"document_id": document_id,
|
| 191 |
"source_file": source_file,
|
| 192 |
"modificationDate": modification_date,
|
| 193 |
-
"status": "Corrupted" if
|
|
|
|
| 194 |
}
|
| 195 |
)
|
| 196 |
except GoogleSheetsConfigError as exc:
|
|
@@ -217,10 +309,11 @@ def orchestrate_drive_documents(
|
|
| 217 |
logger.info("Cleaned up temporary scan directory")
|
| 218 |
|
| 219 |
logger.info(
|
| 220 |
-
"Orchestration finished parsed=%s modified=%s sent=%s corrupted=%s errors=%s",
|
| 221 |
parsed_count,
|
| 222 |
modified_count,
|
| 223 |
sent_count,
|
|
|
|
| 224 |
corrupted_count,
|
| 225 |
len(error_items),
|
| 226 |
)
|
|
@@ -233,6 +326,7 @@ def orchestrate_drive_documents(
|
|
| 233 |
"parsed": parsed_count,
|
| 234 |
"modified": modified_count,
|
| 235 |
"sent": sent_count,
|
|
|
|
| 236 |
"corrupted": corrupted_count,
|
| 237 |
"tracking_warning": tracking_warning,
|
| 238 |
"processed_items": processed_items,
|
|
|
|
| 5 |
import logging
|
| 6 |
import os
|
| 7 |
import shutil
|
| 8 |
+
from datetime import datetime
|
| 9 |
from pathlib import Path
|
| 10 |
from typing import Any
|
| 11 |
|
|
|
|
| 17 |
build_firebase_tracker,
|
| 18 |
check_drive_documents_to_process,
|
| 19 |
)
|
| 20 |
+
from google_drive_service import (
|
| 21 |
+
DRIVE_FULL_SCOPE,
|
| 22 |
GoogleDriveConfigError,
|
| 23 |
+
build_drive_service,
|
| 24 |
+
move_file_to_path,
|
| 25 |
+
)
|
| 26 |
+
from google_drive_scanner import (
|
| 27 |
scan_google_drive_supported_documents,
|
| 28 |
)
|
| 29 |
from google_sheets_service import GoogleSheetsConfigError, append_row_to_google_sheet
|
|
|
|
| 45 |
|
| 46 |
SHEET_COLUMNS = ["fecha", "source_file", "display_description", "total"]
|
| 47 |
CORRUPTED_SHEET_NAME = "corrupted data"
|
| 48 |
+
MANDATORY_ENV_VARS = [
|
| 49 |
+
"OPENAI_API_KEY",
|
| 50 |
+
"GOOGLE_SERVICE_ACCOUNT_JSON",
|
| 51 |
+
"GOOGLE_SHEETS_SPREADSHEET_ID",
|
| 52 |
+
"GOOGLE_DRIVE_FOLDER_IDS",
|
| 53 |
+
"GOOGLE_DRIVE_INVOICES_ROOT_FOLDER_ID",
|
| 54 |
+
"ORCHESTRATOR_API_KEY",
|
| 55 |
+
]
|
| 56 |
|
| 57 |
|
| 58 |
def _build_display_description(parsed: dict[str, Any]) -> str:
|
|
|
|
| 93 |
return OpenAI(api_key=api_key)
|
| 94 |
|
| 95 |
|
| 96 |
+
def _require_env_var(name: str) -> str:
|
| 97 |
+
value = (os.getenv(name) or "").strip()
|
| 98 |
+
if not value:
|
| 99 |
+
raise RuntimeError(f"Missing {name}.")
|
| 100 |
+
return value
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
def _validate_mandatory_env_vars() -> None:
|
| 104 |
+
missing_vars = [name for name in MANDATORY_ENV_VARS if not (os.getenv(name) or "").strip()]
|
| 105 |
+
if missing_vars:
|
| 106 |
+
raise RuntimeError(f"Missing mandatory env vars: {', '.join(missing_vars)}")
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
def _parse_invoice_date(value: Any) -> datetime | None:
|
| 110 |
+
text = str(value or "").strip()
|
| 111 |
+
if not text:
|
| 112 |
+
return None
|
| 113 |
+
|
| 114 |
+
for fmt in ("%Y-%m-%d", "%d/%m/%Y", "%d-%m-%Y", "%Y/%m/%d"):
|
| 115 |
+
try:
|
| 116 |
+
return datetime.strptime(text, fmt)
|
| 117 |
+
except ValueError:
|
| 118 |
+
continue
|
| 119 |
+
|
| 120 |
+
try:
|
| 121 |
+
return datetime.fromisoformat(text.replace("Z", "+00:00"))
|
| 122 |
+
except ValueError:
|
| 123 |
+
return None
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
def _resolve_drive_destination_path(parsed: dict[str, Any]) -> tuple[str, bool]:
|
| 127 |
+
base_path = (os.getenv("GOOGLE_DRIVE_INVOICES_BASE_PATH") or "Facturas").strip().strip("/") or "Facturas"
|
| 128 |
+
|
| 129 |
+
row_for_sheet = _build_sheet_row(parsed)
|
| 130 |
+
if not _is_complete_sheet_row(row_for_sheet):
|
| 131 |
+
return f"Corrupted", True
|
| 132 |
+
|
| 133 |
+
invoice_date = _parse_invoice_date(parsed.get("fecha"))
|
| 134 |
+
if invoice_date is None:
|
| 135 |
+
return f"Corrupted", True
|
| 136 |
+
|
| 137 |
+
return f"{invoice_date.strftime('%Y%m')}", False
|
| 138 |
+
|
| 139 |
+
|
| 140 |
+
def _build_drive_file_url(file_id: str) -> str:
|
| 141 |
+
return f"https://drive.google.com/file/d/{file_id}/view"
|
| 142 |
+
|
| 143 |
+
|
| 144 |
+
def _build_hyperlink_formula(url: str, label: str) -> str:
|
| 145 |
+
safe_url = str(url).replace('"', '""')
|
| 146 |
+
safe_label = str(label).replace('"', '""')
|
| 147 |
+
return f'=HYPERLINK("{safe_url}","{safe_label}")'
|
| 148 |
+
|
| 149 |
+
|
| 150 |
def _parse_document(client: OpenAI, model: str, local_path: str, source_file: str) -> dict[str, Any]:
|
| 151 |
path = Path(local_path)
|
| 152 |
if is_image_document(path):
|
|
|
|
| 173 |
include_subfolders,
|
| 174 |
send_to_sheet,
|
| 175 |
)
|
| 176 |
+
_validate_mandatory_env_vars()
|
| 177 |
tracker = build_firebase_tracker()
|
| 178 |
client = _create_openai_client()
|
| 179 |
+
drive_root_folder_id = _require_env_var("GOOGLE_DRIVE_INVOICES_ROOT_FOLDER_ID")
|
| 180 |
|
| 181 |
scan_temp_dir: str | None = None
|
| 182 |
try:
|
| 183 |
+
drive_service = build_drive_service(scope=DRIVE_FULL_SCOPE)
|
| 184 |
drive_documents, scan_temp_dir = scan_google_drive_supported_documents(include_subfolders=include_subfolders)
|
| 185 |
except GoogleDriveConfigError as exc:
|
| 186 |
logger.error("Google Drive configuration error: %s", exc)
|
|
|
|
| 209 |
parsed_count = 0
|
| 210 |
modified_count = 0
|
| 211 |
sent_count = 0
|
| 212 |
+
moved_count = 0
|
| 213 |
corrupted_count = 0
|
| 214 |
error_items: list[dict[str, str]] = []
|
| 215 |
processed_items: list[dict[str, Any]] = []
|
|
|
|
| 226 |
parsed = _parse_document(client, model, local_path, source_file)
|
| 227 |
logger.info("Parsed document_id=%s source_file=%s", document_id, source_file)
|
| 228 |
|
| 229 |
+
destination_path, is_corrupted = _resolve_drive_destination_path(parsed)
|
| 230 |
+
if is_corrupted:
|
| 231 |
+
corrupted_count += 1
|
| 232 |
+
move_file_to_path(
|
| 233 |
+
service=drive_service,
|
| 234 |
+
file_id=document_id,
|
| 235 |
+
destination_path=destination_path,
|
| 236 |
+
root_folder_id=drive_root_folder_id,
|
| 237 |
+
)
|
| 238 |
+
moved_count += 1
|
| 239 |
+
logger.info(
|
| 240 |
+
"Moved document_id=%s source_file=%s to drive_path=%s",
|
| 241 |
+
document_id,
|
| 242 |
+
source_file,
|
| 243 |
+
destination_path,
|
| 244 |
+
)
|
| 245 |
+
|
| 246 |
if tracker is not None:
|
| 247 |
tracker.save_document_record(
|
| 248 |
document_id=document_id,
|
|
|
|
| 260 |
worksheet_name: str | None = None
|
| 261 |
if send_to_sheet:
|
| 262 |
row_for_sheet = _build_sheet_row(parsed)
|
| 263 |
+
row_for_sheet["source_file"] = _build_hyperlink_formula(
|
| 264 |
+
_build_drive_file_url(document_id),
|
| 265 |
+
source_file,
|
| 266 |
+
)
|
| 267 |
worksheet_name = None if _is_complete_sheet_row(row_for_sheet) else CORRUPTED_SHEET_NAME
|
| 268 |
append_row_to_google_sheet(row_for_sheet, SHEET_COLUMNS, worksheet_name=worksheet_name)
|
| 269 |
logger.info(
|
|
|
|
| 275 |
if tracker is not None and worksheet_name is None:
|
| 276 |
tracker.mark_document_sent(document_id)
|
| 277 |
sent_count += 1
|
|
|
|
|
|
|
| 278 |
|
| 279 |
processed_items.append(
|
| 280 |
{
|
| 281 |
"document_id": document_id,
|
| 282 |
"source_file": source_file,
|
| 283 |
"modificationDate": modification_date,
|
| 284 |
+
"status": "Corrupted" if is_corrupted else ("Sent" if send_to_sheet else status),
|
| 285 |
+
"destination_path": destination_path,
|
| 286 |
}
|
| 287 |
)
|
| 288 |
except GoogleSheetsConfigError as exc:
|
|
|
|
| 309 |
logger.info("Cleaned up temporary scan directory")
|
| 310 |
|
| 311 |
logger.info(
|
| 312 |
+
"Orchestration finished parsed=%s modified=%s sent=%s moved=%s corrupted=%s errors=%s",
|
| 313 |
parsed_count,
|
| 314 |
modified_count,
|
| 315 |
sent_count,
|
| 316 |
+
moved_count,
|
| 317 |
corrupted_count,
|
| 318 |
len(error_items),
|
| 319 |
)
|
|
|
|
| 326 |
"parsed": parsed_count,
|
| 327 |
"modified": modified_count,
|
| 328 |
"sent": sent_count,
|
| 329 |
+
"moved": moved_count,
|
| 330 |
"corrupted": corrupted_count,
|
| 331 |
"tracking_warning": tracking_warning,
|
| 332 |
"processed_items": processed_items,
|
google_drive_scanner.py
CHANGED
|
@@ -1,46 +1,22 @@
|
|
| 1 |
from __future__ import annotations
|
| 2 |
|
| 3 |
-
import io
|
| 4 |
-
import json
|
| 5 |
-
import os
|
| 6 |
-
import re
|
| 7 |
import shutil
|
| 8 |
-
import tempfile
|
| 9 |
from dataclasses import dataclass
|
| 10 |
from pathlib import Path
|
| 11 |
-
from typing import Any
|
| 12 |
|
| 13 |
from dotenv import load_dotenv
|
| 14 |
-
from google.oauth2.service_account import Credentials
|
| 15 |
-
|
| 16 |
-
from document_processing import SUPPORTED_EXTENSIONS
|
| 17 |
from firebase_processed_files import FirebaseProcessedFilesTracker, check_files_already_processed
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 18 |
|
| 19 |
|
| 20 |
load_dotenv()
|
| 21 |
|
| 22 |
|
| 23 |
-
DRIVE_READONLY_SCOPE = "https://www.googleapis.com/auth/drive.readonly"
|
| 24 |
-
FOLDER_MIME_TYPE = "application/vnd.google-apps.folder"
|
| 25 |
-
|
| 26 |
-
# Google Workspace files require export, while binary files can be downloaded directly.
|
| 27 |
-
SUPPORTED_EXPORT_MIME_MAP = {
|
| 28 |
-
"application/vnd.google-apps.document": ("application/vnd.openxmlformats-officedocument.wordprocessingml.document", ".docx"),
|
| 29 |
-
}
|
| 30 |
-
|
| 31 |
-
SUPPORTED_BINARY_MIME_TO_SUFFIX = {
|
| 32 |
-
"application/pdf": ".pdf",
|
| 33 |
-
"text/plain": ".txt",
|
| 34 |
-
"application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx",
|
| 35 |
-
"image/jpeg": ".jpg",
|
| 36 |
-
"image/png": ".png",
|
| 37 |
-
}
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
class GoogleDriveConfigError(RuntimeError):
|
| 41 |
-
"""Raised when required Google Drive environment configuration is missing or invalid."""
|
| 42 |
-
|
| 43 |
-
|
| 44 |
@dataclass
|
| 45 |
class DriveScanResult:
|
| 46 |
files_to_process: list[str]
|
|
@@ -50,190 +26,14 @@ class DriveScanResult:
|
|
| 50 |
warning: str | None = None
|
| 51 |
|
| 52 |
|
| 53 |
-
@dataclass
|
| 54 |
-
class DriveDocument:
|
| 55 |
-
document_id: str
|
| 56 |
-
source_file: str
|
| 57 |
-
mime_type: str
|
| 58 |
-
modificationDate: str | None
|
| 59 |
-
local_path: str
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
def _build_credentials() -> Credentials:
|
| 63 |
-
service_account_json = (os.getenv("GOOGLE_SERVICE_ACCOUNT_JSON") or "").strip()
|
| 64 |
-
service_account_file = (os.getenv("GOOGLE_SERVICE_ACCOUNT_FILE") or "").strip()
|
| 65 |
-
|
| 66 |
-
if service_account_json:
|
| 67 |
-
try:
|
| 68 |
-
info = json.loads(service_account_json)
|
| 69 |
-
except json.JSONDecodeError as exc:
|
| 70 |
-
raise GoogleDriveConfigError("GOOGLE_SERVICE_ACCOUNT_JSON is not valid JSON.") from exc
|
| 71 |
-
return Credentials.from_service_account_info(info, scopes=[DRIVE_READONLY_SCOPE])
|
| 72 |
-
|
| 73 |
-
if service_account_file:
|
| 74 |
-
return Credentials.from_service_account_file(service_account_file, scopes=[DRIVE_READONLY_SCOPE])
|
| 75 |
-
|
| 76 |
-
raise GoogleDriveConfigError(
|
| 77 |
-
"Missing Google credentials. Set GOOGLE_SERVICE_ACCOUNT_JSON or GOOGLE_SERVICE_ACCOUNT_FILE."
|
| 78 |
-
)
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
def _build_drive_service():
|
| 82 |
-
try:
|
| 83 |
-
from googleapiclient.discovery import build
|
| 84 |
-
except ImportError as exc:
|
| 85 |
-
raise GoogleDriveConfigError(
|
| 86 |
-
"google-api-python-client is required for Drive scanning. Add it to requirements and install dependencies."
|
| 87 |
-
) from exc
|
| 88 |
-
|
| 89 |
-
credentials = _build_credentials()
|
| 90 |
-
return build("drive", "v3", credentials=credentials, cache_discovery=False)
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
def _load_folder_ids_from_env() -> list[str]:
|
| 94 |
-
raw = (os.getenv("GOOGLE_DRIVE_FOLDER_IDS") or "").strip()
|
| 95 |
-
if not raw:
|
| 96 |
-
raise GoogleDriveConfigError("Missing GOOGLE_DRIVE_FOLDER_IDS. Set one or more folder IDs separated by commas.")
|
| 97 |
-
|
| 98 |
-
folder_ids = [item.strip() for item in raw.split(",") if item.strip()]
|
| 99 |
-
if not folder_ids:
|
| 100 |
-
raise GoogleDriveConfigError("GOOGLE_DRIVE_FOLDER_IDS is empty after parsing.")
|
| 101 |
-
|
| 102 |
-
return folder_ids
|
| 103 |
-
|
| 104 |
-
|
| 105 |
-
def _sanitize_file_name(name: str) -> str:
|
| 106 |
-
cleaned = re.sub(r'[<>:"/\\|?*\x00-\x1F]', "_", name).strip().rstrip(".")
|
| 107 |
-
return cleaned or "document"
|
| 108 |
-
|
| 109 |
-
|
| 110 |
-
def _ensure_unique_destination(path: Path) -> Path:
|
| 111 |
-
if not path.exists():
|
| 112 |
-
return path
|
| 113 |
-
|
| 114 |
-
counter = 1
|
| 115 |
-
while True:
|
| 116 |
-
candidate = path.with_name(f"{path.stem}_{counter}{path.suffix}")
|
| 117 |
-
if not candidate.exists():
|
| 118 |
-
return candidate
|
| 119 |
-
counter += 1
|
| 120 |
-
|
| 121 |
-
|
| 122 |
-
def _list_files_from_folder(service, folder_id: str, include_subfolders: bool) -> list[dict[str, Any]]:
|
| 123 |
-
folders_to_scan = [folder_id]
|
| 124 |
-
discovered: list[dict[str, Any]] = []
|
| 125 |
-
|
| 126 |
-
while folders_to_scan:
|
| 127 |
-
current_folder = folders_to_scan.pop(0)
|
| 128 |
-
page_token = None
|
| 129 |
-
|
| 130 |
-
while True:
|
| 131 |
-
response = (
|
| 132 |
-
service.files()
|
| 133 |
-
.list(
|
| 134 |
-
q=f"'{current_folder}' in parents and trashed = false",
|
| 135 |
-
fields="nextPageToken, files(id, name, mimeType, modifiedTime)",
|
| 136 |
-
supportsAllDrives=True,
|
| 137 |
-
includeItemsFromAllDrives=True,
|
| 138 |
-
pageSize=1000,
|
| 139 |
-
pageToken=page_token,
|
| 140 |
-
)
|
| 141 |
-
.execute()
|
| 142 |
-
)
|
| 143 |
-
|
| 144 |
-
for file_meta in response.get("files", []):
|
| 145 |
-
mime_type = file_meta.get("mimeType")
|
| 146 |
-
if mime_type == FOLDER_MIME_TYPE:
|
| 147 |
-
if include_subfolders:
|
| 148 |
-
child_folder_id = file_meta.get("id")
|
| 149 |
-
if child_folder_id:
|
| 150 |
-
folders_to_scan.append(child_folder_id)
|
| 151 |
-
continue
|
| 152 |
-
|
| 153 |
-
discovered.append(file_meta)
|
| 154 |
-
|
| 155 |
-
page_token = response.get("nextPageToken")
|
| 156 |
-
if not page_token:
|
| 157 |
-
break
|
| 158 |
-
|
| 159 |
-
return discovered
|
| 160 |
-
|
| 161 |
-
|
| 162 |
-
def _is_supported_drive_file(file_meta: dict[str, Any]) -> bool:
|
| 163 |
-
mime_type = (file_meta.get("mimeType") or "").strip().lower()
|
| 164 |
-
if mime_type in SUPPORTED_BINARY_MIME_TO_SUFFIX:
|
| 165 |
-
return True
|
| 166 |
-
if mime_type in SUPPORTED_EXPORT_MIME_MAP:
|
| 167 |
-
return True
|
| 168 |
-
|
| 169 |
-
name = (file_meta.get("name") or "").strip()
|
| 170 |
-
suffix = Path(name).suffix.lower()
|
| 171 |
-
return suffix in SUPPORTED_EXTENSIONS
|
| 172 |
-
|
| 173 |
-
|
| 174 |
-
def _download_drive_file(service, file_meta: dict[str, Any], destination_dir: Path) -> str:
|
| 175 |
-
try:
|
| 176 |
-
from googleapiclient.http import MediaIoBaseDownload
|
| 177 |
-
except ImportError as exc:
|
| 178 |
-
raise GoogleDriveConfigError(
|
| 179 |
-
"google-api-python-client is required for Drive scanning. Add it to requirements and install dependencies."
|
| 180 |
-
) from exc
|
| 181 |
-
|
| 182 |
-
file_id = (file_meta.get("id") or "").strip()
|
| 183 |
-
file_name = _sanitize_file_name((file_meta.get("name") or "document").strip())
|
| 184 |
-
mime_type = (file_meta.get("mimeType") or "").strip().lower()
|
| 185 |
-
|
| 186 |
-
suffix = Path(file_name).suffix.lower()
|
| 187 |
-
request = None
|
| 188 |
-
|
| 189 |
-
if mime_type in SUPPORTED_EXPORT_MIME_MAP:
|
| 190 |
-
export_mime, export_suffix = SUPPORTED_EXPORT_MIME_MAP[mime_type]
|
| 191 |
-
if suffix != export_suffix:
|
| 192 |
-
file_name = f"{Path(file_name).stem}{export_suffix}"
|
| 193 |
-
suffix = export_suffix
|
| 194 |
-
request = service.files().export_media(fileId=file_id, mimeType=export_mime)
|
| 195 |
-
else:
|
| 196 |
-
if not suffix and mime_type in SUPPORTED_BINARY_MIME_TO_SUFFIX:
|
| 197 |
-
file_name = f"{file_name}{SUPPORTED_BINARY_MIME_TO_SUFFIX[mime_type]}"
|
| 198 |
-
suffix = Path(file_name).suffix.lower()
|
| 199 |
-
request = service.files().get_media(fileId=file_id)
|
| 200 |
-
|
| 201 |
-
if suffix not in SUPPORTED_EXTENSIONS:
|
| 202 |
-
raise ValueError(f"Unsupported downloaded file type: {suffix or '[no extension]'}")
|
| 203 |
-
|
| 204 |
-
destination = _ensure_unique_destination(destination_dir / file_name)
|
| 205 |
-
|
| 206 |
-
with destination.open("wb") as stream:
|
| 207 |
-
downloader = MediaIoBaseDownload(stream, request)
|
| 208 |
-
done = False
|
| 209 |
-
while not done:
|
| 210 |
-
_, done = downloader.next_chunk()
|
| 211 |
-
|
| 212 |
-
return str(destination)
|
| 213 |
-
|
| 214 |
-
|
| 215 |
def scan_google_drive_unprocessed_files(
|
| 216 |
tracker: FirebaseProcessedFilesTracker | None,
|
| 217 |
include_subfolders: bool = True,
|
| 218 |
) -> DriveScanResult:
|
| 219 |
-
folder_ids =
|
| 220 |
-
|
| 221 |
-
|
| 222 |
-
discovered_files: list[dict[str, Any]] = []
|
| 223 |
-
for folder_id in folder_ids:
|
| 224 |
-
discovered_files.extend(_list_files_from_folder(service, folder_id, include_subfolders))
|
| 225 |
|
| 226 |
-
|
| 227 |
-
temp_dir = tempfile.mkdtemp(prefix="drive_docs_")
|
| 228 |
-
destination_dir = Path(temp_dir)
|
| 229 |
-
|
| 230 |
-
downloaded_paths: list[str] = []
|
| 231 |
-
for file_meta in supported_files:
|
| 232 |
-
try:
|
| 233 |
-
downloaded_paths.append(_download_drive_file(service, file_meta, destination_dir))
|
| 234 |
-
except Exception:
|
| 235 |
-
# Keep scan resilient if one file is malformed or inaccessible.
|
| 236 |
-
continue
|
| 237 |
|
| 238 |
files_to_process, skipped_files, warning = check_files_already_processed(tracker, downloaded_paths)
|
| 239 |
|
|
@@ -249,44 +49,17 @@ def scan_google_drive_unprocessed_files(
|
|
| 249 |
return DriveScanResult(
|
| 250 |
files_to_process=files_to_process,
|
| 251 |
skipped_files=skipped_files,
|
| 252 |
-
discovered_count=
|
| 253 |
-
temp_dir=temp_dir,
|
| 254 |
warning=warning,
|
| 255 |
)
|
| 256 |
|
| 257 |
|
| 258 |
def scan_google_drive_supported_documents(include_subfolders: bool = True) -> tuple[list[DriveDocument], str]:
|
| 259 |
"""Scans configured Drive folders and downloads supported documents with metadata."""
|
| 260 |
-
folder_ids =
|
| 261 |
-
|
| 262 |
-
|
| 263 |
-
discovered_files: list[dict[str, Any]] = []
|
| 264 |
-
for folder_id in folder_ids:
|
| 265 |
-
discovered_files.extend(_list_files_from_folder(service, folder_id, include_subfolders))
|
| 266 |
-
|
| 267 |
-
supported_files = [file_meta for file_meta in discovered_files if _is_supported_drive_file(file_meta)]
|
| 268 |
-
temp_dir = tempfile.mkdtemp(prefix="drive_docs_")
|
| 269 |
-
destination_dir = Path(temp_dir)
|
| 270 |
-
|
| 271 |
-
documents: list[DriveDocument] = []
|
| 272 |
-
for file_meta in supported_files:
|
| 273 |
-
try:
|
| 274 |
-
local_path = _download_drive_file(service, file_meta, destination_dir)
|
| 275 |
-
except Exception:
|
| 276 |
-
# Keep scan resilient if one file is malformed or inaccessible.
|
| 277 |
-
continue
|
| 278 |
-
|
| 279 |
-
documents.append(
|
| 280 |
-
DriveDocument(
|
| 281 |
-
document_id=str(file_meta.get("id") or "").strip(),
|
| 282 |
-
source_file=str(file_meta.get("name") or "").strip(),
|
| 283 |
-
mime_type=str(file_meta.get("mimeType") or "").strip(),
|
| 284 |
-
modificationDate=str(file_meta.get("modifiedTime") or "").strip() or None,
|
| 285 |
-
local_path=local_path,
|
| 286 |
-
)
|
| 287 |
-
)
|
| 288 |
-
|
| 289 |
-
return documents, temp_dir
|
| 290 |
|
| 291 |
|
| 292 |
def cleanup_drive_scan_result(scan_result: DriveScanResult | None) -> None:
|
|
|
|
| 1 |
from __future__ import annotations
|
| 2 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3 |
import shutil
|
|
|
|
| 4 |
from dataclasses import dataclass
|
| 5 |
from pathlib import Path
|
|
|
|
| 6 |
|
| 7 |
from dotenv import load_dotenv
|
|
|
|
|
|
|
|
|
|
| 8 |
from firebase_processed_files import FirebaseProcessedFilesTracker, check_files_already_processed
|
| 9 |
+
from google_drive_service import (
|
| 10 |
+
DriveDocument,
|
| 11 |
+
GoogleDriveConfigError,
|
| 12 |
+
load_drive_folder_ids_from_env,
|
| 13 |
+
scan_drive_supported_documents,
|
| 14 |
+
)
|
| 15 |
|
| 16 |
|
| 17 |
load_dotenv()
|
| 18 |
|
| 19 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 20 |
@dataclass
|
| 21 |
class DriveScanResult:
|
| 22 |
files_to_process: list[str]
|
|
|
|
| 26 |
warning: str | None = None
|
| 27 |
|
| 28 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 29 |
def scan_google_drive_unprocessed_files(
|
| 30 |
tracker: FirebaseProcessedFilesTracker | None,
|
| 31 |
include_subfolders: bool = True,
|
| 32 |
) -> DriveScanResult:
|
| 33 |
+
folder_ids = load_drive_folder_ids_from_env()
|
| 34 |
+
scan_result = scan_drive_supported_documents(folder_ids=folder_ids, include_subfolders=include_subfolders)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 35 |
|
| 36 |
+
downloaded_paths = [document.local_path for document in scan_result.documents]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 37 |
|
| 38 |
files_to_process, skipped_files, warning = check_files_already_processed(tracker, downloaded_paths)
|
| 39 |
|
|
|
|
| 49 |
return DriveScanResult(
|
| 50 |
files_to_process=files_to_process,
|
| 51 |
skipped_files=skipped_files,
|
| 52 |
+
discovered_count=scan_result.discovered_count,
|
| 53 |
+
temp_dir=scan_result.temp_dir,
|
| 54 |
warning=warning,
|
| 55 |
)
|
| 56 |
|
| 57 |
|
| 58 |
def scan_google_drive_supported_documents(include_subfolders: bool = True) -> tuple[list[DriveDocument], str]:
|
| 59 |
"""Scans configured Drive folders and downloads supported documents with metadata."""
|
| 60 |
+
folder_ids = load_drive_folder_ids_from_env()
|
| 61 |
+
scan_result = scan_drive_supported_documents(folder_ids=folder_ids, include_subfolders=include_subfolders)
|
| 62 |
+
return scan_result.documents, scan_result.temp_dir
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 63 |
|
| 64 |
|
| 65 |
def cleanup_drive_scan_result(scan_result: DriveScanResult | None) -> None:
|
google_drive_service.py
ADDED
|
@@ -0,0 +1,393 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import json
|
| 4 |
+
import logging
|
| 5 |
+
import os
|
| 6 |
+
import re
|
| 7 |
+
import tempfile
|
| 8 |
+
from dataclasses import dataclass
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
from typing import Any
|
| 11 |
+
|
| 12 |
+
from dotenv import load_dotenv
|
| 13 |
+
from google.oauth2.service_account import Credentials
|
| 14 |
+
|
| 15 |
+
from document_processing import SUPPORTED_EXTENSIONS
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
load_dotenv()
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
logger = logging.getLogger(__name__)
|
| 22 |
+
if not logger.handlers:
|
| 23 |
+
handler = logging.StreamHandler()
|
| 24 |
+
handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(name)s: %(message)s"))
|
| 25 |
+
logger.addHandler(handler)
|
| 26 |
+
logger.setLevel(logging.INFO)
|
| 27 |
+
logger.propagate = False
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
DRIVE_READONLY_SCOPE = "https://www.googleapis.com/auth/drive.readonly"
|
| 31 |
+
DRIVE_FULL_SCOPE = "https://www.googleapis.com/auth/drive"
|
| 32 |
+
FOLDER_MIME_TYPE = "application/vnd.google-apps.folder"
|
| 33 |
+
|
| 34 |
+
# Google Workspace files require export, while binary files can be downloaded directly.
|
| 35 |
+
SUPPORTED_EXPORT_MIME_MAP = {
|
| 36 |
+
"application/vnd.google-apps.document": (
|
| 37 |
+
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
| 38 |
+
".docx",
|
| 39 |
+
),
|
| 40 |
+
}
|
| 41 |
+
|
| 42 |
+
SUPPORTED_BINARY_MIME_TO_SUFFIX = {
|
| 43 |
+
"application/pdf": ".pdf",
|
| 44 |
+
"text/plain": ".txt",
|
| 45 |
+
"application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx",
|
| 46 |
+
"image/jpeg": ".jpg",
|
| 47 |
+
"image/png": ".png",
|
| 48 |
+
}
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
@dataclass
|
| 52 |
+
class DriveDocument:
|
| 53 |
+
document_id: str
|
| 54 |
+
source_file: str
|
| 55 |
+
mime_type: str
|
| 56 |
+
modificationDate: str | None
|
| 57 |
+
local_path: str
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
@dataclass
|
| 61 |
+
class DriveDownloadResult:
|
| 62 |
+
documents: list[DriveDocument]
|
| 63 |
+
discovered_count: int
|
| 64 |
+
temp_dir: str
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
class GoogleDriveConfigError(RuntimeError):
|
| 68 |
+
"""Raised when required Google Drive environment configuration is missing or invalid."""
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def load_drive_folder_ids_from_env(env_var: str = "GOOGLE_DRIVE_FOLDER_IDS") -> list[str]:
|
| 72 |
+
raw = (os.getenv(env_var) or "").strip()
|
| 73 |
+
if not raw:
|
| 74 |
+
raise GoogleDriveConfigError(f"Missing {env_var}. Set one or more folder IDs separated by commas.")
|
| 75 |
+
|
| 76 |
+
folder_ids = [item.strip() for item in raw.split(",") if item.strip()]
|
| 77 |
+
if not folder_ids:
|
| 78 |
+
raise GoogleDriveConfigError(f"{env_var} is empty after parsing.")
|
| 79 |
+
|
| 80 |
+
logger.info("Loaded %s folder id(s) from env var %s", len(folder_ids), env_var)
|
| 81 |
+
|
| 82 |
+
return folder_ids
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
def build_drive_credentials(scope: str = DRIVE_READONLY_SCOPE) -> Credentials:
|
| 86 |
+
service_account_json = (os.getenv("GOOGLE_SERVICE_ACCOUNT_JSON") or "").strip()
|
| 87 |
+
service_account_file = (os.getenv("GOOGLE_SERVICE_ACCOUNT_FILE") or "").strip()
|
| 88 |
+
|
| 89 |
+
if service_account_json:
|
| 90 |
+
try:
|
| 91 |
+
info: dict[str, Any] = json.loads(service_account_json)
|
| 92 |
+
except json.JSONDecodeError as exc:
|
| 93 |
+
raise GoogleDriveConfigError("GOOGLE_SERVICE_ACCOUNT_JSON is not valid JSON.") from exc
|
| 94 |
+
logger.info("Using GOOGLE_SERVICE_ACCOUNT_JSON for Drive credentials")
|
| 95 |
+
return Credentials.from_service_account_info(info, scopes=[scope])
|
| 96 |
+
|
| 97 |
+
if service_account_file:
|
| 98 |
+
logger.info("Using GOOGLE_SERVICE_ACCOUNT_FILE for Drive credentials")
|
| 99 |
+
return Credentials.from_service_account_file(service_account_file, scopes=[scope])
|
| 100 |
+
|
| 101 |
+
raise GoogleDriveConfigError(
|
| 102 |
+
"Missing Google credentials. Set GOOGLE_SERVICE_ACCOUNT_JSON or GOOGLE_SERVICE_ACCOUNT_FILE."
|
| 103 |
+
)
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
def build_drive_service(scope: str = DRIVE_READONLY_SCOPE):
|
| 107 |
+
try:
|
| 108 |
+
from googleapiclient.discovery import build
|
| 109 |
+
except ImportError as exc:
|
| 110 |
+
raise GoogleDriveConfigError(
|
| 111 |
+
"google-api-python-client is required for Drive scanning. Add it to requirements and install dependencies."
|
| 112 |
+
) from exc
|
| 113 |
+
|
| 114 |
+
credentials = build_drive_credentials(scope=scope)
|
| 115 |
+
logger.info("Building Drive service with scope=%s", scope)
|
| 116 |
+
return build("drive", "v3", credentials=credentials, cache_discovery=False)
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
def _escape_drive_query_value(value: str) -> str:
|
| 120 |
+
return value.replace("'", "\\'")
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
def _sanitize_file_name(name: str) -> str:
|
| 124 |
+
cleaned = re.sub(r'[<>:"/\\|?*\x00-\x1F]', "_", name).strip().rstrip(".")
|
| 125 |
+
return cleaned or "document"
|
| 126 |
+
|
| 127 |
+
|
| 128 |
+
def _ensure_unique_destination(path: Path) -> Path:
|
| 129 |
+
if not path.exists():
|
| 130 |
+
return path
|
| 131 |
+
|
| 132 |
+
counter = 1
|
| 133 |
+
while True:
|
| 134 |
+
candidate = path.with_name(f"{path.stem}_{counter}{path.suffix}")
|
| 135 |
+
if not candidate.exists():
|
| 136 |
+
return candidate
|
| 137 |
+
counter += 1
|
| 138 |
+
|
| 139 |
+
|
| 140 |
+
def _list_files_from_folder(service, folder_id: str, include_subfolders: bool) -> list[dict[str, Any]]:
|
| 141 |
+
logger.info("Listing Drive folder id=%s include_subfolders=%s", folder_id, include_subfolders)
|
| 142 |
+
folders_to_scan = [folder_id]
|
| 143 |
+
discovered: list[dict[str, Any]] = []
|
| 144 |
+
|
| 145 |
+
while folders_to_scan:
|
| 146 |
+
current_folder = folders_to_scan.pop(0)
|
| 147 |
+
page_token = None
|
| 148 |
+
|
| 149 |
+
while True:
|
| 150 |
+
response = (
|
| 151 |
+
service.files()
|
| 152 |
+
.list(
|
| 153 |
+
q=f"'{current_folder}' in parents and trashed = false",
|
| 154 |
+
fields="nextPageToken, files(id, name, mimeType, modifiedTime)",
|
| 155 |
+
supportsAllDrives=True,
|
| 156 |
+
includeItemsFromAllDrives=True,
|
| 157 |
+
pageSize=1000,
|
| 158 |
+
pageToken=page_token,
|
| 159 |
+
)
|
| 160 |
+
.execute()
|
| 161 |
+
)
|
| 162 |
+
|
| 163 |
+
for file_meta in response.get("files", []):
|
| 164 |
+
mime_type = file_meta.get("mimeType")
|
| 165 |
+
if mime_type == FOLDER_MIME_TYPE:
|
| 166 |
+
if include_subfolders:
|
| 167 |
+
child_folder_id = file_meta.get("id")
|
| 168 |
+
if child_folder_id:
|
| 169 |
+
folders_to_scan.append(child_folder_id)
|
| 170 |
+
continue
|
| 171 |
+
|
| 172 |
+
discovered.append(file_meta)
|
| 173 |
+
|
| 174 |
+
page_token = response.get("nextPageToken")
|
| 175 |
+
if not page_token:
|
| 176 |
+
break
|
| 177 |
+
|
| 178 |
+
return discovered
|
| 179 |
+
|
| 180 |
+
|
| 181 |
+
def _is_supported_drive_file(file_meta: dict[str, Any]) -> bool:
|
| 182 |
+
mime_type = (file_meta.get("mimeType") or "").strip().lower()
|
| 183 |
+
if mime_type in SUPPORTED_BINARY_MIME_TO_SUFFIX:
|
| 184 |
+
return True
|
| 185 |
+
if mime_type in SUPPORTED_EXPORT_MIME_MAP:
|
| 186 |
+
return True
|
| 187 |
+
|
| 188 |
+
name = (file_meta.get("name") or "").strip()
|
| 189 |
+
suffix = Path(name).suffix.lower()
|
| 190 |
+
return suffix in SUPPORTED_EXTENSIONS
|
| 191 |
+
|
| 192 |
+
|
| 193 |
+
def _download_drive_file(service, file_meta: dict[str, Any], destination_dir: Path) -> str:
|
| 194 |
+
try:
|
| 195 |
+
from googleapiclient.http import MediaIoBaseDownload
|
| 196 |
+
except ImportError as exc:
|
| 197 |
+
raise GoogleDriveConfigError(
|
| 198 |
+
"google-api-python-client is required for Drive scanning. Add it to requirements and install dependencies."
|
| 199 |
+
) from exc
|
| 200 |
+
|
| 201 |
+
file_id = (file_meta.get("id") or "").strip()
|
| 202 |
+
file_name = _sanitize_file_name((file_meta.get("name") or "document").strip())
|
| 203 |
+
mime_type = (file_meta.get("mimeType") or "").strip().lower()
|
| 204 |
+
|
| 205 |
+
suffix = Path(file_name).suffix.lower()
|
| 206 |
+
request = None
|
| 207 |
+
|
| 208 |
+
if mime_type in SUPPORTED_EXPORT_MIME_MAP:
|
| 209 |
+
export_mime, export_suffix = SUPPORTED_EXPORT_MIME_MAP[mime_type]
|
| 210 |
+
if suffix != export_suffix:
|
| 211 |
+
file_name = f"{Path(file_name).stem}{export_suffix}"
|
| 212 |
+
suffix = export_suffix
|
| 213 |
+
request = service.files().export_media(fileId=file_id, mimeType=export_mime)
|
| 214 |
+
else:
|
| 215 |
+
if not suffix and mime_type in SUPPORTED_BINARY_MIME_TO_SUFFIX:
|
| 216 |
+
file_name = f"{file_name}{SUPPORTED_BINARY_MIME_TO_SUFFIX[mime_type]}"
|
| 217 |
+
suffix = Path(file_name).suffix.lower()
|
| 218 |
+
request = service.files().get_media(fileId=file_id)
|
| 219 |
+
|
| 220 |
+
if suffix not in SUPPORTED_EXTENSIONS:
|
| 221 |
+
raise ValueError(f"Unsupported downloaded file type: {suffix or '[no extension]'}")
|
| 222 |
+
|
| 223 |
+
destination = _ensure_unique_destination(destination_dir / file_name)
|
| 224 |
+
|
| 225 |
+
with destination.open("wb") as stream:
|
| 226 |
+
downloader = MediaIoBaseDownload(stream, request)
|
| 227 |
+
done = False
|
| 228 |
+
while not done:
|
| 229 |
+
_, done = downloader.next_chunk()
|
| 230 |
+
|
| 231 |
+
return str(destination)
|
| 232 |
+
|
| 233 |
+
|
| 234 |
+
def scan_drive_supported_documents(folder_ids: list[str], include_subfolders: bool = True) -> DriveDownloadResult:
|
| 235 |
+
service = build_drive_service()
|
| 236 |
+
logger.info("Starting Drive scan for %s folder(s)", len(folder_ids))
|
| 237 |
+
|
| 238 |
+
discovered_files: list[dict[str, Any]] = []
|
| 239 |
+
for folder_id in folder_ids:
|
| 240 |
+
discovered_files.extend(_list_files_from_folder(service, folder_id, include_subfolders))
|
| 241 |
+
|
| 242 |
+
supported_files = [file_meta for file_meta in discovered_files if _is_supported_drive_file(file_meta)]
|
| 243 |
+
logger.info("Drive scan discovered=%s supported=%s", len(discovered_files), len(supported_files))
|
| 244 |
+
temp_dir = tempfile.mkdtemp(prefix="drive_docs_")
|
| 245 |
+
destination_dir = Path(temp_dir)
|
| 246 |
+
|
| 247 |
+
documents: list[DriveDocument] = []
|
| 248 |
+
for file_meta in supported_files:
|
| 249 |
+
try:
|
| 250 |
+
local_path = _download_drive_file(service, file_meta, destination_dir)
|
| 251 |
+
except Exception:
|
| 252 |
+
# Keep scan resilient if one file is malformed or inaccessible.
|
| 253 |
+
logger.exception("Skipping file during Drive download due to error file_id=%s", file_meta.get("id"))
|
| 254 |
+
continue
|
| 255 |
+
|
| 256 |
+
documents.append(
|
| 257 |
+
DriveDocument(
|
| 258 |
+
document_id=str(file_meta.get("id") or "").strip(),
|
| 259 |
+
source_file=str(file_meta.get("name") or "").strip(),
|
| 260 |
+
mime_type=str(file_meta.get("mimeType") or "").strip(),
|
| 261 |
+
modificationDate=str(file_meta.get("modifiedTime") or "").strip() or None,
|
| 262 |
+
local_path=local_path,
|
| 263 |
+
)
|
| 264 |
+
)
|
| 265 |
+
|
| 266 |
+
return DriveDownloadResult(
|
| 267 |
+
documents=documents,
|
| 268 |
+
discovered_count=len(supported_files),
|
| 269 |
+
temp_dir=temp_dir,
|
| 270 |
+
)
|
| 271 |
+
|
| 272 |
+
|
| 273 |
+
def _find_folder_in_parent(service, folder_name: str, parent_folder_id: str | None) -> str | None:
|
| 274 |
+
query_parts = [
|
| 275 |
+
"trashed = false",
|
| 276 |
+
f"mimeType = '{FOLDER_MIME_TYPE}'",
|
| 277 |
+
f"name = '{_escape_drive_query_value(folder_name)}'",
|
| 278 |
+
]
|
| 279 |
+
if parent_folder_id:
|
| 280 |
+
query_parts.append(f"'{parent_folder_id}' in parents")
|
| 281 |
+
|
| 282 |
+
try:
|
| 283 |
+
response = (
|
| 284 |
+
service.files()
|
| 285 |
+
.list(
|
| 286 |
+
q=" and ".join(query_parts),
|
| 287 |
+
fields="files(id,name)",
|
| 288 |
+
supportsAllDrives=True,
|
| 289 |
+
includeItemsFromAllDrives=True,
|
| 290 |
+
pageSize=10,
|
| 291 |
+
)
|
| 292 |
+
.execute()
|
| 293 |
+
)
|
| 294 |
+
except Exception as exc:
|
| 295 |
+
raise GoogleDriveConfigError(f"Failed to query folder '{folder_name}': {exc}") from exc
|
| 296 |
+
|
| 297 |
+
files = response.get("files", [])
|
| 298 |
+
if not files:
|
| 299 |
+
logger.info("Drive folder not found name=%s parent=%s", folder_name, parent_folder_id or "root")
|
| 300 |
+
return None
|
| 301 |
+
|
| 302 |
+
logger.info("Found Drive folder name=%s parent=%s", folder_name, parent_folder_id or "root")
|
| 303 |
+
return str(files[0].get("id") or "").strip() or None
|
| 304 |
+
|
| 305 |
+
|
| 306 |
+
def _create_folder_in_parent(service, folder_name: str, parent_folder_id: str | None) -> str:
|
| 307 |
+
body: dict[str, Any] = {"name": folder_name, "mimeType": FOLDER_MIME_TYPE}
|
| 308 |
+
if parent_folder_id:
|
| 309 |
+
body["parents"] = [parent_folder_id]
|
| 310 |
+
|
| 311 |
+
try:
|
| 312 |
+
created = (
|
| 313 |
+
service.files()
|
| 314 |
+
.create(body=body, fields="id", supportsAllDrives=True)
|
| 315 |
+
.execute()
|
| 316 |
+
)
|
| 317 |
+
except Exception as exc:
|
| 318 |
+
raise GoogleDriveConfigError(
|
| 319 |
+
f"Failed to create folder '{folder_name}' under parent '{parent_folder_id or 'root'}': {exc}"
|
| 320 |
+
) from exc
|
| 321 |
+
|
| 322 |
+
folder_id = str(created.get("id") or "").strip()
|
| 323 |
+
if not folder_id:
|
| 324 |
+
raise GoogleDriveConfigError(f"Folder '{folder_name}' was created but no id was returned.")
|
| 325 |
+
|
| 326 |
+
logger.info("Created Drive folder name=%s parent=%s id=%s", folder_name, parent_folder_id or "root", folder_id)
|
| 327 |
+
|
| 328 |
+
return folder_id
|
| 329 |
+
|
| 330 |
+
|
| 331 |
+
def ensure_drive_folder_path(service, folder_path: str, root_folder_id: str | None = None) -> str:
|
| 332 |
+
normalized = (folder_path or "").strip().strip("/")
|
| 333 |
+
if not normalized:
|
| 334 |
+
raise GoogleDriveConfigError("folder_path is required to resolve a Drive destination.")
|
| 335 |
+
|
| 336 |
+
current_parent = (root_folder_id or "").strip() or None
|
| 337 |
+
for part in [segment.strip() for segment in normalized.split("/") if segment.strip()]:
|
| 338 |
+
folder_id = _find_folder_in_parent(service, part, current_parent)
|
| 339 |
+
if folder_id is None:
|
| 340 |
+
folder_id = _create_folder_in_parent(service, part, current_parent)
|
| 341 |
+
current_parent = folder_id
|
| 342 |
+
|
| 343 |
+
if not current_parent:
|
| 344 |
+
raise GoogleDriveConfigError(f"Could not resolve Drive path '{folder_path}'.")
|
| 345 |
+
|
| 346 |
+
logger.info("Resolved Drive path=%s to folder_id=%s", folder_path, current_parent)
|
| 347 |
+
|
| 348 |
+
return current_parent
|
| 349 |
+
|
| 350 |
+
|
| 351 |
+
def move_file_to_path(service, file_id: str, destination_path: str, root_folder_id: str | None = None) -> str:
|
| 352 |
+
normalized_file_id = (file_id or "").strip()
|
| 353 |
+
if not normalized_file_id:
|
| 354 |
+
raise GoogleDriveConfigError("file_id is required to move a Drive file.")
|
| 355 |
+
|
| 356 |
+
destination_folder_id = ensure_drive_folder_path(service, destination_path, root_folder_id=root_folder_id)
|
| 357 |
+
logger.info("Moving Drive file_id=%s to destination_path=%s", normalized_file_id, destination_path)
|
| 358 |
+
|
| 359 |
+
try:
|
| 360 |
+
metadata = (
|
| 361 |
+
service.files()
|
| 362 |
+
.get(fileId=normalized_file_id, fields="parents", supportsAllDrives=True)
|
| 363 |
+
.execute()
|
| 364 |
+
)
|
| 365 |
+
except Exception as exc:
|
| 366 |
+
raise GoogleDriveConfigError(f"Failed to read current parents for file '{normalized_file_id}': {exc}") from exc
|
| 367 |
+
|
| 368 |
+
current_parents = [str(parent).strip() for parent in metadata.get("parents", []) if str(parent).strip()]
|
| 369 |
+
|
| 370 |
+
if destination_folder_id in current_parents and len(current_parents) == 1:
|
| 371 |
+
logger.info("Drive file already in destination file_id=%s folder_id=%s", normalized_file_id, destination_folder_id)
|
| 372 |
+
return destination_folder_id
|
| 373 |
+
|
| 374 |
+
request_kwargs: dict[str, Any] = {
|
| 375 |
+
"fileId": normalized_file_id,
|
| 376 |
+
"addParents": destination_folder_id,
|
| 377 |
+
"supportsAllDrives": True,
|
| 378 |
+
"fields": "id,parents",
|
| 379 |
+
}
|
| 380 |
+
remove_parents = ",".join(parent for parent in current_parents if parent != destination_folder_id)
|
| 381 |
+
if remove_parents:
|
| 382 |
+
request_kwargs["removeParents"] = remove_parents
|
| 383 |
+
|
| 384 |
+
try:
|
| 385 |
+
service.files().update(**request_kwargs).execute()
|
| 386 |
+
except Exception as exc:
|
| 387 |
+
raise GoogleDriveConfigError(
|
| 388 |
+
f"Failed to move file '{normalized_file_id}' to Drive path '{destination_path}': {exc}"
|
| 389 |
+
) from exc
|
| 390 |
+
|
| 391 |
+
logger.info("Moved Drive file_id=%s to folder_id=%s", normalized_file_id, destination_folder_id)
|
| 392 |
+
|
| 393 |
+
return destination_folder_id
|