document-parser / google_drive_service.py
segovia
multitenant supprt
028d116
Raw History Blame Contribute Delete
15.1 kB
from __future__ import annotations
import json
import logging
import os
import re
import tempfile
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from dotenv import load_dotenv
from google.oauth2.service_account import Credentials
from document_processing import SUPPORTED_EXTENSIONS
load_dotenv()
logger = logging.getLogger(__name__)
if not logger.handlers:
handler = logging.StreamHandler()
handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(name)s: %(message)s"))
logger.addHandler(handler)
logger.setLevel(logging.INFO)
logger.propagate = False
DRIVE_READONLY_SCOPE = "https://www.googleapis.com/auth/drive.readonly"
DRIVE_FULL_SCOPE = "https://www.googleapis.com/auth/drive"
FOLDER_MIME_TYPE = "application/vnd.google-apps.folder"
# Google Workspace files require export, while binary files can be downloaded directly.
SUPPORTED_EXPORT_MIME_MAP = {
"application/vnd.google-apps.document": (
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
".docx",
),
}
SUPPORTED_BINARY_MIME_TO_SUFFIX = {
"application/pdf": ".pdf",
"text/plain": ".txt",
"application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx",
"image/jpeg": ".jpg",
"image/png": ".png",
}
@dataclass
class DriveDocument:
document_id: str
source_file: str
mime_type: str
modificationDate: str | None
local_path: str
@dataclass
class DriveDownloadResult:
documents: list[DriveDocument]
discovered_count: int
temp_dir: str
class GoogleDriveConfigError(RuntimeError):
"""Raised when required Google Drive environment configuration is missing or invalid."""
def load_drive_folder_ids_from_env(env_var: str = "GOOGLE_DRIVE_FOLDER_IDS") -> list[str]:
raw = (os.getenv(env_var) or "").strip()
if not raw:
raise GoogleDriveConfigError(f"Missing {env_var}. Set one or more folder IDs separated by commas.")
folder_ids = [item.strip() for item in raw.split(",") if item.strip()]
if not folder_ids:
raise GoogleDriveConfigError(f"{env_var} is empty after parsing.")
logger.info("Loaded %s folder id(s) from env var %s", len(folder_ids), env_var)
return folder_ids
def build_drive_credentials(scope: str = DRIVE_READONLY_SCOPE, service_account_json: str | None = None) -> Credentials:
"""Build Google Drive credentials.
Args:
scope: OAuth scope for the credentials
service_account_json: Optional JSON string with service account credentials.
If not provided, uses GOOGLE_SERVICE_ACCOUNT_JSON env var.
"""
json_to_use = (service_account_json or "").strip() or (os.getenv("GOOGLE_SERVICE_ACCOUNT_JSON") or "").strip()
service_account_file = (os.getenv("GOOGLE_SERVICE_ACCOUNT_FILE") or "").strip()
if json_to_use:
try:
info: dict[str, Any] = json.loads(json_to_use)
except json.JSONDecodeError as exc:
raise GoogleDriveConfigError("GOOGLE_SERVICE_ACCOUNT_JSON is not valid JSON.") from exc
logger.info("Using GOOGLE_SERVICE_ACCOUNT_JSON for Drive credentials")
return Credentials.from_service_account_info(info, scopes=[scope])
if service_account_file:
logger.info("Using GOOGLE_SERVICE_ACCOUNT_FILE for Drive credentials")
return Credentials.from_service_account_file(service_account_file, scopes=[scope])
raise GoogleDriveConfigError(
"Missing Google credentials. Set GOOGLE_SERVICE_ACCOUNT_JSON or GOOGLE_SERVICE_ACCOUNT_FILE."
)
def build_drive_service(scope: str = DRIVE_READONLY_SCOPE, service_account_json: str | None = None):
"""Build Google Drive service.
Args:
scope: OAuth scope for the credentials
service_account_json: Optional JSON string with service account credentials for tenant.
"""
try:
from googleapiclient.discovery import build
except ImportError as exc:
raise GoogleDriveConfigError(
"google-api-python-client is required for Drive scanning. Add it to requirements and install dependencies."
) from exc
credentials = build_drive_credentials(scope=scope, service_account_json=service_account_json)
logger.info("Building Drive service with scope=%s", scope)
return build("drive", "v3", credentials=credentials, cache_discovery=False)
def _escape_drive_query_value(value: str) -> str:
return value.replace("'", "\\'")
def _sanitize_file_name(name: str) -> str:
cleaned = re.sub(r'[<>:"/\\|?*\x00-\x1F]', "_", name).strip().rstrip(".")
return cleaned or "document"
def _ensure_unique_destination(path: Path) -> Path:
if not path.exists():
return path
counter = 1
while True:
candidate = path.with_name(f"{path.stem}_{counter}{path.suffix}")
if not candidate.exists():
return candidate
counter += 1
def _list_files_from_folder(service, folder_id: str, include_subfolders: bool) -> list[dict[str, Any]]:
logger.info("Listing Drive folder id=%s include_subfolders=%s", folder_id, include_subfolders)
folders_to_scan = [folder_id]
discovered: list[dict[str, Any]] = []
while folders_to_scan:
current_folder = folders_to_scan.pop(0)
page_token = None
while True:
response = (
service.files()
.list(
q=f"'{current_folder}' in parents and trashed = false",
fields="nextPageToken, files(id, name, mimeType, modifiedTime)",
supportsAllDrives=True,
includeItemsFromAllDrives=True,
pageSize=1000,
pageToken=page_token,
)
.execute()
)
for file_meta in response.get("files", []):
mime_type = file_meta.get("mimeType")
if mime_type == FOLDER_MIME_TYPE:
if include_subfolders:
child_folder_id = file_meta.get("id")
if child_folder_id:
folders_to_scan.append(child_folder_id)
continue
discovered.append(file_meta)
page_token = response.get("nextPageToken")
if not page_token:
break
return discovered
def _is_supported_drive_file(file_meta: dict[str, Any]) -> bool:
mime_type = (file_meta.get("mimeType") or "").strip().lower()
if mime_type in SUPPORTED_BINARY_MIME_TO_SUFFIX:
return True
if mime_type in SUPPORTED_EXPORT_MIME_MAP:
return True
name = (file_meta.get("name") or "").strip()
suffix = Path(name).suffix.lower()
return suffix in SUPPORTED_EXTENSIONS
def _download_drive_file(service, file_meta: dict[str, Any], destination_dir: Path) -> str:
try:
from googleapiclient.http import MediaIoBaseDownload
except ImportError as exc:
raise GoogleDriveConfigError(
"google-api-python-client is required for Drive scanning. Add it to requirements and install dependencies."
) from exc
file_id = (file_meta.get("id") or "").strip()
file_name = _sanitize_file_name((file_meta.get("name") or "document").strip())
mime_type = (file_meta.get("mimeType") or "").strip().lower()
suffix = Path(file_name).suffix.lower()
request = None
if mime_type in SUPPORTED_EXPORT_MIME_MAP:
export_mime, export_suffix = SUPPORTED_EXPORT_MIME_MAP[mime_type]
if suffix != export_suffix:
file_name = f"{Path(file_name).stem}{export_suffix}"
suffix = export_suffix
request = service.files().export_media(fileId=file_id, mimeType=export_mime)
else:
if not suffix and mime_type in SUPPORTED_BINARY_MIME_TO_SUFFIX:
file_name = f"{file_name}{SUPPORTED_BINARY_MIME_TO_SUFFIX[mime_type]}"
suffix = Path(file_name).suffix.lower()
request = service.files().get_media(fileId=file_id)
if suffix not in SUPPORTED_EXTENSIONS:
raise ValueError(f"Unsupported downloaded file type: {suffix or '[no extension]'}")
destination = _ensure_unique_destination(destination_dir / file_name)
with destination.open("wb") as stream:
downloader = MediaIoBaseDownload(stream, request)
done = False
while not done:
_, done = downloader.next_chunk()
return str(destination)
def scan_drive_supported_documents(folder_ids: list[str], include_subfolders: bool = True) -> DriveDownloadResult:
service = build_drive_service()
logger.info("Starting Drive scan for %s folder(s)", len(folder_ids))
discovered_files: list[dict[str, Any]] = []
for folder_id in folder_ids:
discovered_files.extend(_list_files_from_folder(service, folder_id, include_subfolders))
supported_files = [file_meta for file_meta in discovered_files if _is_supported_drive_file(file_meta)]
logger.info("Drive scan discovered=%s supported=%s", len(discovered_files), len(supported_files))
temp_dir = tempfile.mkdtemp(prefix="drive_docs_")
destination_dir = Path(temp_dir)
documents: list[DriveDocument] = []
for file_meta in supported_files:
try:
local_path = _download_drive_file(service, file_meta, destination_dir)
except Exception:
# Keep scan resilient if one file is malformed or inaccessible.
logger.exception("Skipping file during Drive download due to error file_id=%s", file_meta.get("id"))
continue
documents.append(
DriveDocument(
document_id=str(file_meta.get("id") or "").strip(),
source_file=str(file_meta.get("name") or "").strip(),
mime_type=str(file_meta.get("mimeType") or "").strip(),
modificationDate=str(file_meta.get("modifiedTime") or "").strip() or None,
local_path=local_path,
)
)
return DriveDownloadResult(
documents=documents,
discovered_count=len(supported_files),
temp_dir=temp_dir,
)
def _find_folder_in_parent(service, folder_name: str, parent_folder_id: str | None) -> str | None:
query_parts = [
"trashed = false",
f"mimeType = '{FOLDER_MIME_TYPE}'",
f"name = '{_escape_drive_query_value(folder_name)}'",
]
if parent_folder_id:
query_parts.append(f"'{parent_folder_id}' in parents")
try:
response = (
service.files()
.list(
q=" and ".join(query_parts),
fields="files(id,name)",
supportsAllDrives=True,
includeItemsFromAllDrives=True,
pageSize=10,
)
.execute()
)
except Exception as exc:
raise GoogleDriveConfigError(f"Failed to query folder '{folder_name}': {exc}") from exc
files = response.get("files", [])
if not files:
logger.info("Drive folder not found name=%s parent=%s", folder_name, parent_folder_id or "root")
return None
logger.info("Found Drive folder name=%s parent=%s", folder_name, parent_folder_id or "root")
return str(files[0].get("id") or "").strip() or None
def _create_folder_in_parent(service, folder_name: str, parent_folder_id: str | None) -> str:
body: dict[str, Any] = {"name": folder_name, "mimeType": FOLDER_MIME_TYPE}
if parent_folder_id:
body["parents"] = [parent_folder_id]
try:
created = (
service.files()
.create(body=body, fields="id", supportsAllDrives=True)
.execute()
)
except Exception as exc:
raise GoogleDriveConfigError(
f"Failed to create folder '{folder_name}' under parent '{parent_folder_id or 'root'}': {exc}"
) from exc
folder_id = str(created.get("id") or "").strip()
if not folder_id:
raise GoogleDriveConfigError(f"Folder '{folder_name}' was created but no id was returned.")
logger.info("Created Drive folder name=%s parent=%s id=%s", folder_name, parent_folder_id or "root", folder_id)
return folder_id
def ensure_drive_folder_path(service, folder_path: str, root_folder_id: str | None = None) -> str:
normalized = (folder_path or "").strip().strip("/")
if not normalized:
raise GoogleDriveConfigError("folder_path is required to resolve a Drive destination.")
current_parent = (root_folder_id or "").strip() or None
for part in [segment.strip() for segment in normalized.split("/") if segment.strip()]:
folder_id = _find_folder_in_parent(service, part, current_parent)
if folder_id is None:
folder_id = _create_folder_in_parent(service, part, current_parent)
current_parent = folder_id
if not current_parent:
raise GoogleDriveConfigError(f"Could not resolve Drive path '{folder_path}'.")
logger.info("Resolved Drive path=%s to folder_id=%s", folder_path, current_parent)
return current_parent
def move_file_to_path(service, file_id: str, destination_path: str, root_folder_id: str | None = None) -> str:
normalized_file_id = (file_id or "").strip()
if not normalized_file_id:
raise GoogleDriveConfigError("file_id is required to move a Drive file.")
destination_folder_id = ensure_drive_folder_path(service, destination_path, root_folder_id=root_folder_id)
logger.info("Moving Drive file_id=%s to destination_path=%s", normalized_file_id, destination_path)
try:
metadata = (
service.files()
.get(fileId=normalized_file_id, fields="parents", supportsAllDrives=True)
.execute()
)
except Exception as exc:
raise GoogleDriveConfigError(f"Failed to read current parents for file '{normalized_file_id}': {exc}") from exc
current_parents = [str(parent).strip() for parent in metadata.get("parents", []) if str(parent).strip()]
if destination_folder_id in current_parents and len(current_parents) == 1:
logger.info("Drive file already in destination file_id=%s folder_id=%s", normalized_file_id, destination_folder_id)
return destination_folder_id
request_kwargs: dict[str, Any] = {
"fileId": normalized_file_id,
"addParents": destination_folder_id,
"supportsAllDrives": True,
"fields": "id,parents",
}
remove_parents = ",".join(parent for parent in current_parents if parent != destination_folder_id)
if remove_parents:
request_kwargs["removeParents"] = remove_parents
try:
service.files().update(**request_kwargs).execute()
except Exception as exc:
raise GoogleDriveConfigError(
f"Failed to move file '{normalized_file_id}' to Drive path '{destination_path}': {exc}"
) from exc
logger.info("Moved Drive file_id=%s to folder_id=%s", normalized_file_id, destination_folder_id)
return destination_folder_id