File size: 15,052 Bytes
cb04ca0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
028d116
 
 
 
 
 
 
 
 
cb04ca0
 
028d116
cb04ca0
028d116
cb04ca0
 
 
 
 
 
 
 
 
 
 
 
 
 
028d116
 
 
 
 
 
 
cb04ca0
 
 
 
 
 
 
028d116
cb04ca0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
from __future__ import annotations

import json
import logging
import os
import re
import tempfile
from dataclasses import dataclass
from pathlib import Path
from typing import Any

from dotenv import load_dotenv
from google.oauth2.service_account import Credentials

from document_processing import SUPPORTED_EXTENSIONS


load_dotenv()


logger = logging.getLogger(__name__)
if not logger.handlers:
    handler = logging.StreamHandler()
    handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(name)s: %(message)s"))
    logger.addHandler(handler)
logger.setLevel(logging.INFO)
logger.propagate = False


DRIVE_READONLY_SCOPE = "https://www.googleapis.com/auth/drive.readonly"
DRIVE_FULL_SCOPE = "https://www.googleapis.com/auth/drive"
FOLDER_MIME_TYPE = "application/vnd.google-apps.folder"

# Google Workspace files require export, while binary files can be downloaded directly.
SUPPORTED_EXPORT_MIME_MAP = {
    "application/vnd.google-apps.document": (
        "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
        ".docx",
    ),
}

SUPPORTED_BINARY_MIME_TO_SUFFIX = {
    "application/pdf": ".pdf",
    "text/plain": ".txt",
    "application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx",
    "image/jpeg": ".jpg",
    "image/png": ".png",
}


@dataclass
class DriveDocument:
    document_id: str
    source_file: str
    mime_type: str
    modificationDate: str | None
    local_path: str


@dataclass
class DriveDownloadResult:
    documents: list[DriveDocument]
    discovered_count: int
    temp_dir: str


class GoogleDriveConfigError(RuntimeError):
    """Raised when required Google Drive environment configuration is missing or invalid."""


def load_drive_folder_ids_from_env(env_var: str = "GOOGLE_DRIVE_FOLDER_IDS") -> list[str]:
    raw = (os.getenv(env_var) or "").strip()
    if not raw:
        raise GoogleDriveConfigError(f"Missing {env_var}. Set one or more folder IDs separated by commas.")

    folder_ids = [item.strip() for item in raw.split(",") if item.strip()]
    if not folder_ids:
        raise GoogleDriveConfigError(f"{env_var} is empty after parsing.")

    logger.info("Loaded %s folder id(s) from env var %s", len(folder_ids), env_var)

    return folder_ids


def build_drive_credentials(scope: str = DRIVE_READONLY_SCOPE, service_account_json: str | None = None) -> Credentials:
    """Build Google Drive credentials.
    
    Args:
        scope: OAuth scope for the credentials
        service_account_json: Optional JSON string with service account credentials.
                            If not provided, uses GOOGLE_SERVICE_ACCOUNT_JSON env var.
    """
    json_to_use = (service_account_json or "").strip() or (os.getenv("GOOGLE_SERVICE_ACCOUNT_JSON") or "").strip()
    service_account_file = (os.getenv("GOOGLE_SERVICE_ACCOUNT_FILE") or "").strip()

    if json_to_use:
        try:
            info: dict[str, Any] = json.loads(json_to_use)
        except json.JSONDecodeError as exc:
            raise GoogleDriveConfigError("GOOGLE_SERVICE_ACCOUNT_JSON is not valid JSON.") from exc
        logger.info("Using GOOGLE_SERVICE_ACCOUNT_JSON for Drive credentials")
        return Credentials.from_service_account_info(info, scopes=[scope])

    if service_account_file:
        logger.info("Using GOOGLE_SERVICE_ACCOUNT_FILE for Drive credentials")
        return Credentials.from_service_account_file(service_account_file, scopes=[scope])

    raise GoogleDriveConfigError(
        "Missing Google credentials. Set GOOGLE_SERVICE_ACCOUNT_JSON or GOOGLE_SERVICE_ACCOUNT_FILE."
    )


def build_drive_service(scope: str = DRIVE_READONLY_SCOPE, service_account_json: str | None = None):
    """Build Google Drive service.
    
    Args:
        scope: OAuth scope for the credentials
        service_account_json: Optional JSON string with service account credentials for tenant.
    """
    try:
        from googleapiclient.discovery import build
    except ImportError as exc:
        raise GoogleDriveConfigError(
            "google-api-python-client is required for Drive scanning. Add it to requirements and install dependencies."
        ) from exc

    credentials = build_drive_credentials(scope=scope, service_account_json=service_account_json)
    logger.info("Building Drive service with scope=%s", scope)
    return build("drive", "v3", credentials=credentials, cache_discovery=False)


def _escape_drive_query_value(value: str) -> str:
    return value.replace("'", "\\'")


def _sanitize_file_name(name: str) -> str:
    cleaned = re.sub(r'[<>:"/\\|?*\x00-\x1F]', "_", name).strip().rstrip(".")
    return cleaned or "document"


def _ensure_unique_destination(path: Path) -> Path:
    if not path.exists():
        return path

    counter = 1
    while True:
        candidate = path.with_name(f"{path.stem}_{counter}{path.suffix}")
        if not candidate.exists():
            return candidate
        counter += 1


def _list_files_from_folder(service, folder_id: str, include_subfolders: bool) -> list[dict[str, Any]]:
    logger.info("Listing Drive folder id=%s include_subfolders=%s", folder_id, include_subfolders)
    folders_to_scan = [folder_id]
    discovered: list[dict[str, Any]] = []

    while folders_to_scan:
        current_folder = folders_to_scan.pop(0)
        page_token = None

        while True:
            response = (
                service.files()
                .list(
                    q=f"'{current_folder}' in parents and trashed = false",
                    fields="nextPageToken, files(id, name, mimeType, modifiedTime)",
                    supportsAllDrives=True,
                    includeItemsFromAllDrives=True,
                    pageSize=1000,
                    pageToken=page_token,
                )
                .execute()
            )

            for file_meta in response.get("files", []):
                mime_type = file_meta.get("mimeType")
                if mime_type == FOLDER_MIME_TYPE:
                    if include_subfolders:
                        child_folder_id = file_meta.get("id")
                        if child_folder_id:
                            folders_to_scan.append(child_folder_id)
                    continue

                discovered.append(file_meta)

            page_token = response.get("nextPageToken")
            if not page_token:
                break

    return discovered


def _is_supported_drive_file(file_meta: dict[str, Any]) -> bool:
    mime_type = (file_meta.get("mimeType") or "").strip().lower()
    if mime_type in SUPPORTED_BINARY_MIME_TO_SUFFIX:
        return True
    if mime_type in SUPPORTED_EXPORT_MIME_MAP:
        return True

    name = (file_meta.get("name") or "").strip()
    suffix = Path(name).suffix.lower()
    return suffix in SUPPORTED_EXTENSIONS


def _download_drive_file(service, file_meta: dict[str, Any], destination_dir: Path) -> str:
    try:
        from googleapiclient.http import MediaIoBaseDownload
    except ImportError as exc:
        raise GoogleDriveConfigError(
            "google-api-python-client is required for Drive scanning. Add it to requirements and install dependencies."
        ) from exc

    file_id = (file_meta.get("id") or "").strip()
    file_name = _sanitize_file_name((file_meta.get("name") or "document").strip())
    mime_type = (file_meta.get("mimeType") or "").strip().lower()

    suffix = Path(file_name).suffix.lower()
    request = None

    if mime_type in SUPPORTED_EXPORT_MIME_MAP:
        export_mime, export_suffix = SUPPORTED_EXPORT_MIME_MAP[mime_type]
        if suffix != export_suffix:
            file_name = f"{Path(file_name).stem}{export_suffix}"
            suffix = export_suffix
        request = service.files().export_media(fileId=file_id, mimeType=export_mime)
    else:
        if not suffix and mime_type in SUPPORTED_BINARY_MIME_TO_SUFFIX:
            file_name = f"{file_name}{SUPPORTED_BINARY_MIME_TO_SUFFIX[mime_type]}"
            suffix = Path(file_name).suffix.lower()
        request = service.files().get_media(fileId=file_id)

    if suffix not in SUPPORTED_EXTENSIONS:
        raise ValueError(f"Unsupported downloaded file type: {suffix or '[no extension]'}")

    destination = _ensure_unique_destination(destination_dir / file_name)

    with destination.open("wb") as stream:
        downloader = MediaIoBaseDownload(stream, request)
        done = False
        while not done:
            _, done = downloader.next_chunk()

    return str(destination)


def scan_drive_supported_documents(folder_ids: list[str], include_subfolders: bool = True) -> DriveDownloadResult:
    service = build_drive_service()
    logger.info("Starting Drive scan for %s folder(s)", len(folder_ids))

    discovered_files: list[dict[str, Any]] = []
    for folder_id in folder_ids:
        discovered_files.extend(_list_files_from_folder(service, folder_id, include_subfolders))

    supported_files = [file_meta for file_meta in discovered_files if _is_supported_drive_file(file_meta)]
    logger.info("Drive scan discovered=%s supported=%s", len(discovered_files), len(supported_files))
    temp_dir = tempfile.mkdtemp(prefix="drive_docs_")
    destination_dir = Path(temp_dir)

    documents: list[DriveDocument] = []
    for file_meta in supported_files:
        try:
            local_path = _download_drive_file(service, file_meta, destination_dir)
        except Exception:
            # Keep scan resilient if one file is malformed or inaccessible.
            logger.exception("Skipping file during Drive download due to error file_id=%s", file_meta.get("id"))
            continue

        documents.append(
            DriveDocument(
                document_id=str(file_meta.get("id") or "").strip(),
                source_file=str(file_meta.get("name") or "").strip(),
                mime_type=str(file_meta.get("mimeType") or "").strip(),
                modificationDate=str(file_meta.get("modifiedTime") or "").strip() or None,
                local_path=local_path,
            )
        )

    return DriveDownloadResult(
        documents=documents,
        discovered_count=len(supported_files),
        temp_dir=temp_dir,
    )


def _find_folder_in_parent(service, folder_name: str, parent_folder_id: str | None) -> str | None:
    query_parts = [
        "trashed = false",
        f"mimeType = '{FOLDER_MIME_TYPE}'",
        f"name = '{_escape_drive_query_value(folder_name)}'",
    ]
    if parent_folder_id:
        query_parts.append(f"'{parent_folder_id}' in parents")

    try:
        response = (
            service.files()
            .list(
                q=" and ".join(query_parts),
                fields="files(id,name)",
                supportsAllDrives=True,
                includeItemsFromAllDrives=True,
                pageSize=10,
            )
            .execute()
        )
    except Exception as exc:
        raise GoogleDriveConfigError(f"Failed to query folder '{folder_name}': {exc}") from exc

    files = response.get("files", [])
    if not files:
        logger.info("Drive folder not found name=%s parent=%s", folder_name, parent_folder_id or "root")
        return None

    logger.info("Found Drive folder name=%s parent=%s", folder_name, parent_folder_id or "root")
    return str(files[0].get("id") or "").strip() or None


def _create_folder_in_parent(service, folder_name: str, parent_folder_id: str | None) -> str:
    body: dict[str, Any] = {"name": folder_name, "mimeType": FOLDER_MIME_TYPE}
    if parent_folder_id:
        body["parents"] = [parent_folder_id]

    try:
        created = (
            service.files()
            .create(body=body, fields="id", supportsAllDrives=True)
            .execute()
        )
    except Exception as exc:
        raise GoogleDriveConfigError(
            f"Failed to create folder '{folder_name}' under parent '{parent_folder_id or 'root'}': {exc}"
        ) from exc

    folder_id = str(created.get("id") or "").strip()
    if not folder_id:
        raise GoogleDriveConfigError(f"Folder '{folder_name}' was created but no id was returned.")

    logger.info("Created Drive folder name=%s parent=%s id=%s", folder_name, parent_folder_id or "root", folder_id)

    return folder_id


def ensure_drive_folder_path(service, folder_path: str, root_folder_id: str | None = None) -> str:
    normalized = (folder_path or "").strip().strip("/")
    if not normalized:
        raise GoogleDriveConfigError("folder_path is required to resolve a Drive destination.")

    current_parent = (root_folder_id or "").strip() or None
    for part in [segment.strip() for segment in normalized.split("/") if segment.strip()]:
        folder_id = _find_folder_in_parent(service, part, current_parent)
        if folder_id is None:
            folder_id = _create_folder_in_parent(service, part, current_parent)
        current_parent = folder_id

    if not current_parent:
        raise GoogleDriveConfigError(f"Could not resolve Drive path '{folder_path}'.")

    logger.info("Resolved Drive path=%s to folder_id=%s", folder_path, current_parent)

    return current_parent


def move_file_to_path(service, file_id: str, destination_path: str, root_folder_id: str | None = None) -> str:
    normalized_file_id = (file_id or "").strip()
    if not normalized_file_id:
        raise GoogleDriveConfigError("file_id is required to move a Drive file.")

    destination_folder_id = ensure_drive_folder_path(service, destination_path, root_folder_id=root_folder_id)
    logger.info("Moving Drive file_id=%s to destination_path=%s", normalized_file_id, destination_path)

    try:
        metadata = (
            service.files()
            .get(fileId=normalized_file_id, fields="parents", supportsAllDrives=True)
            .execute()
        )
    except Exception as exc:
        raise GoogleDriveConfigError(f"Failed to read current parents for file '{normalized_file_id}': {exc}") from exc

    current_parents = [str(parent).strip() for parent in metadata.get("parents", []) if str(parent).strip()]

    if destination_folder_id in current_parents and len(current_parents) == 1:
        logger.info("Drive file already in destination file_id=%s folder_id=%s", normalized_file_id, destination_folder_id)
        return destination_folder_id

    request_kwargs: dict[str, Any] = {
        "fileId": normalized_file_id,
        "addParents": destination_folder_id,
        "supportsAllDrives": True,
        "fields": "id,parents",
    }
    remove_parents = ",".join(parent for parent in current_parents if parent != destination_folder_id)
    if remove_parents:
        request_kwargs["removeParents"] = remove_parents

    try:
        service.files().update(**request_kwargs).execute()
    except Exception as exc:
        raise GoogleDriveConfigError(
            f"Failed to move file '{normalized_file_id}' to Drive path '{destination_path}': {exc}"
        ) from exc

    logger.info("Moved Drive file_id=%s to folder_id=%s", normalized_file_id, destination_folder_id)

    return destination_folder_id