Spaces:
Sleeping
Sleeping
File size: 2,245 Bytes
db9f03c cb04ca0 db9f03c cb04ca0 db9f03c cb04ca0 db9f03c cb04ca0 db9f03c cb04ca0 db9f03c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 | from __future__ import annotations
import shutil
from dataclasses import dataclass
from pathlib import Path
from dotenv import load_dotenv
from firebase_processed_files import FirebaseProcessedFilesTracker, check_files_already_processed
from google_drive_service import (
DriveDocument,
GoogleDriveConfigError,
load_drive_folder_ids_from_env,
scan_drive_supported_documents,
)
load_dotenv()
@dataclass
class DriveScanResult:
files_to_process: list[str]
skipped_files: list[tuple[str, str]]
discovered_count: int
temp_dir: str
warning: str | None = None
def scan_google_drive_unprocessed_files(
tracker: FirebaseProcessedFilesTracker | None,
include_subfolders: bool = True,
) -> DriveScanResult:
folder_ids = load_drive_folder_ids_from_env()
scan_result = scan_drive_supported_documents(folder_ids=folder_ids, include_subfolders=include_subfolders)
downloaded_paths = [document.local_path for document in scan_result.documents]
files_to_process, skipped_files, warning = check_files_already_processed(tracker, downloaded_paths)
skipped_set = set(path for path, _ in skipped_files)
for downloaded in downloaded_paths:
path = Path(downloaded)
if path.name in skipped_set:
try:
path.unlink(missing_ok=True)
except Exception:
pass
return DriveScanResult(
files_to_process=files_to_process,
skipped_files=skipped_files,
discovered_count=scan_result.discovered_count,
temp_dir=scan_result.temp_dir,
warning=warning,
)
def scan_google_drive_supported_documents(include_subfolders: bool = True) -> tuple[list[DriveDocument], str]:
"""Scans configured Drive folders and downloads supported documents with metadata."""
folder_ids = load_drive_folder_ids_from_env()
scan_result = scan_drive_supported_documents(folder_ids=folder_ids, include_subfolders=include_subfolders)
return scan_result.documents, scan_result.temp_dir
def cleanup_drive_scan_result(scan_result: DriveScanResult | None) -> None:
if scan_result is None:
return
if scan_result.temp_dir:
shutil.rmtree(scan_result.temp_dir, ignore_errors=True) |