Spaces:
Sleeping
Sleeping
Download case_study_pipeline/drive_upload.py from UWRFR/UWaterlooRFRChatBot: direct link, hf CLI and curl.
- Browser
- Download file 8.48 kB
-
https://huggingface.co/spaces/UWRFR/UWaterlooRFRChatBot/resolve/main/case_study_pipeline/drive_upload.py
- Command line
-
hf download hf://spaces/UWRFR/UWaterlooRFRChatBot/case_study_pipeline/drive_upload.py
-
curl -L -o drive_upload.py https://huggingface.co/spaces/UWRFR/UWaterlooRFRChatBot/resolve/main/case_study_pipeline/drive_upload.py
8.48 kB
| """ | |
| Google Drive upload helper for the RFR case study pipeline. | |
| Uploads pipeline output PDFs to a specified Google Drive folder immediately | |
| after each step writes them to disk. Uses OAuth2 credentials so the pipeline | |
| can run headlessly after a one-time browser authorization. | |
| SETUP (one time per machine) | |
| ----------------------------- | |
| 1. In Google Cloud Console, create a project, enable the Drive API, and | |
| create OAuth2 credentials (Desktop app). Download as credentials.json. | |
| 2. Add these two lines to rfr-rag/.env: | |
| GOOGLE_CREDENTIALS_FILE=/path/to/credentials.json | |
| GOOGLE_DRIVE_FOLDER_ID=<your-folder-id> | |
| The folder ID is the last segment of the Drive folder URL. | |
| 3. On the very first run the script opens a browser tab for authorization. | |
| After you approve, a token is saved to ~/.rfr-drive-token.json and all | |
| subsequent runs are silent β no flags needed. | |
| USAGE IN THE PIPELINE | |
| ---------------------- | |
| Pass --drive-folder-id <ID> to any pipeline entry point. The folder ID is | |
| the last segment of the Drive folder's URL, e.g.: | |
| https://drive.google.com/drive/folders/11vGsMiYTaR2wERyiISjrwBq6akVzzZH6 | |
| ^^^^^^^^^^^^^^^^^^^^^^^^ | |
| If --drive-folder-id is omitted, Drive uploads are silently skipped. | |
| If credentials are missing or the upload fails, a warning is printed and | |
| the pipeline continues -- Drive errors never abort a run. | |
| """ | |
| from __future__ import annotations | |
| import os | |
| from pathlib import Path | |
| from typing import Optional | |
| # Use the same scope as the downloader and onboarding flow. The narrower | |
| # drive.file scope cannot upload into folders the app did not create itself. | |
| SCOPES = ["https://www.googleapis.com/auth/drive"] | |
| _DEFAULT_TOKEN_FILE = Path.home() / ".rfr-drive-token.json" | |
| class DriveUploader: | |
| """ | |
| Thin wrapper around the Google Drive v3 API for uploading PDFs. | |
| Instantiate once per pipeline run and call upload_pdf() after each file | |
| is written. Credentials are resolved in this order: | |
| 1. ``credentials_file`` argument | |
| 2. ``GOOGLE_CREDENTIALS_FILE`` environment variable | |
| 3. Already-saved OAuth token at ~/.rfr-drive-token.json (refresh only) | |
| """ | |
| def __init__( | |
| self, | |
| folder_id: str, | |
| credentials_file: str | Path | None = None, | |
| token_file: str | Path | None = None, | |
| ) -> None: | |
| self.folder_id = folder_id | |
| self._token_file = Path(token_file) if token_file else _DEFAULT_TOKEN_FILE | |
| self._service = self._build_service(credentials_file) | |
| # ------------------------------------------------------------------ | |
| # Internal auth helpers | |
| # ------------------------------------------------------------------ | |
| def _build_service(self, credentials_file: str | Path | None): | |
| """Authenticate and return a Drive v3 service object.""" | |
| # Lazy import so the pipeline doesn't crash when google libs are absent | |
| # and Drive upload wasn't requested. | |
| try: | |
| from google.auth.transport.requests import Request | |
| from google.oauth2.credentials import Credentials | |
| from google_auth_oauthlib.flow import InstalledAppFlow | |
| from googleapiclient.discovery import build | |
| except ImportError as exc: | |
| raise ImportError( | |
| "Google API client libraries are not installed.\n" | |
| "Run: pip install google-api-python-client google-auth-oauthlib google-auth-httplib2" | |
| ) from exc | |
| creds: Optional[Credentials] = None | |
| # 1. Try to load an existing token (saved from a previous auth). | |
| if self._token_file.exists(): | |
| creds = Credentials.from_authorized_user_file(str(self._token_file), SCOPES) | |
| # 2. Refresh or re-authorize as needed. | |
| if not creds or not creds.valid: | |
| if creds and creds.expired and creds.refresh_token: | |
| creds.refresh(Request()) | |
| else: | |
| creds_path = self._resolve_credentials_file(credentials_file) | |
| flow = InstalledAppFlow.from_client_secrets_file(str(creds_path), SCOPES) | |
| creds = flow.run_local_server(port=0) | |
| # Persist the (possibly refreshed) token for future runs. | |
| self._token_file.write_text(creds.to_json()) | |
| return build("drive", "v3", credentials=creds) | |
| def _resolve_credentials_file(credentials_file: str | Path | None) -> Path: | |
| """Return the credentials.json path, or raise a clear error.""" | |
| if credentials_file: | |
| path = Path(credentials_file) | |
| if path.exists(): | |
| return path | |
| raise FileNotFoundError(f"credentials file not found: {path}") | |
| env_val = os.getenv("GOOGLE_CREDENTIALS_FILE") | |
| if env_val: | |
| path = Path(env_val) | |
| if path.exists(): | |
| return path | |
| raise FileNotFoundError( | |
| f"GOOGLE_CREDENTIALS_FILE points to a missing file: {path}" | |
| ) | |
| raise FileNotFoundError( | |
| "No Google OAuth2 credentials file found.\n" | |
| "Either set GOOGLE_CREDENTIALS_FILE=/path/to/credentials.json in .env,\n" | |
| "or pass --drive-credentials /path/to/credentials.json at the command line." | |
| ) | |
| # ------------------------------------------------------------------ | |
| # Public API | |
| # ------------------------------------------------------------------ | |
| def upload_pdf(self, pdf_path: Path, filename: str | None = None) -> str: | |
| """ | |
| Upload *pdf_path* to self.folder_id. | |
| Parameters | |
| ---------- | |
| pdf_path: | |
| Local path to the PDF file to upload. | |
| filename: | |
| Name to give the file in Drive (defaults to pdf_path.name). | |
| Returns | |
| ------- | |
| str | |
| The Drive file ID of the newly created file. | |
| """ | |
| from googleapiclient.http import MediaFileUpload # type: ignore[import] | |
| name = filename or pdf_path.name | |
| file_metadata = {"name": name, "parents": [self.folder_id]} | |
| media = MediaFileUpload(str(pdf_path), mimetype="application/pdf", resumable=False) | |
| result = ( | |
| self._service.files() | |
| .create(body=file_metadata, media_body=media, fields="id,name") | |
| .execute() | |
| ) | |
| drive_id: str = result.get("id", "") | |
| print(f" [Drive] Uploaded '{name}' β https://drive.google.com/file/d/{drive_id}/view") | |
| return drive_id | |
| # --------------------------------------------------------------------------- | |
| # Factory β used by pipeline entry points | |
| # --------------------------------------------------------------------------- | |
| def make_uploader( | |
| folder_id: str | None = None, | |
| credentials_file: str | Path | None = None, | |
| token_file: str | Path | None = None, | |
| ) -> Optional["DriveUploader"]: | |
| """ | |
| Return a DriveUploader if a folder ID is available, else None. | |
| The folder ID is resolved in this order: | |
| 1. ``folder_id`` argument (e.g. from --drive-folder-id on the CLI) | |
| 2. ``GOOGLE_DRIVE_FOLDER_ID`` environment variable / .env entry | |
| Failures (missing libs, bad credentials) are caught and printed as | |
| warnings so the pipeline can continue without Drive uploads. | |
| """ | |
| resolved_folder_id = folder_id or os.getenv("GOOGLE_DRIVE_FOLDER_ID") | |
| if not resolved_folder_id: | |
| return None | |
| folder_id = resolved_folder_id | |
| try: | |
| uploader = DriveUploader( | |
| folder_id=folder_id, | |
| credentials_file=credentials_file, | |
| token_file=token_file, | |
| ) | |
| print(f" [Drive] Uploader ready (folder: {folder_id})") | |
| return uploader | |
| except Exception as exc: | |
| print(f"\n [Drive] Warning: could not initialize uploader β {exc}") | |
| print(" [Drive] Drive uploads will be skipped for this run.\n") | |
| return None | |
| def try_upload(uploader: Optional["DriveUploader"], pdf_path: Path | None) -> None: | |
| """ | |
| Upload *pdf_path* if both *uploader* and *pdf_path* are non-None. | |
| Errors are caught and printed as warnings β they never propagate to the | |
| caller, so a transient Drive error can't abort a pipeline run. | |
| """ | |
| if uploader is None or pdf_path is None: | |
| return | |
| try: | |
| uploader.upload_pdf(pdf_path) | |
| except Exception as exc: | |
| print(f" [Drive] Warning: upload of '{pdf_path.name}' failed β {exc}") | |