""" onboard_case_study.py — Set up a new RFR case study end-to-end. What this script does: 1. Creates the local data/raw/CS/ directory structure 2. Creates a Google Drive folder "RFR - CS " under the RFR parent folder 3. Saves the new folder ID to case_study_pipeline/drive_folders.json 4. (Optional) Copies PDFs from an existing Drive folder you point it at into the new Drive folder, and downloads them into data/raw/CS/ too 5. Prints the filled-in paper discovery prompt (ready to paste into Consensus) Usage: python3 -m case_study_pipeline.onboard_case_study \\ --cs-num 16 \\ --name "Fairbourne, Wales" \\ --country "UK" \\ --hazard "coastal erosion and flooding" \\ --stage "pre-retreat" \\ --actors "Gwynedd Council" If you've already collected some papers in a Drive folder somewhere (e.g. a scratch folder from manual searching) and want them copied into the new case study's Drive folder *and* downloaded locally in one step, add: python3 -m case_study_pipeline.onboard_case_study \\ --cs-num 16 \\ --name "Fairbourne, Wales" \\ --country "UK" \\ --hazard "coastal erosion and flooding" \\ --stage "pre-retreat" \\ --source-drive-folder "https://drive.google.com/drive/folders/" (--source-drive-folder accepts either a full Drive folder URL or a bare folder ID.) After running: - Drop any remaining source PDFs into data/raw/CS/ - Run the pipeline: python3 -m case_study_pipeline.run_case_study --case-folder data/raw/CS - Outputs auto-upload to the correct Drive folder (no extra flags needed) """ from __future__ import annotations import argparse import io import json import re import sys from pathlib import Path REPO_ROOT = Path(__file__).resolve().parent.parent PIPELINE_DIR = Path(__file__).resolve().parent DRIVE_FOLDERS_FILE = PIPELINE_DIR / "drive_folders.json" PAPER_DISCOVERY_PROMPT = PIPELINE_DIR / "context" / "paper_discovery_consensus_prompt.md" # --------------------------------------------------------------------------- # Drive folder creation # --------------------------------------------------------------------------- def get_drive_service(credentials_file: Path, token_file: Path): """Authenticate once and return a Drive v3 service object (full read/write scope).""" try: from google.auth.transport.requests import Request from google.oauth2.credentials import Credentials from google_auth_oauthlib.flow import InstalledAppFlow from googleapiclient.discovery import build except ImportError: print(" [Drive] Google API libraries not installed.") print(" Run: pip install google-api-python-client google-auth-oauthlib google-auth-httplib2") return None SCOPES = ["https://www.googleapis.com/auth/drive"] creds = None if token_file.exists(): creds = Credentials.from_authorized_user_file(str(token_file), SCOPES) if not creds or not creds.valid: if creds and creds.expired and creds.refresh_token: creds.refresh(Request()) else: flow = InstalledAppFlow.from_client_secrets_file(str(credentials_file), SCOPES) creds = flow.run_local_server(port=0) token_file.write_text(creds.to_json()) return build("drive", "v3", credentials=creds) def create_drive_folder(service, folder_name: str, parent_folder_id: str) -> str: """Create a folder in Google Drive and return its ID.""" file_metadata = { "name": folder_name, "mimeType": "application/vnd.google-apps.folder", "parents": [parent_folder_id], } folder = service.files().create(body=file_metadata, fields="id,name").execute() return folder.get("id", "") def parse_drive_folder_id(value: str) -> str: """Accept either a bare Drive folder ID or a full Drive folder URL.""" match = re.search(r"/folders/([a-zA-Z0-9_-]+)", value) if match: return match.group(1) return value.strip() def copy_pdfs_from_drive_folder( service, source_folder_id: str, dest_folder_id: str, local_dest_dir: Path ) -> list[str]: """Copy every PDF in source_folder_id into dest_folder_id (a server-side Drive copy — file bytes never pass through this machine), and also download a local copy of each into local_dest_dir so the pipeline can read them. Skips files that already exist locally (by filename).""" from googleapiclient.http import MediaIoBaseDownload local_dest_dir.mkdir(parents=True, exist_ok=True) results = service.files().list( q=f"'{source_folder_id}' in parents and mimeType='application/pdf' and trashed=false", fields="files(id, name)", pageSize=100, ).execute() files = results.get("files", []) if not files: print(f" No PDFs found in source folder {source_folder_id}.") return [] copied = [] for f in files: name = f["name"] if not name.endswith(".pdf"): name += ".pdf" print(f" {name} ...", end="", flush=True) service.files().copy( fileId=f["id"], body={"parents": [dest_folder_id], "name": name}, ).execute() print(" copied to Drive", end="", flush=True) dest_path = local_dest_dir / name if dest_path.exists(): print(" [already local]") else: request = service.files().get_media(fileId=f["id"]) buf = io.BytesIO() downloader = MediaIoBaseDownload(buf, request) done = False while not done: _, done = downloader.next_chunk() dest_path.write_bytes(buf.getvalue()) print(f", saved locally ({len(buf.getvalue()):,} bytes)") copied.append(name) return copied # --------------------------------------------------------------------------- # drive_folders.json helpers # --------------------------------------------------------------------------- def load_drive_folders() -> dict: if DRIVE_FOLDERS_FILE.exists(): return json.loads(DRIVE_FOLDERS_FILE.read_text()) return {"_comment": "Maps case study IDs to Google Drive folder IDs.", "_parent_folder_id": ""} def save_drive_folders(data: dict) -> None: DRIVE_FOLDERS_FILE.write_text(json.dumps(data, indent=2) + "\n") def get_folder_id(case_id: str) -> str | None: """Look up a Drive folder ID by case ID (e.g. 'CS12'). Returns None if not found.""" data = load_drive_folders() return data.get(case_id) # --------------------------------------------------------------------------- # Paper discovery prompt filler # --------------------------------------------------------------------------- def fill_paper_discovery_prompt(cs_num: int, name: str, country: str, hazard: str, stage: str, actors: str) -> str: template = PAPER_DISCOVERY_PROMPT.read_text() # Replace the CASE PROFILE placeholders filled = template.replace("", str(cs_num)) filled = filled.replace("", name) filled = filled.replace(", ", country) filled = filled.replace( "", hazard, ) filled = filled.replace( "", stage, ) filled = filled.replace("", actors or "unknown") # Fill BASE QUERY placeholders filled = filled.replace("", name) filled = filled.replace("", country) filled = filled.replace("", hazard) filled = filled.replace("", country) filled = filled.replace("", hazard) filled = filled.replace("", name) return filled # --------------------------------------------------------------------------- # Local directory setup # --------------------------------------------------------------------------- def create_local_structure(cs_num: int) -> Path: case_dir = REPO_ROOT / "data" / "raw" / f"CS{cs_num}" case_dir.mkdir(parents=True, exist_ok=True) (case_dir / "pipeline_output").mkdir(exist_ok=True) return case_dir # --------------------------------------------------------------------------- # Main # --------------------------------------------------------------------------- def main() -> None: parser = argparse.ArgumentParser( description="Onboard a new RFR case study: create Drive folder, local dirs, and paper discovery prompt." ) parser.add_argument("--cs-num", required=True, type=int, help="Case study number, e.g. 16") parser.add_argument("--name", required=True, help="Place name, e.g. 'Fairbourne, Wales'") parser.add_argument("--country", required=True, help="Country (and region), e.g. 'UK, Wales'") parser.add_argument("--hazard", required=True, help="Primary hazard, e.g. 'coastal erosion and flooding'") parser.add_argument("--stage", default="unknown", help="MR stage: pre-retreat / during / post-retreat") parser.add_argument("--actors", default="", help="Known actors or program name (optional)") parser.add_argument( "--drive-credentials", help="Path to OAuth credentials JSON (defaults to client_secret_*.json in repo root)", ) parser.add_argument( "--skip-drive", action="store_true", help="Skip Drive folder creation (local setup only)", ) parser.add_argument( "--source-drive-folder", help=( "Drive folder ID or URL containing PDFs you've already collected. " "They are copied into the new case study's Drive folder and " "downloaded into data/raw/CS/ as well." ), ) args = parser.parse_args() case_id = f"CS{args.cs_num}" folder_name = f"RFR - {case_id} {args.name}" print(f"\n{'='*60}") print(f" Onboarding {case_id}: {args.name}") print(f"{'='*60}\n") # 1. Local directory case_dir = create_local_structure(args.cs_num) print(f"✓ Local directory: {case_dir}") print(f" Drop source PDFs into: {case_dir}/") # 2. Drive folder drive_data = load_drive_folders() parent_folder_id = drive_data.get("_parent_folder_id", "") folder_id = drive_data.get(case_id) # may already be registered from a prior run # We need an authenticated Drive service if we have to create the folder, # or if we need to copy PDFs in from a source folder. need_service = (not args.skip_drive and not folder_id) or bool(args.source_drive_folder) service = None if need_service: creds_path = None if args.drive_credentials: creds_path = Path(args.drive_credentials) else: matches = list(REPO_ROOT.glob("client_secret*.json")) if matches: creds_path = matches[0] if not creds_path or not creds_path.exists(): print(f"\n⚠ Could not find OAuth credentials file. Pass --drive-credentials .") else: token_file = Path.home() / ".rfr-drive-token.json" service = get_drive_service(creds_path, token_file) if args.skip_drive: print(f"\n⚠ Skipping Drive folder creation (--skip-drive). Add folder ID manually to drive_folders.json.") elif folder_id: print(f"\n✓ Drive folder already registered: {folder_id}") elif not parent_folder_id or parent_folder_id == "FILL_IN_RFR_PARENT_FOLDER_ID": print(f"\n⚠ Parent Drive folder ID not set in drive_folders.json (_parent_folder_id).") print(f" Set it to the ID of the main RFR Drive folder, then re-run.") print(f" (The ID is the last segment of the folder's URL.)") elif service: print(f"\nCreating Drive folder '{folder_name}'...") folder_id = create_drive_folder(service, folder_name, parent_folder_id) if folder_id: drive_data[case_id] = folder_id save_drive_folders(drive_data) print(f"✓ Drive folder created: https://drive.google.com/drive/folders/{folder_id}") print(f"✓ drive_folders.json updated") else: print(f"✗ Drive folder creation failed. Add the folder ID manually to drive_folders.json.") else: print(f"\n⚠ Could not create Drive folder (no credentials available).") # 2b. Copy PDFs in from an existing Drive folder, if one was given if args.source_drive_folder: if not service: print(f"\n⚠ --source-drive-folder given but no Drive credentials available — skipping copy.") elif not folder_id: print(f"\n⚠ --source-drive-folder given but the destination Drive folder isn't set up — skipping copy.") else: source_id = parse_drive_folder_id(args.source_drive_folder) local_dest = REPO_ROOT / "data" / "raw" / case_id print(f"\nCopying PDFs from source folder ({source_id}) into '{folder_name}'...") copied = copy_pdfs_from_drive_folder(service, source_id, folder_id, local_dest) print(f"✓ Copied {len(copied)} PDF(s) into the Drive folder and into {local_dest}/") # 3. Write case_meta.json so the ingestion bridge can tag RAG chunks import json as _json _meta_path = REPO_ROOT / "data" / "raw" / case_id / "case_meta.json" _meta_path.write_text(_json.dumps({ "case_id": case_id, "name": args.name, "location": args.name, # user can edit this later for a finer location string "country": args.country, }, indent=2)) print(f"✓ case_meta.json written: {_meta_path}") # 4. Paper discovery prompt prompt_output = REPO_ROOT / "data" / "raw" / f"CS{args.cs_num}" / f"CS{args.cs_num}_paper_discovery_prompt.md" filled = fill_paper_discovery_prompt( args.cs_num, args.name, args.country, args.hazard, args.stage, args.actors ) prompt_output.write_text(filled) print(f"\n✓ Paper discovery prompt saved: {prompt_output}") # 4. Print next steps print(f"\n{'='*60}") print(f" NEXT STEPS for {case_id}") print(f"{'='*60}") print(f"1. Open {prompt_output.name} and run the queries in Consensus") print(f"2. Drop the source PDFs into: data/raw/{case_id}/") print(f"3. Run the pipeline:") print(f" python3 -m case_study_pipeline.run_case_study --case-folder data/raw/{case_id}") print(f" (Outputs will auto-upload to the '{folder_name}' Drive folder)") print() if __name__ == "__main__": main()