Anna Zhou commited on
Commit
2eb6704
·
1 Parent(s): 42d028c

Save case study requests to Drive

Browse files
Files changed (3) hide show
  1. README.md +8 -0
  2. app/app.py +23 -45
  3. app/drive_request_uploader.py +105 -0
README.md CHANGED
@@ -39,6 +39,14 @@ python rag/pipeline.py
39
  streamlit run app/app.py
40
  ```
41
 
 
 
 
 
 
 
 
 
42
  ## Ingesting documents
43
 
44
  ```bash
 
39
  streamlit run app/app.py
40
  ```
41
 
42
+ ## Hugging Face request intake
43
+
44
+ The **New Case Study Requests** tab saves each submitted request to Google Drive,
45
+ under folder `10n_-uOCT2GXu_G3r1qi5qJr8hHoy-E_y`. In Hugging Face Spaces, set
46
+ `GOOGLE_SERVICE_ACCOUNT_JSON` as a secret containing the service account JSON,
47
+ and share the Drive intake folder with that service account email. To use a
48
+ different parent folder, set `RFR_REQUESTS_DRIVE_FOLDER_ID`.
49
+
50
  ## Ingesting documents
51
 
52
  ```bash
app/app.py CHANGED
@@ -12,6 +12,7 @@ sys.path.insert(0, str(Path(__file__).parent.parent))
12
  from rag.pipeline import RAGProviderError, generate_answer, retrieve_chunks
13
  from ingest.geo_metadata import geographic_metadata, normalize_country
14
  sys.path.insert(0, str(Path(__file__).parent))
 
15
  from sheets_logger import log_query
16
 
17
 
@@ -56,18 +57,6 @@ def _history_dir() -> Path:
56
  return Path(".streamlit_history")
57
 
58
 
59
- def _requests_dir() -> Path:
60
- configured = os.getenv("RFR_REQUESTS_DIR")
61
- if configured:
62
- return Path(configured)
63
-
64
- bucket_mount = Path("/data")
65
- if bucket_mount.exists() and os.access(bucket_mount, os.W_OK):
66
- return bucket_mount / "new_case_requests"
67
-
68
- return Path(".new_case_requests")
69
-
70
-
71
  @st.cache_data
72
  def load_case_meta() -> dict[str, dict]:
73
  """Load case metadata for all case studies with local case_meta.json files."""
@@ -582,20 +571,6 @@ def _safe_slug(value: str, fallback: str) -> str:
582
  return cleaned[:60] or fallback
583
 
584
 
585
- def _append_request_index(record: dict) -> None:
586
- requests_dir = _requests_dir()
587
- requests_dir.mkdir(parents=True, exist_ok=True)
588
- index_path = requests_dir / "requests_index.json"
589
- try:
590
- existing = json.loads(index_path.read_text()) if index_path.exists() else []
591
- except Exception:
592
- existing = []
593
- if not isinstance(existing, list):
594
- existing = []
595
- existing.insert(0, record)
596
- index_path.write_text(json.dumps(existing[:500], indent=2))
597
-
598
-
599
  def _render_new_case_requests_page() -> None:
600
  st.title("New Case Study Requests")
601
  st.caption(
@@ -604,8 +579,8 @@ def _render_new_case_requests_page() -> None:
604
  )
605
 
606
  st.info(
607
- "Do not upload sensitive or confidential material. Uploaded files and request notes may be "
608
- "stored in the app's mounted Hugging Face storage so the project team can retrieve them."
609
  )
610
 
611
  with st.form("new_case_request_form", clear_on_submit=False):
@@ -636,7 +611,7 @@ def _render_new_case_requests_page() -> None:
636
  with st.expander("How the team processes submitted requests", expanded=False):
637
  st.markdown(
638
  """
639
- 1. Review the request files in the mounted storage folder.
640
  2. Create a new `CSxx` folder under `data/raw/` on a team machine.
641
  3. Move accepted papers into that folder and add or update `case_meta.json`.
642
  4. Run the one-provider extraction pipeline locally.
@@ -652,17 +627,7 @@ def _render_new_case_requests_page() -> None:
652
  created_at = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
653
  request_id = f"{created_at}_{uuid.uuid4().hex[:8]}"
654
  request_slug = _safe_slug(case_title or location or "new-case-request", "new-case-request")
655
- request_dir = _requests_dir() / f"{request_id}_{request_slug}"
656
- request_dir.mkdir(parents=True, exist_ok=True)
657
-
658
- saved_files = []
659
- for uploaded in uploaded_files or []:
660
- filename = _safe_filename(uploaded.name)
661
- dest = request_dir / filename
662
- if dest.exists():
663
- dest = request_dir / f"{uuid.uuid4().hex[:6]}_{filename}"
664
- dest.write_bytes(uploaded.getbuffer())
665
- saved_files.append(dest.name)
666
 
667
  record = {
668
  "request_id": request_id,
@@ -673,15 +638,28 @@ def _render_new_case_requests_page() -> None:
673
  "source_links": source_links.strip(),
674
  "notes": notes.strip(),
675
  "submitter": submitter.strip(),
676
- "saved_files": saved_files,
677
- "storage_folder": str(request_dir),
678
  "status": "new",
679
  }
680
- (request_dir / "request.json").write_text(json.dumps(record, indent=2))
681
- _append_request_index(record)
 
 
 
 
 
 
 
 
 
 
 
682
 
683
- st.success("Request submitted. The RFR team can now review it from the app storage folder.")
684
  st.markdown(f"**Request ID:** `{request_id}`")
 
 
 
685
  if saved_files:
686
  st.markdown("**Files saved:**")
687
  for filename in saved_files:
 
12
  from rag.pipeline import RAGProviderError, generate_answer, retrieve_chunks
13
  from ingest.geo_metadata import geographic_metadata, normalize_country
14
  sys.path.insert(0, str(Path(__file__).parent))
15
+ from drive_request_uploader import upload_case_request_to_drive
16
  from sheets_logger import log_query
17
 
18
 
 
57
  return Path(".streamlit_history")
58
 
59
 
 
 
 
 
 
 
 
 
 
 
 
 
60
  @st.cache_data
61
  def load_case_meta() -> dict[str, dict]:
62
  """Load case metadata for all case studies with local case_meta.json files."""
 
571
  return cleaned[:60] or fallback
572
 
573
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
574
  def _render_new_case_requests_page() -> None:
575
  st.title("New Case Study Requests")
576
  st.caption(
 
579
  )
580
 
581
  st.info(
582
+ "Do not upload sensitive or confidential material. Submitted files and request notes are "
583
+ "saved to the project team's Google Drive intake folder for review."
584
  )
585
 
586
  with st.form("new_case_request_form", clear_on_submit=False):
 
611
  with st.expander("How the team processes submitted requests", expanded=False):
612
  st.markdown(
613
  """
614
+ 1. Review the request files in the Google Drive intake folder.
615
  2. Create a new `CSxx` folder under `data/raw/` on a team machine.
616
  3. Move accepted papers into that folder and add or update `case_meta.json`.
617
  4. Run the one-provider extraction pipeline locally.
 
627
  created_at = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
628
  request_id = f"{created_at}_{uuid.uuid4().hex[:8]}"
629
  request_slug = _safe_slug(case_title or location or "new-case-request", "new-case-request")
630
+ request_folder_name = f"{request_id}_{request_slug}"
 
 
 
 
 
 
 
 
 
 
631
 
632
  record = {
633
  "request_id": request_id,
 
638
  "source_links": source_links.strip(),
639
  "notes": notes.strip(),
640
  "submitter": submitter.strip(),
641
+ "submitted_files": [_safe_filename(uploaded.name) for uploaded in uploaded_files or []],
 
642
  "status": "new",
643
  }
644
+ try:
645
+ drive_record = upload_case_request_to_drive(
646
+ request_folder_name=request_folder_name,
647
+ request_record=record,
648
+ uploaded_files=list(uploaded_files or []),
649
+ )
650
+ except Exception as exc:
651
+ st.error(
652
+ "The request could not be saved to Google Drive. Please try again later "
653
+ "or contact the RFR team."
654
+ )
655
+ st.caption(f"Drive upload error: {exc}")
656
+ return
657
 
658
+ st.success("Request submitted. The RFR team can now review it in the Google Drive intake folder.")
659
  st.markdown(f"**Request ID:** `{request_id}`")
660
+ if drive_record.get("drive_folder_url"):
661
+ st.markdown(f"**Drive folder:** [Open request folder]({drive_record['drive_folder_url']})")
662
+ saved_files = drive_record.get("saved_files") or []
663
  if saved_files:
664
  st.markdown("**Files saved:**")
665
  for filename in saved_files:
app/drive_request_uploader.py ADDED
@@ -0,0 +1,105 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Google Drive intake uploads for new case study requests.
2
+
3
+ Uses the same GOOGLE_SERVICE_ACCOUNT_JSON secret pattern as sheets_logger.py.
4
+ The target Drive folder must be shared with the service account email.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import io
10
+ import json
11
+ import mimetypes
12
+ import os
13
+ from typing import Any
14
+
15
+
16
+ DEFAULT_REQUESTS_DRIVE_FOLDER_ID = "10n_-uOCT2GXu_G3r1qi5qJr8hHoy-E_y"
17
+
18
+
19
+ def _drive_service():
20
+ try:
21
+ from google.oauth2.service_account import Credentials
22
+ from googleapiclient.discovery import build
23
+ except ImportError as exc:
24
+ raise RuntimeError(
25
+ "Google Drive libraries are not installed. Check requirements.txt."
26
+ ) from exc
27
+
28
+ creds_json = os.getenv("GOOGLE_SERVICE_ACCOUNT_JSON")
29
+ if not creds_json:
30
+ raise RuntimeError(
31
+ "GOOGLE_SERVICE_ACCOUNT_JSON is not set. Add it as a Hugging Face Space secret."
32
+ )
33
+
34
+ creds_dict = json.loads(creds_json)
35
+ scopes = ["https://www.googleapis.com/auth/drive"]
36
+ creds = Credentials.from_service_account_info(creds_dict, scopes=scopes)
37
+ return build("drive", "v3", credentials=creds)
38
+
39
+
40
+ def _create_folder(service, parent_folder_id: str, name: str) -> dict:
41
+ metadata = {
42
+ "name": name,
43
+ "mimeType": "application/vnd.google-apps.folder",
44
+ "parents": [parent_folder_id],
45
+ }
46
+ return service.files().create(body=metadata, fields="id,name,webViewLink").execute()
47
+
48
+
49
+ def _upload_bytes(
50
+ service,
51
+ parent_folder_id: str,
52
+ filename: str,
53
+ data: bytes,
54
+ mimetype: str | None = None,
55
+ ) -> dict:
56
+ from googleapiclient.http import MediaIoBaseUpload
57
+
58
+ resolved_mimetype = mimetype or mimetypes.guess_type(filename)[0] or "application/octet-stream"
59
+ media = MediaIoBaseUpload(io.BytesIO(data), mimetype=resolved_mimetype, resumable=False)
60
+ metadata = {"name": filename, "parents": [parent_folder_id]}
61
+ return (
62
+ service.files()
63
+ .create(body=metadata, media_body=media, fields="id,name,webViewLink")
64
+ .execute()
65
+ )
66
+
67
+
68
+ def upload_case_request_to_drive(
69
+ request_folder_name: str,
70
+ request_record: dict,
71
+ uploaded_files: list[Any],
72
+ ) -> dict:
73
+ parent_folder_id = os.getenv("RFR_REQUESTS_DRIVE_FOLDER_ID", DEFAULT_REQUESTS_DRIVE_FOLDER_ID)
74
+ service = _drive_service()
75
+
76
+ folder = _create_folder(service, parent_folder_id, request_folder_name)
77
+ folder_id = folder["id"]
78
+
79
+ uploaded = []
80
+ for uploaded_file in uploaded_files:
81
+ file_result = _upload_bytes(
82
+ service,
83
+ folder_id,
84
+ uploaded_file.name,
85
+ uploaded_file.getvalue(),
86
+ getattr(uploaded_file, "type", None),
87
+ )
88
+ uploaded.append(file_result)
89
+
90
+ final_record = {
91
+ **request_record,
92
+ "drive_folder_id": folder_id,
93
+ "drive_folder_url": folder.get("webViewLink") or f"https://drive.google.com/drive/folders/{folder_id}",
94
+ "saved_files": [item.get("name", "") for item in uploaded],
95
+ "uploaded_drive_files": uploaded,
96
+ }
97
+ _upload_bytes(
98
+ service,
99
+ folder_id,
100
+ "request.json",
101
+ json.dumps(final_record, indent=2).encode("utf-8"),
102
+ "application/json",
103
+ )
104
+
105
+ return final_record