Spaces:
Sleeping
Sleeping
Anna Zhou commited on
Commit ·
2eb6704
1
Parent(s): 42d028c
Save case study requests to Drive
Browse files- README.md +8 -0
- app/app.py +23 -45
- app/drive_request_uploader.py +105 -0
README.md
CHANGED
|
@@ -39,6 +39,14 @@ python rag/pipeline.py
|
|
| 39 |
streamlit run app/app.py
|
| 40 |
```
|
| 41 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 42 |
## Ingesting documents
|
| 43 |
|
| 44 |
```bash
|
|
|
|
| 39 |
streamlit run app/app.py
|
| 40 |
```
|
| 41 |
|
| 42 |
+
## Hugging Face request intake
|
| 43 |
+
|
| 44 |
+
The **New Case Study Requests** tab saves each submitted request to Google Drive,
|
| 45 |
+
under folder `10n_-uOCT2GXu_G3r1qi5qJr8hHoy-E_y`. In Hugging Face Spaces, set
|
| 46 |
+
`GOOGLE_SERVICE_ACCOUNT_JSON` as a secret containing the service account JSON,
|
| 47 |
+
and share the Drive intake folder with that service account email. To use a
|
| 48 |
+
different parent folder, set `RFR_REQUESTS_DRIVE_FOLDER_ID`.
|
| 49 |
+
|
| 50 |
## Ingesting documents
|
| 51 |
|
| 52 |
```bash
|
app/app.py
CHANGED
|
@@ -12,6 +12,7 @@ sys.path.insert(0, str(Path(__file__).parent.parent))
|
|
| 12 |
from rag.pipeline import RAGProviderError, generate_answer, retrieve_chunks
|
| 13 |
from ingest.geo_metadata import geographic_metadata, normalize_country
|
| 14 |
sys.path.insert(0, str(Path(__file__).parent))
|
|
|
|
| 15 |
from sheets_logger import log_query
|
| 16 |
|
| 17 |
|
|
@@ -56,18 +57,6 @@ def _history_dir() -> Path:
|
|
| 56 |
return Path(".streamlit_history")
|
| 57 |
|
| 58 |
|
| 59 |
-
def _requests_dir() -> Path:
|
| 60 |
-
configured = os.getenv("RFR_REQUESTS_DIR")
|
| 61 |
-
if configured:
|
| 62 |
-
return Path(configured)
|
| 63 |
-
|
| 64 |
-
bucket_mount = Path("/data")
|
| 65 |
-
if bucket_mount.exists() and os.access(bucket_mount, os.W_OK):
|
| 66 |
-
return bucket_mount / "new_case_requests"
|
| 67 |
-
|
| 68 |
-
return Path(".new_case_requests")
|
| 69 |
-
|
| 70 |
-
|
| 71 |
@st.cache_data
|
| 72 |
def load_case_meta() -> dict[str, dict]:
|
| 73 |
"""Load case metadata for all case studies with local case_meta.json files."""
|
|
@@ -582,20 +571,6 @@ def _safe_slug(value: str, fallback: str) -> str:
|
|
| 582 |
return cleaned[:60] or fallback
|
| 583 |
|
| 584 |
|
| 585 |
-
def _append_request_index(record: dict) -> None:
|
| 586 |
-
requests_dir = _requests_dir()
|
| 587 |
-
requests_dir.mkdir(parents=True, exist_ok=True)
|
| 588 |
-
index_path = requests_dir / "requests_index.json"
|
| 589 |
-
try:
|
| 590 |
-
existing = json.loads(index_path.read_text()) if index_path.exists() else []
|
| 591 |
-
except Exception:
|
| 592 |
-
existing = []
|
| 593 |
-
if not isinstance(existing, list):
|
| 594 |
-
existing = []
|
| 595 |
-
existing.insert(0, record)
|
| 596 |
-
index_path.write_text(json.dumps(existing[:500], indent=2))
|
| 597 |
-
|
| 598 |
-
|
| 599 |
def _render_new_case_requests_page() -> None:
|
| 600 |
st.title("New Case Study Requests")
|
| 601 |
st.caption(
|
|
@@ -604,8 +579,8 @@ def _render_new_case_requests_page() -> None:
|
|
| 604 |
)
|
| 605 |
|
| 606 |
st.info(
|
| 607 |
-
"Do not upload sensitive or confidential material.
|
| 608 |
-
"
|
| 609 |
)
|
| 610 |
|
| 611 |
with st.form("new_case_request_form", clear_on_submit=False):
|
|
@@ -636,7 +611,7 @@ def _render_new_case_requests_page() -> None:
|
|
| 636 |
with st.expander("How the team processes submitted requests", expanded=False):
|
| 637 |
st.markdown(
|
| 638 |
"""
|
| 639 |
-
1. Review the request files in the
|
| 640 |
2. Create a new `CSxx` folder under `data/raw/` on a team machine.
|
| 641 |
3. Move accepted papers into that folder and add or update `case_meta.json`.
|
| 642 |
4. Run the one-provider extraction pipeline locally.
|
|
@@ -652,17 +627,7 @@ def _render_new_case_requests_page() -> None:
|
|
| 652 |
created_at = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
|
| 653 |
request_id = f"{created_at}_{uuid.uuid4().hex[:8]}"
|
| 654 |
request_slug = _safe_slug(case_title or location or "new-case-request", "new-case-request")
|
| 655 |
-
|
| 656 |
-
request_dir.mkdir(parents=True, exist_ok=True)
|
| 657 |
-
|
| 658 |
-
saved_files = []
|
| 659 |
-
for uploaded in uploaded_files or []:
|
| 660 |
-
filename = _safe_filename(uploaded.name)
|
| 661 |
-
dest = request_dir / filename
|
| 662 |
-
if dest.exists():
|
| 663 |
-
dest = request_dir / f"{uuid.uuid4().hex[:6]}_{filename}"
|
| 664 |
-
dest.write_bytes(uploaded.getbuffer())
|
| 665 |
-
saved_files.append(dest.name)
|
| 666 |
|
| 667 |
record = {
|
| 668 |
"request_id": request_id,
|
|
@@ -673,15 +638,28 @@ def _render_new_case_requests_page() -> None:
|
|
| 673 |
"source_links": source_links.strip(),
|
| 674 |
"notes": notes.strip(),
|
| 675 |
"submitter": submitter.strip(),
|
| 676 |
-
"
|
| 677 |
-
"storage_folder": str(request_dir),
|
| 678 |
"status": "new",
|
| 679 |
}
|
| 680 |
-
|
| 681 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 682 |
|
| 683 |
-
st.success("Request submitted. The RFR team can now review it
|
| 684 |
st.markdown(f"**Request ID:** `{request_id}`")
|
|
|
|
|
|
|
|
|
|
| 685 |
if saved_files:
|
| 686 |
st.markdown("**Files saved:**")
|
| 687 |
for filename in saved_files:
|
|
|
|
| 12 |
from rag.pipeline import RAGProviderError, generate_answer, retrieve_chunks
|
| 13 |
from ingest.geo_metadata import geographic_metadata, normalize_country
|
| 14 |
sys.path.insert(0, str(Path(__file__).parent))
|
| 15 |
+
from drive_request_uploader import upload_case_request_to_drive
|
| 16 |
from sheets_logger import log_query
|
| 17 |
|
| 18 |
|
|
|
|
| 57 |
return Path(".streamlit_history")
|
| 58 |
|
| 59 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 60 |
@st.cache_data
|
| 61 |
def load_case_meta() -> dict[str, dict]:
|
| 62 |
"""Load case metadata for all case studies with local case_meta.json files."""
|
|
|
|
| 571 |
return cleaned[:60] or fallback
|
| 572 |
|
| 573 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 574 |
def _render_new_case_requests_page() -> None:
|
| 575 |
st.title("New Case Study Requests")
|
| 576 |
st.caption(
|
|
|
|
| 579 |
)
|
| 580 |
|
| 581 |
st.info(
|
| 582 |
+
"Do not upload sensitive or confidential material. Submitted files and request notes are "
|
| 583 |
+
"saved to the project team's Google Drive intake folder for review."
|
| 584 |
)
|
| 585 |
|
| 586 |
with st.form("new_case_request_form", clear_on_submit=False):
|
|
|
|
| 611 |
with st.expander("How the team processes submitted requests", expanded=False):
|
| 612 |
st.markdown(
|
| 613 |
"""
|
| 614 |
+
1. Review the request files in the Google Drive intake folder.
|
| 615 |
2. Create a new `CSxx` folder under `data/raw/` on a team machine.
|
| 616 |
3. Move accepted papers into that folder and add or update `case_meta.json`.
|
| 617 |
4. Run the one-provider extraction pipeline locally.
|
|
|
|
| 627 |
created_at = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
|
| 628 |
request_id = f"{created_at}_{uuid.uuid4().hex[:8]}"
|
| 629 |
request_slug = _safe_slug(case_title or location or "new-case-request", "new-case-request")
|
| 630 |
+
request_folder_name = f"{request_id}_{request_slug}"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 631 |
|
| 632 |
record = {
|
| 633 |
"request_id": request_id,
|
|
|
|
| 638 |
"source_links": source_links.strip(),
|
| 639 |
"notes": notes.strip(),
|
| 640 |
"submitter": submitter.strip(),
|
| 641 |
+
"submitted_files": [_safe_filename(uploaded.name) for uploaded in uploaded_files or []],
|
|
|
|
| 642 |
"status": "new",
|
| 643 |
}
|
| 644 |
+
try:
|
| 645 |
+
drive_record = upload_case_request_to_drive(
|
| 646 |
+
request_folder_name=request_folder_name,
|
| 647 |
+
request_record=record,
|
| 648 |
+
uploaded_files=list(uploaded_files or []),
|
| 649 |
+
)
|
| 650 |
+
except Exception as exc:
|
| 651 |
+
st.error(
|
| 652 |
+
"The request could not be saved to Google Drive. Please try again later "
|
| 653 |
+
"or contact the RFR team."
|
| 654 |
+
)
|
| 655 |
+
st.caption(f"Drive upload error: {exc}")
|
| 656 |
+
return
|
| 657 |
|
| 658 |
+
st.success("Request submitted. The RFR team can now review it in the Google Drive intake folder.")
|
| 659 |
st.markdown(f"**Request ID:** `{request_id}`")
|
| 660 |
+
if drive_record.get("drive_folder_url"):
|
| 661 |
+
st.markdown(f"**Drive folder:** [Open request folder]({drive_record['drive_folder_url']})")
|
| 662 |
+
saved_files = drive_record.get("saved_files") or []
|
| 663 |
if saved_files:
|
| 664 |
st.markdown("**Files saved:**")
|
| 665 |
for filename in saved_files:
|
app/drive_request_uploader.py
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Google Drive intake uploads for new case study requests.
|
| 2 |
+
|
| 3 |
+
Uses the same GOOGLE_SERVICE_ACCOUNT_JSON secret pattern as sheets_logger.py.
|
| 4 |
+
The target Drive folder must be shared with the service account email.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
import io
|
| 10 |
+
import json
|
| 11 |
+
import mimetypes
|
| 12 |
+
import os
|
| 13 |
+
from typing import Any
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
DEFAULT_REQUESTS_DRIVE_FOLDER_ID = "10n_-uOCT2GXu_G3r1qi5qJr8hHoy-E_y"
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def _drive_service():
|
| 20 |
+
try:
|
| 21 |
+
from google.oauth2.service_account import Credentials
|
| 22 |
+
from googleapiclient.discovery import build
|
| 23 |
+
except ImportError as exc:
|
| 24 |
+
raise RuntimeError(
|
| 25 |
+
"Google Drive libraries are not installed. Check requirements.txt."
|
| 26 |
+
) from exc
|
| 27 |
+
|
| 28 |
+
creds_json = os.getenv("GOOGLE_SERVICE_ACCOUNT_JSON")
|
| 29 |
+
if not creds_json:
|
| 30 |
+
raise RuntimeError(
|
| 31 |
+
"GOOGLE_SERVICE_ACCOUNT_JSON is not set. Add it as a Hugging Face Space secret."
|
| 32 |
+
)
|
| 33 |
+
|
| 34 |
+
creds_dict = json.loads(creds_json)
|
| 35 |
+
scopes = ["https://www.googleapis.com/auth/drive"]
|
| 36 |
+
creds = Credentials.from_service_account_info(creds_dict, scopes=scopes)
|
| 37 |
+
return build("drive", "v3", credentials=creds)
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def _create_folder(service, parent_folder_id: str, name: str) -> dict:
|
| 41 |
+
metadata = {
|
| 42 |
+
"name": name,
|
| 43 |
+
"mimeType": "application/vnd.google-apps.folder",
|
| 44 |
+
"parents": [parent_folder_id],
|
| 45 |
+
}
|
| 46 |
+
return service.files().create(body=metadata, fields="id,name,webViewLink").execute()
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def _upload_bytes(
|
| 50 |
+
service,
|
| 51 |
+
parent_folder_id: str,
|
| 52 |
+
filename: str,
|
| 53 |
+
data: bytes,
|
| 54 |
+
mimetype: str | None = None,
|
| 55 |
+
) -> dict:
|
| 56 |
+
from googleapiclient.http import MediaIoBaseUpload
|
| 57 |
+
|
| 58 |
+
resolved_mimetype = mimetype or mimetypes.guess_type(filename)[0] or "application/octet-stream"
|
| 59 |
+
media = MediaIoBaseUpload(io.BytesIO(data), mimetype=resolved_mimetype, resumable=False)
|
| 60 |
+
metadata = {"name": filename, "parents": [parent_folder_id]}
|
| 61 |
+
return (
|
| 62 |
+
service.files()
|
| 63 |
+
.create(body=metadata, media_body=media, fields="id,name,webViewLink")
|
| 64 |
+
.execute()
|
| 65 |
+
)
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
def upload_case_request_to_drive(
|
| 69 |
+
request_folder_name: str,
|
| 70 |
+
request_record: dict,
|
| 71 |
+
uploaded_files: list[Any],
|
| 72 |
+
) -> dict:
|
| 73 |
+
parent_folder_id = os.getenv("RFR_REQUESTS_DRIVE_FOLDER_ID", DEFAULT_REQUESTS_DRIVE_FOLDER_ID)
|
| 74 |
+
service = _drive_service()
|
| 75 |
+
|
| 76 |
+
folder = _create_folder(service, parent_folder_id, request_folder_name)
|
| 77 |
+
folder_id = folder["id"]
|
| 78 |
+
|
| 79 |
+
uploaded = []
|
| 80 |
+
for uploaded_file in uploaded_files:
|
| 81 |
+
file_result = _upload_bytes(
|
| 82 |
+
service,
|
| 83 |
+
folder_id,
|
| 84 |
+
uploaded_file.name,
|
| 85 |
+
uploaded_file.getvalue(),
|
| 86 |
+
getattr(uploaded_file, "type", None),
|
| 87 |
+
)
|
| 88 |
+
uploaded.append(file_result)
|
| 89 |
+
|
| 90 |
+
final_record = {
|
| 91 |
+
**request_record,
|
| 92 |
+
"drive_folder_id": folder_id,
|
| 93 |
+
"drive_folder_url": folder.get("webViewLink") or f"https://drive.google.com/drive/folders/{folder_id}",
|
| 94 |
+
"saved_files": [item.get("name", "") for item in uploaded],
|
| 95 |
+
"uploaded_drive_files": uploaded,
|
| 96 |
+
}
|
| 97 |
+
_upload_bytes(
|
| 98 |
+
service,
|
| 99 |
+
folder_id,
|
| 100 |
+
"request.json",
|
| 101 |
+
json.dumps(final_record, indent=2).encode("utf-8"),
|
| 102 |
+
"application/json",
|
| 103 |
+
)
|
| 104 |
+
|
| 105 |
+
return final_record
|