Anna Zhou commited on
Commit
2f024ae
·
1 Parent(s): 3a9eebe

Use OAuth for Drive request uploads

Browse files
README.md CHANGED
@@ -42,10 +42,25 @@ streamlit run app/app.py
42
  ## Hugging Face request intake
43
 
44
  The **New Case Study Requests** tab saves each submitted request to Google Drive,
45
- under folder `10n_-uOCT2GXu_G3r1qi5qJr8hHoy-E_y`. In Hugging Face Spaces, set
46
- `GOOGLE_SERVICE_ACCOUNT_JSON` as a secret containing the service account JSON,
47
- and share the Drive intake folder with that service account email. To use a
48
- different parent folder, set `RFR_REQUESTS_DRIVE_FOLDER_ID`.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
49
 
50
  ## Ingesting documents
51
 
 
42
  ## Hugging Face request intake
43
 
44
  The **New Case Study Requests** tab saves each submitted request to Google Drive,
45
+ under folder `10n_-uOCT2GXu_G3r1qi5qJr8hHoy-E_y`. In Hugging Face Spaces, use
46
+ OAuth credentials for the Google account that should own submitted files, usually
47
+ `rfr.rag.uwaterloo@gmail.com`. Set these secrets:
48
+
49
+ ```text
50
+ GOOGLE_OAUTH_CLIENT_ID
51
+ GOOGLE_OAUTH_CLIENT_SECRET
52
+ GOOGLE_OAUTH_REFRESH_TOKEN
53
+ ```
54
+
55
+ To generate those values, run:
56
+
57
+ ```bash
58
+ python3 -m case_study_pipeline.create_drive_oauth_secrets \
59
+ --credentials-file /path/to/oauth-client.json \
60
+ --space-id UWRFR/UWaterlooRFRChatBot
61
+ ```
62
+
63
+ To use a different parent folder, set `RFR_REQUESTS_DRIVE_FOLDER_ID`.
64
 
65
  ## Ingesting documents
66
 
app/app.py CHANGED
@@ -664,9 +664,10 @@ def _render_new_case_requests_page() -> None:
664
  if "Service Accounts do not have storage quota" in error_text:
665
  st.caption(
666
  "Drive upload error: Google service accounts cannot own new files in a regular "
667
- "My Drive folder. Move the intake folder to a Google Shared Drive, give the "
668
- "service account Content manager or Manager access, and update "
669
- "RFR_REQUESTS_DRIVE_FOLDER_ID to that Shared Drive folder ID."
 
670
  )
671
  else:
672
  st.caption(f"Drive upload error: {exc}")
 
664
  if "Service Accounts do not have storage quota" in error_text:
665
  st.caption(
666
  "Drive upload error: Google service accounts cannot own new files in a regular "
667
+ "My Drive folder. Add OAuth secrets for the shared Gmail account "
668
+ "GOOGLE_OAUTH_CLIENT_ID, GOOGLE_OAUTH_CLIENT_SECRET, and "
669
+ "GOOGLE_OAUTH_REFRESH_TOKEN, then restart the Space. Alternatively, move the "
670
+ "intake folder to a Google Shared Drive."
671
  )
672
  else:
673
  st.caption(f"Drive upload error: {exc}")
app/drive_request_uploader.py CHANGED
@@ -1,7 +1,9 @@
1
  """Google Drive intake uploads for new case study requests.
2
 
3
- Uses the same GOOGLE_SERVICE_ACCOUNT_JSON secret pattern as sheets_logger.py.
4
- The target Drive folder must be shared with the service account email.
 
 
5
  """
6
 
7
  from __future__ import annotations
@@ -14,10 +16,22 @@ from typing import Any
14
 
15
 
16
  DEFAULT_REQUESTS_DRIVE_FOLDER_ID = "10n_-uOCT2GXu_G3r1qi5qJr8hHoy-E_y"
 
 
 
 
 
 
 
 
 
 
17
 
18
 
19
  def _drive_service():
20
  try:
 
 
21
  from google.oauth2.service_account import Credentials
22
  from googleapiclient.discovery import build
23
  except ImportError as exc:
@@ -25,15 +39,29 @@ def _drive_service():
25
  "Google Drive libraries are not installed. Check requirements.txt."
26
  ) from exc
27
 
 
 
 
 
 
 
 
 
 
 
 
 
28
  creds_json = os.getenv("GOOGLE_SERVICE_ACCOUNT_JSON")
29
  if not creds_json:
30
  raise RuntimeError(
31
- "GOOGLE_SERVICE_ACCOUNT_JSON is not set. Add it as a Hugging Face Space secret."
 
 
 
32
  )
33
 
34
  creds_dict = json.loads(creds_json)
35
- scopes = ["https://www.googleapis.com/auth/drive"]
36
- creds = Credentials.from_service_account_info(creds_dict, scopes=scopes)
37
  return build("drive", "v3", credentials=creds)
38
 
39
 
 
1
  """Google Drive intake uploads for new case study requests.
2
 
3
+ Preferred hosted auth uses OAuth user credentials for the shared
4
+ ``rfr.rag.uwaterloo@gmail.com`` account. This avoids service-account Drive
5
+ storage quota limits in regular My Drive folders. If OAuth secrets are absent,
6
+ the uploader falls back to the older service-account secret.
7
  """
8
 
9
  from __future__ import annotations
 
16
 
17
 
18
  DEFAULT_REQUESTS_DRIVE_FOLDER_ID = "10n_-uOCT2GXu_G3r1qi5qJr8hHoy-E_y"
19
+ DRIVE_SCOPES = ["https://www.googleapis.com/auth/drive"]
20
+
21
+
22
+ def _oauth_env_is_configured() -> bool:
23
+ required = (
24
+ "GOOGLE_OAUTH_CLIENT_ID",
25
+ "GOOGLE_OAUTH_CLIENT_SECRET",
26
+ "GOOGLE_OAUTH_REFRESH_TOKEN",
27
+ )
28
+ return all(os.getenv(key) for key in required)
29
 
30
 
31
  def _drive_service():
32
  try:
33
+ from google.auth.transport.requests import Request
34
+ from google.oauth2.credentials import Credentials as OAuthCredentials
35
  from google.oauth2.service_account import Credentials
36
  from googleapiclient.discovery import build
37
  except ImportError as exc:
 
39
  "Google Drive libraries are not installed. Check requirements.txt."
40
  ) from exc
41
 
42
+ if _oauth_env_is_configured():
43
+ creds = OAuthCredentials(
44
+ token=None,
45
+ refresh_token=os.environ["GOOGLE_OAUTH_REFRESH_TOKEN"],
46
+ token_uri=os.getenv("GOOGLE_OAUTH_TOKEN_URI", "https://oauth2.googleapis.com/token"),
47
+ client_id=os.environ["GOOGLE_OAUTH_CLIENT_ID"],
48
+ client_secret=os.environ["GOOGLE_OAUTH_CLIENT_SECRET"],
49
+ scopes=DRIVE_SCOPES,
50
+ )
51
+ creds.refresh(Request())
52
+ return build("drive", "v3", credentials=creds)
53
+
54
  creds_json = os.getenv("GOOGLE_SERVICE_ACCOUNT_JSON")
55
  if not creds_json:
56
  raise RuntimeError(
57
+ "Google Drive credentials are not set. Add OAuth secrets "
58
+ "GOOGLE_OAUTH_CLIENT_ID, GOOGLE_OAUTH_CLIENT_SECRET, and "
59
+ "GOOGLE_OAUTH_REFRESH_TOKEN, or add GOOGLE_SERVICE_ACCOUNT_JSON "
60
+ "as a Hugging Face Space secret."
61
  )
62
 
63
  creds_dict = json.loads(creds_json)
64
+ creds = Credentials.from_service_account_info(creds_dict, scopes=DRIVE_SCOPES)
 
65
  return build("drive", "v3", credentials=creds)
66
 
67
 
case_study_pipeline/create_drive_oauth_secrets.py ADDED
@@ -0,0 +1,89 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Create Hugging Face OAuth secrets for Drive request uploads.
2
+
3
+ Run this locally once with a Google OAuth Desktop client credentials file, then
4
+ sign in as the Google account that should own uploaded request folders.
5
+
6
+ Example:
7
+ python3 -m case_study_pipeline.create_drive_oauth_secrets \
8
+ --credentials-file /path/to/oauth-client.json \
9
+ --space-id UWRFR/UWaterlooRFRChatBot
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ import argparse
15
+ import json
16
+ import shlex
17
+ from pathlib import Path
18
+
19
+ SCOPES = ["https://www.googleapis.com/auth/drive"]
20
+
21
+
22
+ def _load_client_config(credentials_file: Path) -> tuple[str, str]:
23
+ data = json.loads(credentials_file.read_text())
24
+ config = data.get("installed") or data.get("web")
25
+ if not config:
26
+ raise ValueError(
27
+ "OAuth credentials file must contain an 'installed' or 'web' client config."
28
+ )
29
+ return config["client_id"], config["client_secret"]
30
+
31
+
32
+ def main() -> None:
33
+ parser = argparse.ArgumentParser(
34
+ description="Generate Hugging Face Space secrets for Google Drive OAuth uploads."
35
+ )
36
+ parser.add_argument(
37
+ "--credentials-file",
38
+ required=True,
39
+ type=Path,
40
+ help="Google OAuth client JSON file, usually a Desktop app credential.",
41
+ )
42
+ parser.add_argument(
43
+ "--space-id",
44
+ default="UWRFR/UWaterlooRFRChatBot",
45
+ help="Hugging Face Space ID to show in the generated commands.",
46
+ )
47
+ args = parser.parse_args()
48
+
49
+ try:
50
+ from google_auth_oauthlib.flow import InstalledAppFlow
51
+ except ImportError as exc:
52
+ raise SystemExit(
53
+ "Google OAuth libraries are not installed. Run: "
54
+ "pip install google-auth-oauthlib google-api-python-client google-auth-httplib2"
55
+ ) from exc
56
+
57
+ credentials_file = args.credentials_file.expanduser().resolve()
58
+ client_id, client_secret = _load_client_config(credentials_file)
59
+
60
+ flow = InstalledAppFlow.from_client_secrets_file(str(credentials_file), SCOPES)
61
+ creds = flow.run_local_server(port=0, prompt="consent")
62
+ if not creds.refresh_token:
63
+ raise SystemExit(
64
+ "Google did not return a refresh token. In Google Account permissions, "
65
+ "remove this app's previous access, then rerun with prompt=consent."
66
+ )
67
+
68
+ print("\nAdd these secrets to Hugging Face:")
69
+ print(
70
+ "hf spaces secrets add "
71
+ f"{shlex.quote(args.space_id)} "
72
+ f"-s GOOGLE_OAUTH_CLIENT_ID={shlex.quote(client_id)}"
73
+ )
74
+ print(
75
+ "hf spaces secrets add "
76
+ f"{shlex.quote(args.space_id)} "
77
+ f"-s GOOGLE_OAUTH_CLIENT_SECRET={shlex.quote(client_secret)}"
78
+ )
79
+ print(
80
+ "hf spaces secrets add "
81
+ f"{shlex.quote(args.space_id)} "
82
+ f"-s GOOGLE_OAUTH_REFRESH_TOKEN={shlex.quote(creds.refresh_token)}"
83
+ )
84
+ print("\nThen restart the Space:")
85
+ print(f"hf spaces restart {shlex.quote(args.space_id)}")
86
+
87
+
88
+ if __name__ == "__main__":
89
+ main()
docs/02_hugging_face_and_api_keys.md CHANGED
@@ -72,7 +72,9 @@ OPENAI_MODEL
72
  For Google Drive request uploads:
73
 
74
  ```text
75
- GOOGLE_SERVICE_ACCOUNT_JSON
 
 
76
  ```
77
 
78
  Optional override for the new case study request intake folder:
@@ -111,15 +113,44 @@ Example:
111
  gpt-4o-mini
112
  ```
113
 
114
- ### `GOOGLE_SERVICE_ACCOUNT_JSON`
115
 
116
  Used by the website to upload New Case Study Requests into Google Drive.
117
 
118
- This should contain the full JSON service account credential, not just the
119
- service account email.
 
 
 
 
 
 
120
 
121
- The Drive intake folder must be shared with the service account email. If it is
122
- not shared, uploads will fail even if the secret is present.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
123
 
124
  ## How to Add or Replace a Secret
125
 
@@ -140,6 +171,7 @@ Open Space logs and look for messages like:
140
  ANTHROPIC_API_KEY is not set
141
  OPENAI_API_KEY is not set
142
  GOOGLE_SERVICE_ACCOUNT_JSON is not set
 
143
  credit balance is too low
144
  invalid_api_key
145
  permission denied
@@ -147,9 +179,11 @@ permission denied
147
 
148
  For Drive upload problems, check:
149
 
150
- - The `GOOGLE_SERVICE_ACCOUNT_JSON` secret exists.
151
- - It is valid JSON.
152
- - The Drive intake folder is shared with the service account email.
 
 
153
  - The Space was restarted after the secret was added.
154
 
155
  ## Do Not Put API Keys in GitHub
@@ -165,4 +199,3 @@ Never commit API keys to:
165
  Use Hugging Face secrets for the live website.
166
 
167
  Use a local `.env` file for your own computer. Do not commit `.env`.
168
-
 
72
  For Google Drive request uploads:
73
 
74
  ```text
75
+ GOOGLE_OAUTH_CLIENT_ID
76
+ GOOGLE_OAUTH_CLIENT_SECRET
77
+ GOOGLE_OAUTH_REFRESH_TOKEN
78
  ```
79
 
80
  Optional override for the new case study request intake folder:
 
113
  gpt-4o-mini
114
  ```
115
 
116
+ ### Google Drive Request Upload Secrets
117
 
118
  Used by the website to upload New Case Study Requests into Google Drive.
119
 
120
+ Preferred setup uses OAuth for the shared Google account that should own the
121
+ uploaded folders and PDFs, usually:
122
+
123
+ ```text
124
+ rfr.rag.uwaterloo@gmail.com
125
+ ```
126
+
127
+ Set these Hugging Face secrets:
128
 
129
+ ```text
130
+ GOOGLE_OAUTH_CLIENT_ID
131
+ GOOGLE_OAUTH_CLIENT_SECRET
132
+ GOOGLE_OAUTH_REFRESH_TOKEN
133
+ ```
134
+
135
+ Generate the secret values locally:
136
+
137
+ ```bash
138
+ python3 -m case_study_pipeline.create_drive_oauth_secrets \
139
+ --credentials-file /path/to/oauth-client.json \
140
+ --space-id UWRFR/UWaterlooRFRChatBot
141
+ ```
142
+
143
+ When the browser opens, sign in as `rfr.rag.uwaterloo@gmail.com`. The script
144
+ prints the `hf spaces secrets add ...` commands to run.
145
+
146
+ The older fallback secret is:
147
+
148
+ ```text
149
+ GOOGLE_SERVICE_ACCOUNT_JSON
150
+ ```
151
+
152
+ That service-account option works best with a Google Shared Drive. It can fail
153
+ in normal My Drive folders because service accounts do not have storage quota.
154
 
155
  ## How to Add or Replace a Secret
156
 
 
171
  ANTHROPIC_API_KEY is not set
172
  OPENAI_API_KEY is not set
173
  GOOGLE_SERVICE_ACCOUNT_JSON is not set
174
+ GOOGLE_OAUTH_REFRESH_TOKEN is not set
175
  credit balance is too low
176
  invalid_api_key
177
  permission denied
 
179
 
180
  For Drive upload problems, check:
181
 
182
+ - The three OAuth secrets exist: `GOOGLE_OAUTH_CLIENT_ID`,
183
+ `GOOGLE_OAUTH_CLIENT_SECRET`, and `GOOGLE_OAUTH_REFRESH_TOKEN`.
184
+ - The OAuth token was created by signing in as the account that owns or can edit
185
+ the intake folder.
186
+ - The intake folder ID is set correctly with `RFR_REQUESTS_DRIVE_FOLDER_ID`.
187
  - The Space was restarted after the secret was added.
188
 
189
  ## Do Not Put API Keys in GitHub
 
199
  Use Hugging Face secrets for the live website.
200
 
201
  Use a local `.env` file for your own computer. Do not commit `.env`.
 
docs/05_debugging_common_problems.md CHANGED
@@ -53,14 +53,17 @@ If Anthropic credits are unavailable, switch the app to OpenAI only if
53
  The request tab uploads to Google Drive using:
54
 
55
  ```text
56
- GOOGLE_SERVICE_ACCOUNT_JSON
 
 
57
  ```
58
 
59
  Check:
60
 
61
- - The secret exists in Hugging Face.
62
- - The secret value is the full service account JSON.
63
- - The intake folder is shared with the service account email.
 
64
  - The intake folder ID is correct:
65
 
66
  ```text
@@ -73,6 +76,10 @@ If using a different folder, set:
73
  RFR_REQUESTS_DRIVE_FOLDER_ID
74
  ```
75
 
 
 
 
 
76
  ## New Case Does Not Appear in Case Studies
77
 
78
  Usually one of these files was not rebuilt or committed:
@@ -218,4 +225,3 @@ Check:
218
  - Your browser completed Google login.
219
  - The target case folder exists in `case_study_pipeline/drive_folders.json`.
220
  - The Google account has permission to write to the Drive folder.
221
-
 
53
  The request tab uploads to Google Drive using:
54
 
55
  ```text
56
+ GOOGLE_OAUTH_CLIENT_ID
57
+ GOOGLE_OAUTH_CLIENT_SECRET
58
+ GOOGLE_OAUTH_REFRESH_TOKEN
59
  ```
60
 
61
  Check:
62
 
63
+ - The three OAuth secrets exist in Hugging Face.
64
+ - The refresh token was generated by signing in as the Google account that
65
+ should own uploaded request folders, usually `rfr.rag.uwaterloo@gmail.com`.
66
+ - The intake folder is owned by or shared with that Google account.
67
  - The intake folder ID is correct:
68
 
69
  ```text
 
76
  RFR_REQUESTS_DRIVE_FOLDER_ID
77
  ```
78
 
79
+ If the error says service accounts do not have storage quota, the app is falling
80
+ back to `GOOGLE_SERVICE_ACCOUNT_JSON`. Add the OAuth secrets above, or move the
81
+ intake folder to a Google Shared Drive.
82
+
83
  ## New Case Does Not Appear in Case Studies
84
 
85
  Usually one of these files was not rebuilt or committed:
 
225
  - Your browser completed Google login.
226
  - The target case folder exists in `case_study_pipeline/drive_folders.json`.
227
  - The Google account has permission to write to the Drive folder.
 
docs/website_maintenance.md CHANGED
@@ -126,10 +126,12 @@ OPENAI_API_KEY
126
  OPENAI_MODEL
127
  ```
128
 
129
- Required for Google Sheets logging and Google Drive case-study request uploads:
130
 
131
  ```text
132
- GOOGLE_SERVICE_ACCOUNT_JSON
 
 
133
  ```
134
 
135
  Optional Drive override for new case study requests:
@@ -144,8 +146,20 @@ By default, new case study requests are uploaded to this Drive folder:
144
  10n_-uOCT2GXu_G3r1qi5qJr8hHoy-E_y
145
  ```
146
 
147
- The Drive folder must be shared with the service account email inside
148
- `GOOGLE_SERVICE_ACCOUNT_JSON`. Otherwise uploads from the website will fail.
 
 
 
 
 
 
 
 
 
 
 
 
149
 
150
  ## How New Case Study Requests Work
151
 
@@ -351,9 +365,11 @@ Confirm it matches the commit you pushed.
351
 
352
  Check:
353
 
354
- - `GOOGLE_SERVICE_ACCOUNT_JSON` exists as a Hugging Face secret
355
- - the Drive folder is shared with the service account email
356
- - the Space has restarted after the secret was added
 
 
357
 
358
  ### LLM key fails
359
 
 
126
  OPENAI_MODEL
127
  ```
128
 
129
+ Required for Google Drive case-study request uploads:
130
 
131
  ```text
132
+ GOOGLE_OAUTH_CLIENT_ID
133
+ GOOGLE_OAUTH_CLIENT_SECRET
134
+ GOOGLE_OAUTH_REFRESH_TOKEN
135
  ```
136
 
137
  Optional Drive override for new case study requests:
 
146
  10n_-uOCT2GXu_G3r1qi5qJr8hHoy-E_y
147
  ```
148
 
149
+ The OAuth token should be generated by signing in as the Google account that
150
+ should own request uploads, usually `rfr.rag.uwaterloo@gmail.com`.
151
+
152
+ Generate the Hugging Face OAuth secrets locally:
153
+
154
+ ```bash
155
+ python3 -m case_study_pipeline.create_drive_oauth_secrets \
156
+ --credentials-file /path/to/oauth-client.json \
157
+ --space-id UWRFR/UWaterlooRFRChatBot
158
+ ```
159
+
160
+ The older fallback `GOOGLE_SERVICE_ACCOUNT_JSON` service-account setup can fail
161
+ in regular My Drive folders because service accounts do not have storage quota.
162
+ Use OAuth or a Google Shared Drive for request uploads.
163
 
164
  ## How New Case Study Requests Work
165
 
 
365
 
366
  Check:
367
 
368
+ - `GOOGLE_OAUTH_CLIENT_ID`, `GOOGLE_OAUTH_CLIENT_SECRET`, and
369
+ `GOOGLE_OAUTH_REFRESH_TOKEN` exist as Hugging Face secrets
370
+ - the Drive folder is owned by or shared with the OAuth Google account
371
+ - `RFR_REQUESTS_DRIVE_FOLDER_ID` points to the correct folder
372
+ - the Space has restarted after secrets or variables were added
373
 
374
  ### LLM key fails
375