Thanmay Mohandas Das commited on
Commit
34d2be5
·
unverified ·
2 Parent(s): e9940ca2407d83

Merge branch 'main' into artifact/quiz

Browse files
README_PYTHON.md ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Python Version
2
+
3
+ **Gradio does not work reliably with Python 3.13.** Use Python 3.10, 3.11, or 3.12.
4
+
5
+ ## Quick run
6
+
7
+ ```powershell
8
+ .\run.bat
9
+ ```
10
+
11
+ Or manually:
12
+
13
+ ```powershell
14
+ py -3.10 -m pip install -r requirements.txt
15
+ py -3.10 app.py
16
+ ```
17
+
18
+ ## Install Python 3.10
19
+
20
+ If you don't have Python 3.10:
21
+
22
+ 1. Download from https://www.python.org/downloads/release/python-31011/
23
+ 2. Run installer, check "Add Python to PATH"
24
+ 3. Restart terminal, then run `.\run.bat`
app.py CHANGED
@@ -1,5 +1,16 @@
1
  from pathlib import Path
2
  import shutil
 
 
 
 
 
 
 
 
 
 
 
3
 
4
  from dotenv import load_dotenv
5
 
@@ -7,14 +18,22 @@ from dotenv import load_dotenv
7
  load_dotenv(Path(__file__).resolve().parent.parent / ".env")
8
  load_dotenv(Path(__file__).resolve().parent / ".env")
9
 
 
10
  from datetime import datetime
11
  import gradio as gr
 
12
  import gradio_client.utils as gradio_client_utils
13
 
 
14
  from backend.ingestion_service import ingest_pdf_chunks, ingest_url_chunks, remove_chunks_for_source
15
  from backend.notebook_service import create_notebook, list_notebooks, rename_notebook, delete_notebook
 
 
 
 
16
 
17
  import hashlib
 
18
 
19
  _original_gradio_get_type = gradio_client_utils.get_type
20
  _original_json_schema_to_python_type = gradio_client_utils._json_schema_to_python_type
@@ -35,43 +54,76 @@ def _patched_json_schema_to_python_type(schema, defs=None):
35
  gradio_client_utils.get_type = _patched_gradio_get_type
36
  gradio_client_utils._json_schema_to_python_type = _patched_json_schema_to_python_type
37
 
38
- # Theme: adapts to light/dark mode
39
  theme = gr.themes.Soft(
40
  primary_hue="blue",
41
  secondary_hue="slate",
42
- font=gr.themes.GoogleFont("Inter"),
43
  )
44
 
45
  CUSTOM_CSS = """
46
- .container { max-width: 720px; margin: 0 auto; padding: 0 24px; }
47
- .login-center { display: flex; flex-direction: column; align-items: center; justify-content: center; gap: 12px; padding: 24px 0; }
48
- .login-center .login-btn-wrap { display: flex; justify-content: center; width: 100%; }
49
- .login-center .login-btn-wrap button { display: inline-flex; align-items: center; gap: 8px; }
50
- .hero { font-size: 1.5rem; font-weight: 600; color: #1e293b; margin-bottom: 8px; }
51
- .sub { font-size: 0.875rem; color: #64748b; margin-bottom: 24px; }
52
- .nb-row { display: flex; align-items: center; gap: 12px; padding: 10px 0; border-bottom: 1px solid #e2e8f0; }
53
- .nb-row:last-child { border-bottom: none; }
54
- .gr-button { min-height: 36px !important; padding: 0 16px !important; font-weight: 500 !important; border-radius: 8px !important; }
55
- .gr-input { min-height: 40px !important; border-radius: 8px !important; }
56
- .status { font-size: 0.875rem; color: #64748b; margin-top: 16px; padding: 12px 16px; background: #f8fafc; border-radius: 8px; }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
57
  @media (prefers-color-scheme: dark) {
58
- .hero { color: #f1f5f9 !important; }
59
- .sub { color: #94a3b8 !important; }
60
- .nb-row { border-color: #334155 !important; }
61
- .status { color: #94a3b8 !important; background: #1e293b !important; }
 
 
 
62
  }
63
- .dark .hero { color: #f1f5f9 !important; }
64
- .dark .sub { color: #94a3b8 !important; }
65
- .dark .nb-row { border-color: #334155 !important; }
66
- .dark .status { color: #94a3b8 !important; background: #1e293b !important; }
 
 
 
67
  """
68
 
69
- MAX_NOTEBOOKS = 20
70
-
71
-
72
  def _user_id(profile: gr.OAuthProfile | None) -> str | None:
73
  """Extract user_id from HF OAuth profile. None if not logged in."""
74
- return profile.name if profile else None
 
 
 
 
 
 
 
 
75
 
76
 
77
  def _get_notebooks(user_id: str | None):
@@ -80,86 +132,116 @@ def _get_notebooks(user_id: str | None):
80
  return list_notebooks(user_id)
81
 
82
 
83
- def _safe_create(new_name, state, selected_id, profile: gr.OAuthProfile | None):
84
  """Create notebook with name from text box."""
85
  try:
86
  user_id = _user_id(profile)
87
  if not user_id:
88
- return gr.skip(), gr.skip(), gr.skip(), "Please sign in with Hugging Face", *([gr.skip()] * (MAX_NOTEBOOKS * 2))
89
  name = (new_name or "").strip() or "Untitled Notebook"
90
  nb = create_notebook(user_id, name)
91
  if nb:
92
  notebooks = _get_notebooks(user_id)
93
- state = [(n["notebook_id"], n["name"]) for n in notebooks]
94
- updates = _build_row_updates(notebooks)
95
- new_selected = nb["notebook_id"]
96
  status = f"Created: {nb['name']}"
97
- return "", state, new_selected, status, *updates
98
- return gr.skip(), gr.skip(), gr.skip(), "Failed to create", *([gr.skip()] * (MAX_NOTEBOOKS * 2))
99
  except Exception as e:
100
- return gr.skip(), gr.skip(), gr.skip(), f"Error: {e}", *([gr.skip()] * (MAX_NOTEBOOKS * 2))
101
 
102
 
103
- def _safe_rename(idx, new_name, state, selected_id, profile: gr.OAuthProfile | None):
104
  """Rename notebook at index."""
105
  try:
106
  if idx is None or idx < 0 or idx >= len(state):
107
- return gr.skip(), gr.skip(), gr.skip(), *([gr.skip()] * (MAX_NOTEBOOKS * 2))
108
  nb_id, _ = state[idx]
109
  name = (new_name or "").strip()
110
  if not name:
111
- return gr.skip(), gr.skip(), gr.skip(), "Enter a name.", *([gr.skip()] * (MAX_NOTEBOOKS * 2))
112
  user_id = _user_id(profile)
113
  if not user_id:
114
- return gr.skip(), gr.skip(), gr.skip(), "Please sign in", *([gr.skip()] * (MAX_NOTEBOOKS * 2))
115
  ok = rename_notebook(user_id, nb_id, name)
116
  if ok:
117
  notebooks = _get_notebooks(user_id)
118
- state = [(n["notebook_id"], n["name"]) for n in notebooks]
119
- updates = _build_row_updates(notebooks)
120
- return state, selected_id, f"Renamed to: {name}", *updates
121
- return gr.skip(), gr.skip(), gr.skip(), "Failed to rename", *([gr.skip()] * (MAX_NOTEBOOKS * 2))
122
  except Exception as e:
123
- return gr.skip(), gr.skip(), gr.skip(), f"Error: {e}", *([gr.skip()] * (MAX_NOTEBOOKS * 2))
124
 
125
 
126
- def _safe_delete(idx, state, selected_id, profile: gr.OAuthProfile | None):
127
  """Delete notebook at index."""
128
  try:
129
  if idx is None or idx < 0 or idx >= len(state):
130
- return gr.skip(), gr.skip(), gr.skip(), *([gr.skip()] * (MAX_NOTEBOOKS * 2))
131
  nb_id, _ = state[idx]
132
  user_id = _user_id(profile)
133
  if not user_id:
134
- return gr.skip(), gr.skip(), gr.skip(), "Please sign in", *([gr.skip()] * (MAX_NOTEBOOKS * 2))
135
  ok = delete_notebook(user_id, nb_id)
136
  if ok:
137
  notebooks = _get_notebooks(user_id)
138
- state = [(n["notebook_id"], n["name"]) for n in notebooks]
139
- updates = _build_row_updates(notebooks)
140
  new_selected = notebooks[0]["notebook_id"] if notebooks else None
141
- return state, new_selected, "Notebook deleted", *updates
142
- return gr.skip(), gr.skip(), gr.skip(), "Failed to delete", *([gr.skip()] * (MAX_NOTEBOOKS * 2))
143
  except Exception as e:
144
- return gr.skip(), gr.skip(), gr.skip(), f"Error: {e}", *([gr.skip()] * (MAX_NOTEBOOKS * 2))
145
-
146
 
147
- def _select_notebook(idx, state):
148
- """Set selected notebook when user interacts with a row."""
149
- if idx is None or idx < 0 or idx >= len(state):
150
- return gr.skip()
151
- return state[idx][0]
152
 
153
-
154
- def _initial_load(profile: gr.OAuthProfile | None):
155
  """Load notebooks on app load. Uses HF OAuth profile for user_id."""
156
  user_id = _user_id(profile)
157
  notebooks = _get_notebooks(user_id)
158
  state = [(n["notebook_id"], n["name"]) for n in notebooks]
159
  selected = notebooks[0]["notebook_id"] if notebooks else None
160
- updates = _build_row_updates(notebooks)
161
  status = f"Signed in as {user_id}" if user_id else "Sign in with Hugging Face to manage notebooks."
162
- return state, selected, status, *updates
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
163
 
164
 
165
  def _safe_upload_pdfs(files, selected_id, profile: gr.OAuthProfile | None):
@@ -217,7 +299,7 @@ def _safe_upload_pdfs(files, selected_id, profile: gr.OAuthProfile | None):
217
  return f"Error uploading PDFs: {error}"
218
 
219
 
220
- def _list_uploaded_pdfs(selected_id, profile: gr.OAuthProfile | None):
221
  """List uploaded PDFs for the selected notebook."""
222
  user_id = _user_id(profile)
223
  if not user_id or not selected_id:
@@ -232,7 +314,7 @@ def _list_uploaded_pdfs(selected_id, profile: gr.OAuthProfile | None):
232
  return gr.update(choices=pdf_names, value=selected_name)
233
 
234
 
235
- def _safe_remove_pdf(file_name, selected_id, profile: gr.OAuthProfile | None):
236
  """Remove one uploaded PDF from the selected notebook."""
237
  try:
238
  user_id = _user_id(profile)
@@ -260,7 +342,7 @@ def _url_source_id(url: str) -> str:
260
  return f"url_{h}"
261
 
262
 
263
- def _safe_ingest_url(url, selected_id, profile: gr.OAuthProfile | None):
264
  """Ingest one URL into chunks table for the selected notebook."""
265
  try:
266
  user_id = _user_id(profile)
@@ -288,7 +370,7 @@ def _safe_ingest_url(url, selected_id, profile: gr.OAuthProfile | None):
288
  except Exception as error:
289
  return "", f"Error ingesting URL: {error}"
290
 
291
- def _safe_remove_url(url, selected_id, profile: gr.OAuthProfile | None):
292
  try:
293
  user_id = _user_id(profile)
294
  if not user_id:
@@ -310,17 +392,7 @@ def _safe_remove_url(url, selected_id, profile: gr.OAuthProfile | None):
310
 
311
 
312
 
313
- def _build_row_updates(notebooks):
314
- """Return gr.update values for each row: visibility, then text value."""
315
- out = []
316
- for i in range(MAX_NOTEBOOKS):
317
- visible = i < len(notebooks)
318
- name = notebooks[i]["name"] if visible else ""
319
- out.append(gr.update(visible=visible))
320
- out.append(gr.update(value=name, visible=visible))
321
- return out
322
-
323
- #Upload Handler Functions
324
  def _do_upload(text_content, title, notebook_id, profile: gr.OAuthProfile | None):
325
  """Handle direct text input and ingestion."""
326
  from backend.ingestion_txt import ingest_txt
@@ -380,6 +452,41 @@ def _load_sources(notebook_id, profile: gr.OAuthProfile | None):
380
  sources = list_sources(notebook_id)
381
  return _format_sources(sources)
382
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
383
  # Quiz Handlers
384
  def _get_notebook_pdfs(notebook_id, profile: gr.OAuthProfile | None):
385
  user_id = _user_id(profile)
@@ -461,6 +568,46 @@ def _submit_quiz(questions, *answers):
461
 
462
  lines.append(f"\n**Score: {score}/{len(questions)}**")
463
  return "\n\n".join(lines)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
464
 
465
  def _get_quiz_pdfs(source_type, notebook_id):
466
  if source_type != "PDF":
@@ -521,82 +668,186 @@ with gr.Blocks(
521
  theme=theme,
522
  css=CUSTOM_CSS,
523
  ) as demo:
524
- gr.HTML('<div class="container"><p class="hero">Notebook Manager</p><p class="sub">Create notebook below, then manage with Rename and Delete</p></div>')
525
-
526
- with gr.Row(elem_classes=["login-center"]):
527
- gr.Markdown("**Sign in with Hugging Face to access your notebooks**")
528
- with gr.Row(elem_classes=["login-btn-wrap"]):
529
- login_btn = gr.LoginButton(value="🤗 Login with Hugging Face", size="lg")
530
-
531
- nb_state = gr.State([])
532
- selected_notebook_id = gr.State(None)
533
-
534
- # Create section: text box + Create button
535
- with gr.Row():
536
- create_txt = gr.Textbox(
537
- label="Create notebook",
538
- placeholder="Enter new notebook name",
539
- value="",
540
- scale=3,
541
- )
542
- create_btn = gr.Button("Create", variant="primary", scale=1)
543
-
544
- with gr.Row():
545
- pdf_upload_btn = gr.UploadButton(
546
- "Upload PDFs",
547
- file_types=[".pdf"],
548
- file_count="multiple",
549
- type="filepath",
550
- variant="secondary",
551
- )
552
-
553
- with gr.Row():
554
- uploaded_pdf_dd = gr.Dropdown(
555
- label="Uploaded PDFs",
556
- choices=[],
557
- value=None,
558
- scale=3,
559
- allow_custom_value=False,
560
- )
561
- remove_pdf_btn = gr.Button("Remove selected PDF", variant="stop", scale=1)
562
-
563
- with gr.Row():
564
- url_txt = gr.Textbox(
565
- label="Ingest web URL",
566
- placeholder="https://example.com",
567
- value="",
568
- scale=3,
569
- )
570
- ingest_url_btn = gr.Button("Ingest URL", variant="primary", scale=1)
571
- remove_url_btn = gr.Button("Delete URL", variant="stop", scale=1)
572
-
573
- gr.Markdown("---")
574
- gr.Markdown("**Your notebooks** (selected notebook used for chat/ingestion)")
575
-
576
- # Rows: each notebook has [name] [Rename] [Delete]
577
- row_components = []
578
- row_outputs = []
579
- for i in range(MAX_NOTEBOOKS):
580
- with gr.Row(visible=False) as row:
581
- name_txt = gr.Textbox(
582
- value="",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
583
  show_label=False,
584
- scale=3,
585
- min_width=200,
586
  )
587
- rename_btn = gr.Button("Rename", scale=1, min_width=80)
588
- delete_btn = gr.Button("Delete", variant="stop", scale=1, min_width=80)
589
- select_btn = gr.Button("Select", scale=1, min_width=70)
590
- row_components.append({"row": row, "name": name_txt, "rename": rename_btn, "delete": delete_btn, "select": select_btn})
591
- row_outputs.extend([row, name_txt])
592
 
593
- status = gr.Markdown("Sign in with Hugging Face to manage notebooks.", elem_classes=["status"])
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
594
 
595
- # Create button
596
  create_btn.click(
597
  _safe_create,
598
  inputs=[create_txt, nb_state, selected_notebook_id],
599
- outputs=[create_txt, nb_state, selected_notebook_id, status] + row_outputs,
600
  api_name=False,
601
  ).then(_list_uploaded_pdfs, inputs=[selected_notebook_id], outputs=[uploaded_pdf_dd])
602
 
@@ -628,26 +879,7 @@ with gr.Blocks(
628
  api_name=False,
629
  ).then(_list_uploaded_pdfs, inputs=[selected_notebook_id], outputs=[uploaded_pdf_dd])
630
 
631
- # Per-row: Rename, Delete, Select (profile injected by Gradio for OAuth)
632
- for i in range(MAX_NOTEBOOKS):
633
- rename_btn = row_components[i]["rename"]
634
- delete_btn = row_components[i]["delete"]
635
- select_btn = row_components[i]["select"]
636
- name_txt = row_components[i]["name"]
637
 
638
- rename_btn.click(
639
- _safe_rename,
640
- inputs=[gr.State(i), name_txt, nb_state, selected_notebook_id],
641
- outputs=[nb_state, selected_notebook_id, status] + row_outputs,
642
- api_name=False,
643
- )
644
- delete_btn.click(
645
- _safe_delete,
646
- inputs=[gr.State(i), nb_state, selected_notebook_id],
647
- outputs=[nb_state, selected_notebook_id, status] + row_outputs,
648
- api_name=False,
649
- ).then(_list_uploaded_pdfs, inputs=[selected_notebook_id], outputs=[uploaded_pdf_dd])
650
-
651
  # Text Input Section
652
  gr.Markdown("---")
653
  gr.Markdown("## Add Text")
@@ -670,6 +902,17 @@ with gr.Blocks(
670
 
671
  upload_status = gr.Markdown("", elem_classes=["status"])
672
 
 
 
 
 
 
 
 
 
 
 
 
673
  # Quiz Section
674
  gr.Markdown("---")
675
  gr.Markdown("## Generate Quiz")
@@ -746,6 +989,34 @@ with gr.Blocks(
746
  outputs=[upload_status],
747
  )
748
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
749
  quiz_source_type.change(
750
  _quiz_pdf_dropdown_update,
751
  inputs=[quiz_source_type, selected_notebook_id],
@@ -791,7 +1062,17 @@ with gr.Blocks(
791
  api_name=False,
792
  )
793
 
794
- demo.launch()
795
-
796
-
 
 
 
 
 
 
 
797
 
 
 
 
 
1
  from pathlib import Path
2
  import shutil
3
+ import sys
4
+ import warnings
5
+
6
+ # Flush print immediately
7
+ def _log(msg):
8
+ print(msg, flush=True)
9
+
10
+ _log("1. Loading env...")
11
+ # Suppress noisy dependency warnings
12
+ warnings.filterwarnings("ignore", message=".*urllib3.*")
13
+ warnings.filterwarnings("ignore", message=".*chardet.*")
14
 
15
  from dotenv import load_dotenv
16
 
 
18
  load_dotenv(Path(__file__).resolve().parent.parent / ".env")
19
  load_dotenv(Path(__file__).resolve().parent / ".env")
20
 
21
+ _log("2. Loading Gradio...")
22
  from datetime import datetime
23
  import gradio as gr
24
+ _log("2a. Loading gradio_client...")
25
  import gradio_client.utils as gradio_client_utils
26
 
27
+ _log("3. Loading backend...")
28
  from backend.ingestion_service import ingest_pdf_chunks, ingest_url_chunks, remove_chunks_for_source
29
  from backend.notebook_service import create_notebook, list_notebooks, rename_notebook, delete_notebook
30
+ from backend.podcast_service import generate_podcast, generate_podcast_audio
31
+ from backend.chat_service import load_chat
32
+ from backend.rag_service import rag_chat
33
+ from backend.report_service import generate_report
34
 
35
  import hashlib
36
+ _log("4. Imports done.")
37
 
38
  _original_gradio_get_type = gradio_client_utils.get_type
39
  _original_json_schema_to_python_type = gradio_client_utils._json_schema_to_python_type
 
54
  gradio_client_utils.get_type = _patched_gradio_get_type
55
  gradio_client_utils._json_schema_to_python_type = _patched_json_schema_to_python_type
56
 
57
+ # Theme: adapts to light/dark mode (use default font to avoid network fetch on startup)
58
  theme = gr.themes.Soft(
59
  primary_hue="blue",
60
  secondary_hue="slate",
 
61
  )
62
 
63
  CUSTOM_CSS = """
64
+ .gradio-container { max-width: 1000px !important; margin: 0 auto !important; }
65
+ .container { max-width: 1000px; margin: 0 auto; padding: 0 24px; }
66
+
67
+ .header-bar { padding: 12px 0; border-bottom: 1px solid #e2e8f0; margin-bottom: 24px; display: flex !important; justify-content: space-between !important; align-items: center !important; white-space: nowrap; }
68
+ .login-center { display: flex; justify-content: center; width: 100%; }
69
+ #auth-text { white-space: nowrap; margin: 8px 0 16px 0; font-size: 0.95rem; opacity: 0.9; }
70
+ .gr-button { padding: 14px 28px !important; font-size: 0.9rem !important; border-radius: 12px !important; white-space: nowrap !important; width: auto !important; }
71
+ .gr-button[aria-label*="Logout"] { min-width: auto !important; display: inline-flex !important; align-items: center !important; justify-content: center !important; }
72
+ .header-bar .gr-button { padding-left: 40px !important; padding-right: 40px !important; min-width: 220px !important; font-size: 0.8rem !important; }
73
+ .dark .header-bar { border-bottom: 1px solid #334155; }
74
+
75
+ .hero-section { margin-bottom: 16px; }
76
+ .login-container { padding: 12px 0; }
77
+ .create-strip { padding: 18px; border-radius: 16px; }
78
+ .create-row { display: flex !important; align-items: center !important; gap: 16px !important; }
79
+ .create-label { white-space: nowrap; font-size: 0.95rem; margin: 0; min-width: 180px; }
80
+ .create-row .gr-textbox { flex: 1 !important; }
81
+ .create-row .gr-textbox textarea,
82
+ .create-row .gr-textbox input { border-radius: 10px !important; }
83
+ .create-row .gr-button { border-radius: 10px !important; padding: 10px 20px !important; }
84
+ .hero-title { font-size: 2rem; font-weight: 700; color: #1e293b; margin: 0 0 8px 0; }
85
+ .hero-sub { font-size: 1rem; color: #64748b; margin: 0; line-height: 1.5; }
86
+
87
+ .section-card { padding: 24px; border-radius: 16px; background: #f8fafc; margin-bottom: 24px; box-shadow: 0 2px 8px rgba(0,0,0,0.06); }
88
+ .notebook-card { padding: 14px 20px; border-radius: 12px; background: #fff; margin-bottom: 8px; border: 1px solid #e2e8f0; display: flex; align-items: center; gap: 12px; transition: background 0.15s ease; }
89
+ .notebook-card:hover { background: #f8fafc; }
90
+
91
+ .section-title { font-size: 1.125rem; font-weight: 600; color: #1e293b; margin: 0 0 16px 0; }
92
+ .section-row { display: flex !important; align-items: center !important; gap: 16px !important; margin-bottom: 12px; }
93
+ .section-row .gr-textbox { flex: 1 !important; }
94
+ .section-row .gr-button { border-radius: 10px !important; padding: 10px 20px !important; }
95
+
96
+ .status { font-size: 0.875rem; color: #64748b; margin-top: 16px; padding: 12px 16px; background: #f1f5f9; border-radius: 12px; }
97
+
98
  @media (prefers-color-scheme: dark) {
99
+ .hero-title { color: #f1f5f9 !important; }
100
+ .hero-sub { color: #94a3b8 !important; }
101
+ .section-card { background: #1e293b !important; box-shadow: 0 2px 8px rgba(0,0,0,0.3); }
102
+ .section-title { color: #f1f5f9 !important; }
103
+ .notebook-card { background: #334155 !important; border-color: #475569; }
104
+ .notebook-card:hover { background: #475569 !important; }
105
+ .status { color: #94a3b8 !important; background: #334155 !important; }
106
  }
107
+ .dark .hero-title { color: #f1f5f9 !important; }
108
+ .dark .hero-sub { color: #94a3b8 !important; }
109
+ .dark .section-card { background: #1e293b !important; }
110
+ .dark .section-title { color: #f1f5f9 !important; }
111
+ .dark .notebook-card { background: #334155 !important; border-color: #475569; }
112
+ .dark .notebook-card:hover { background: #475569 !important; }
113
+ .dark .status { color: #94a3b8 !important; background: #334155 !important; }
114
  """
115
 
 
 
 
116
  def _user_id(profile: gr.OAuthProfile | None) -> str | None:
117
  """Extract user_id from HF OAuth profile. None if not logged in."""
118
+ if not profile:
119
+ return None
120
+ return (
121
+ getattr(profile, "id", None)
122
+ or getattr(profile, "sub", None)
123
+ or getattr(profile, "preferred_username", None)
124
+ or getattr(profile, "username", None)
125
+ or getattr(profile, "name", None)
126
+ )
127
 
128
 
129
  def _get_notebooks(user_id: str | None):
 
132
  return list_notebooks(user_id)
133
 
134
 
135
+ def _safe_create(new_name, state, selected_id, profile: gr.OAuthProfile | None = None):
136
  """Create notebook with name from text box."""
137
  try:
138
  user_id = _user_id(profile)
139
  if not user_id:
140
+ return gr.skip(), gr.skip(), gr.skip(), "Please sign in with Hugging Face"
141
  name = (new_name or "").strip() or "Untitled Notebook"
142
  nb = create_notebook(user_id, name)
143
  if nb:
144
  notebooks = _get_notebooks(user_id)
145
+ new_state = [(n["notebook_id"], n["name"]) for n in notebooks]
 
 
146
  status = f"Created: {nb['name']}"
147
+ return "", new_state, nb["notebook_id"], status
148
+ return gr.skip(), gr.skip(), gr.skip(), "Failed to create"
149
  except Exception as e:
150
+ return gr.skip(), gr.skip(), gr.skip(), f"Error: {e}"
151
 
152
 
153
+ def _safe_rename(idx, new_name, state, selected_id, profile: gr.OAuthProfile | None = None):
154
  """Rename notebook at index."""
155
  try:
156
  if idx is None or idx < 0 or idx >= len(state):
157
+ return gr.skip(), gr.skip(), "Invalid selection"
158
  nb_id, _ = state[idx]
159
  name = (new_name or "").strip()
160
  if not name:
161
+ return gr.skip(), gr.skip(), "Enter a name."
162
  user_id = _user_id(profile)
163
  if not user_id:
164
+ return gr.skip(), gr.skip(), "Please sign in"
165
  ok = rename_notebook(user_id, nb_id, name)
166
  if ok:
167
  notebooks = _get_notebooks(user_id)
168
+ new_state = [(n["notebook_id"], n["name"]) for n in notebooks]
169
+ return new_state, selected_id, f"Renamed to: {name}"
170
+ return gr.skip(), gr.skip(), "Failed to rename"
 
171
  except Exception as e:
172
+ return gr.skip(), gr.skip(), f"Error: {e}"
173
 
174
 
175
+ def _safe_delete(idx, state, selected_id, profile: gr.OAuthProfile | None = None):
176
  """Delete notebook at index."""
177
  try:
178
  if idx is None or idx < 0 or idx >= len(state):
179
+ return gr.skip(), gr.skip(), "Invalid selection"
180
  nb_id, _ = state[idx]
181
  user_id = _user_id(profile)
182
  if not user_id:
183
+ return gr.skip(), gr.skip(), "Please sign in"
184
  ok = delete_notebook(user_id, nb_id)
185
  if ok:
186
  notebooks = _get_notebooks(user_id)
187
+ new_state = [(n["notebook_id"], n["name"]) for n in notebooks]
 
188
  new_selected = notebooks[0]["notebook_id"] if notebooks else None
189
+ return new_state, new_selected, "Notebook deleted"
190
+ return gr.skip(), gr.skip(), "Failed to delete"
191
  except Exception as e:
192
+ return gr.skip(), gr.skip(), f"Error: {e}"
 
193
 
 
 
 
 
 
194
 
195
+ def _initial_load(profile: gr.OAuthProfile | None = None):
 
196
  """Load notebooks on app load. Uses HF OAuth profile for user_id."""
197
  user_id = _user_id(profile)
198
  notebooks = _get_notebooks(user_id)
199
  state = [(n["notebook_id"], n["name"]) for n in notebooks]
200
  selected = notebooks[0]["notebook_id"] if notebooks else None
 
201
  status = f"Signed in as {user_id}" if user_id else "Sign in with Hugging Face to manage notebooks."
202
+ auth_update = f"You are logged in as {getattr(profile, 'name', None) or user_id} ({_user_id(profile)})" if user_id else ""
203
+ auth_row_visible = bool(user_id)
204
+ return state, selected, status, auth_update, gr.update(visible=auth_row_visible), gr.update(visible=bool(user_id)), gr.update(visible=not bool(user_id))
205
+
206
+
207
+ REPORT_SCOPE_LABELS = {
208
+ "All sources (PDFs, URLs, text)": "all",
209
+ "PDF uploads only": "pdf",
210
+ "Web URLs only": "url",
211
+ "Uploaded text only": "text",
212
+ }
213
+
214
+ REPORT_SCOPE_DESCRIPTIONS = {
215
+ "all": "PDFs, URLs, and uploaded text",
216
+ "pdf": "uploaded PDFs",
217
+ "url": "ingested web URLs",
218
+ "text": "uploaded text files",
219
+ }
220
+
221
+ DEFAULT_REPORT_SCOPE_LABEL = "All sources (PDFs, URLs, text)"
222
+
223
+
224
+ def _resolve_report_scope(label: str) -> tuple[str, str]:
225
+ value = REPORT_SCOPE_LABELS.get(label, "all")
226
+ desc = REPORT_SCOPE_DESCRIPTIONS.get(value, "selected sources")
227
+ return value, desc
228
+
229
+
230
+ def _generate_report(scope_label, notebook_id, profile: gr.OAuthProfile | None):
231
+ scope_value, scope_desc = _resolve_report_scope(scope_label)
232
+ user_id = _user_id(profile)
233
+ if not user_id:
234
+ return "Please sign in with Hugging Face before generating a report.", ""
235
+ if not notebook_id:
236
+ return "Select a notebook first to generate a report.", ""
237
+ try:
238
+ report_text = generate_report(notebook_id, scope_value)
239
+ status = f"Report ready for {scope_desc}."
240
+ return status, report_text
241
+ except ValueError as error:
242
+ return f"⚠️ {error}", ""
243
+ except Exception as error:
244
+ return f"Error generating report: {error}", ""
245
 
246
 
247
  def _safe_upload_pdfs(files, selected_id, profile: gr.OAuthProfile | None):
 
299
  return f"Error uploading PDFs: {error}"
300
 
301
 
302
+ def _list_uploaded_pdfs(selected_id, profile: gr.OAuthProfile | None = None):
303
  """List uploaded PDFs for the selected notebook."""
304
  user_id = _user_id(profile)
305
  if not user_id or not selected_id:
 
314
  return gr.update(choices=pdf_names, value=selected_name)
315
 
316
 
317
+ def _safe_remove_pdf(file_name, selected_id, profile: gr.OAuthProfile | None = None):
318
  """Remove one uploaded PDF from the selected notebook."""
319
  try:
320
  user_id = _user_id(profile)
 
342
  return f"url_{h}"
343
 
344
 
345
+ def _safe_ingest_url(url, selected_id, profile: gr.OAuthProfile | None = None):
346
  """Ingest one URL into chunks table for the selected notebook."""
347
  try:
348
  user_id = _user_id(profile)
 
370
  except Exception as error:
371
  return "", f"Error ingesting URL: {error}"
372
 
373
+ def _safe_remove_url(url, selected_id, profile: gr.OAuthProfile | None = None):
374
  try:
375
  user_id = _user_id(profile)
376
  if not user_id:
 
392
 
393
 
394
 
395
+ # ── Upload Handler Functions ──────────────────────────────────
 
 
 
 
 
 
 
 
 
 
396
  def _do_upload(text_content, title, notebook_id, profile: gr.OAuthProfile | None):
397
  """Handle direct text input and ingestion."""
398
  from backend.ingestion_txt import ingest_txt
 
452
  sources = list_sources(notebook_id)
453
  return _format_sources(sources)
454
 
455
+
456
+ def _safe_generate_podcast(notebook_id, profile: gr.OAuthProfile | None = None):
457
+ user_id = _user_id(profile)
458
+ if not user_id:
459
+ return "Please sign in first.", ""
460
+ if not notebook_id:
461
+ return "Please select a notebook first.", ""
462
+
463
+ try:
464
+ result = generate_podcast(notebook_id=str(notebook_id), user_id=user_id)
465
+ status = (
466
+ f"Podcast generated. Artifact: {result['artifact_id'] or 'saved'} | "
467
+ f"Sources: {result['sources_count']} | Chunks: {result['chunks_used']}"
468
+ )
469
+ return status, result["script"]
470
+ except Exception as error:
471
+ return f"Error generating podcast: {error}", ""
472
+
473
+
474
+ def _safe_generate_podcast_audio(notebook_id, script, profile: gr.OAuthProfile | None = None):
475
+ user_id = _user_id(profile)
476
+ if not user_id:
477
+ return "Please sign in first.", None
478
+ if not notebook_id:
479
+ return "Please select a notebook first.", None
480
+ if not script or not script.strip():
481
+ return "Generate a podcast script first.", None
482
+
483
+ try:
484
+ result = generate_podcast_audio(notebook_id=str(notebook_id), user_id=user_id, script=script)
485
+ status = f"Podcast audio generated. Artifact: {result['artifact_id'] or 'saved'}"
486
+ return status, result["audio_path"]
487
+ except Exception as error:
488
+ return f"Error generating podcast audio: {error}", None
489
+
490
  # Quiz Handlers
491
  def _get_notebook_pdfs(notebook_id, profile: gr.OAuthProfile | None):
492
  user_id = _user_id(profile)
 
568
 
569
  lines.append(f"\n**Score: {score}/{len(questions)}**")
570
  return "\n\n".join(lines)
571
+ def _chat_history_to_pairs(messages: list[dict]) -> list[tuple[str, str]]:
572
+ """Convert load_chat output to Gradio Chatbot format [(user, assistant), ...]."""
573
+ pairs = []
574
+ i = 0
575
+ while i < len(messages):
576
+ m = messages[i]
577
+ if m["role"] == "user":
578
+ user_content = m["content"] or ""
579
+ asst_content = ""
580
+ if i + 1 < len(messages) and messages[i + 1]["role"] == "assistant":
581
+ asst_content = messages[i + 1]["content"] or ""
582
+ i += 1
583
+ pairs.append((user_content, asst_content))
584
+ i += 1
585
+ return pairs
586
+
587
+
588
+ def _load_chat_history(notebook_id) -> tuple[list[tuple[str, str]], list[tuple[str, str]]]:
589
+ """Load chat for notebook. Returns (history_pairs, history_pairs) for State and Chatbot."""
590
+ if not notebook_id:
591
+ return [], []
592
+ messages = load_chat(notebook_id)
593
+ pairs = _chat_history_to_pairs(messages)
594
+ return pairs, pairs
595
+
596
+
597
+ def _on_chat_submit(query, notebook_id, chat_history, profile: gr.OAuthProfile | None):
598
+ """Handle chat submit: call RAG, return updated history."""
599
+ if not notebook_id:
600
+ return "", chat_history, "Select a notebook first."
601
+ if not query or not query.strip():
602
+ return "", chat_history, "Enter a message."
603
+ user_id = _user_id(profile)
604
+ if not user_id:
605
+ return "", chat_history, "Please sign in first."
606
+ try:
607
+ answer, updated = rag_chat(notebook_id, query.strip(), chat_history)
608
+ return "", updated, ""
609
+ except Exception as e:
610
+ return "", chat_history, f"Error: {e}"
611
 
612
  def _get_quiz_pdfs(source_type, notebook_id):
613
  if source_type != "PDF":
 
668
  theme=theme,
669
  css=CUSTOM_CSS,
670
  ) as demo:
671
+ with gr.Row(elem_classes=["header-bar"]):
672
+ gr.Markdown("### 📓 NotebookLM Clone")
673
+ login_btn = gr.LoginButton(value="🤗 Login with Hugging Face", size="lg")
674
+
675
+ with gr.Row(visible=False) as auth_info_row:
676
+ auth_text = gr.Markdown("", elem_id="auth-text")
677
+
678
+ gr.HTML("""
679
+ <div class="container hero-section">
680
+ <h1 class="hero-title">📓 NotebookLM Clone</h1>
681
+ <p class="hero-sub">Chat with your documents. Generate reports, quizzes, and podcasts with citations.</p>
682
+ </div>
683
+ """)
684
+
685
+ with gr.Column(visible=False, elem_classes=["login-container"]) as login_container:
686
+ gr.Markdown("**Sign in with Hugging Face to access your notebooks.**", elem_classes=["login-center"])
687
+
688
+ with gr.Column(visible=False) as app_content:
689
+ nb_state = gr.State([])
690
+ selected_notebook_id = gr.State(None)
691
+
692
+ with gr.Group(elem_classes=["create-strip"]):
693
+ with gr.Row(elem_classes=["create-row"]):
694
+ gr.Markdown("Create new notebook", elem_classes=["create-label"])
695
+ create_txt = gr.Textbox(
696
+ placeholder="Enter new notebook name",
697
+ show_label=False,
698
+ container=False,
699
+ value="",
700
+ )
701
+ create_btn = gr.Button("Create", variant="primary", size="sm")
702
+
703
+ with gr.Group(elem_classes=["section-card"]):
704
+ gr.Markdown("**Sources**", elem_classes=["section-title"])
705
+ gr.Markdown("*Upload PDFs, ingest URLs, or add text to your selected notebook*")
706
+ with gr.Row(elem_classes=["section-row"]):
707
+ pdf_upload_btn = gr.UploadButton(
708
+ "Upload PDFs",
709
+ file_types=[".pdf"],
710
+ file_count="multiple",
711
+ type="filepath",
712
+ variant="secondary",
713
+ )
714
+ with gr.Row(elem_classes=["section-row"]):
715
+ uploaded_pdf_dd = gr.Dropdown(
716
+ label="Uploaded PDFs",
717
+ choices=[],
718
+ value=None,
719
+ scale=3,
720
+ allow_custom_value=False,
721
+ )
722
+ remove_pdf_btn = gr.Button("Remove selected PDF", variant="stop", scale=1)
723
+ with gr.Row(elem_classes=["section-row"]):
724
+ url_txt = gr.Textbox(
725
+ label="Ingest web URL",
726
+ placeholder="https://example.com",
727
+ value="",
728
+ scale=3,
729
+ )
730
+ ingest_url_btn = gr.Button("Ingest URL", variant="primary", scale=1)
731
+ remove_url_btn = gr.Button("Delete URL", variant="stop", scale=1)
732
+
733
+ gr.HTML("<br>")
734
+ gr.Markdown("**Your Notebooks**", elem_classes=["section-title"])
735
+ gr.Markdown("*Selected notebook is used for chat and ingestion*", elem_id="sub-hint")
736
+ gr.HTML("<br>")
737
+
738
+ status = gr.Markdown("Sign in with Hugging Face to manage notebooks.", elem_classes=["status"])
739
+
740
+ @gr.render(inputs=[nb_state])
741
+ def render_notebooks(state):
742
+ if not state:
743
+ gr.Markdown("No notebooks yet. Create one to get started.")
744
+ else:
745
+ for i, (nb_id, name) in enumerate(state):
746
+ idx = i
747
+ with gr.Row(elem_classes=["notebook-card"]):
748
+ name_txt = gr.Textbox(value=name, show_label=False, scale=4, min_width=240, key=f"nb-name-{nb_id}")
749
+ select_btn = gr.Button("Select", variant="primary", scale=1, min_width=80, size="sm")
750
+ rename_btn = gr.Button("Rename", variant="secondary", scale=1, min_width=80, size="sm")
751
+ delete_btn = gr.Button("Delete", variant="secondary", scale=1, min_width=80, size="sm")
752
+
753
+ def on_select(nb_id=nb_id):
754
+ return nb_id
755
+
756
+ def on_select_status():
757
+ return "Selected notebook updated. Use this for chat/ingestion."
758
+
759
+ select_btn.click(
760
+ on_select,
761
+ inputs=None,
762
+ outputs=[selected_notebook_id],
763
+ ).then(on_select_status, None, [status]).then(_list_uploaded_pdfs, inputs=[selected_notebook_id], outputs=[uploaded_pdf_dd])
764
+
765
+ rename_btn.click(
766
+ _safe_rename,
767
+ inputs=[gr.State(idx), name_txt, nb_state, selected_notebook_id],
768
+ outputs=[nb_state, selected_notebook_id, status],
769
+ api_name=False,
770
+ )
771
+
772
+ delete_btn.click(
773
+ _safe_delete,
774
+ inputs=[gr.State(idx), nb_state, selected_notebook_id],
775
+ outputs=[nb_state, selected_notebook_id, status],
776
+ api_name=False,
777
+ ).then(_list_uploaded_pdfs, inputs=[selected_notebook_id], outputs=[uploaded_pdf_dd])
778
+
779
+ gr.HTML("<br>")
780
+
781
+ with gr.Group(elem_classes=["section-card"]):
782
+ gr.Markdown("**Add Text**", elem_classes=["section-title"])
783
+ gr.Markdown("*Select a notebook above, then paste or type your text*")
784
+ with gr.Row():
785
+ txt_title = gr.Textbox(
786
+ label="Title",
787
+ placeholder="Give this text a name (e.g. 'Lecture Notes Week 1')",
788
+ scale=1,
789
+ )
790
+ txt_input = gr.Textbox(
791
+ label="Text Content",
792
+ placeholder="Paste or type your text here...",
793
+ lines=10,
794
+ )
795
+ submit_btn = gr.Button("Save & Process", variant="primary")
796
+ upload_status = gr.Markdown("", elem_classes=["status"])
797
+ sources_display = gr.Markdown("")
798
+
799
+ gr.HTML("<br>")
800
+ with gr.Group(elem_classes=["section-card"]):
801
+ gr.Markdown("**Reports**", elem_classes=["section-title"])
802
+ gr.Markdown("*Generate a concise report based on your uploaded PDFs, ingested URLs, or added text.*")
803
+ with gr.Row(elem_classes=["section-row"]):
804
+ report_scope_dd = gr.Dropdown(
805
+ label="Report scope",
806
+ choices=list(REPORT_SCOPE_LABELS.keys()),
807
+ value=DEFAULT_REPORT_SCOPE_LABEL,
808
+ scale=3,
809
+ )
810
+ report_btn = gr.Button("Generate report", variant="primary", scale=1)
811
+ report_status = gr.Markdown("Select a scope and click generate.", elem_classes=["status"])
812
+ report_output = gr.Markdown("", elem_id="report-output")
813
+
814
+ with gr.Group(elem_classes=["section-card"]):
815
+ gr.Markdown("**Chat**", elem_classes=["section-title"])
816
+ gr.Markdown("*Ask questions about your notebook sources. Answers are grounded in retrieved chunks with citations.*")
817
+ chat_history_state = gr.State([])
818
+ chatbot = gr.Chatbot(label="Chat history", height=400)
819
+ chat_input = gr.Textbox(
820
+ label="Message",
821
+ placeholder="Ask a question about your sources...",
822
  show_label=False,
823
+ lines=2,
 
824
  )
825
+ chat_submit_btn = gr.Button("Send", variant="primary")
826
+ chat_status = gr.Markdown("", elem_classes=["status"])
 
 
 
827
 
828
+ demo.load(
829
+ _initial_load,
830
+ inputs=None,
831
+ outputs=[nb_state, selected_notebook_id, status, auth_text, auth_info_row, app_content, login_container],
832
+ api_name=False,
833
+ )
834
+ demo.load(_list_uploaded_pdfs, inputs=[selected_notebook_id], outputs=[uploaded_pdf_dd], api_name=False)
835
+
836
+ def _on_notebook_select_for_chat(notebook_id):
837
+ hist, _ = _load_chat_history(notebook_id)
838
+ return hist, hist
839
+
840
+ selected_notebook_id.change(
841
+ _on_notebook_select_for_chat,
842
+ inputs=[selected_notebook_id],
843
+ outputs=[chat_history_state, chatbot],
844
+ api_name=False,
845
+ )
846
 
 
847
  create_btn.click(
848
  _safe_create,
849
  inputs=[create_txt, nb_state, selected_notebook_id],
850
+ outputs=[create_txt, nb_state, selected_notebook_id, status],
851
  api_name=False,
852
  ).then(_list_uploaded_pdfs, inputs=[selected_notebook_id], outputs=[uploaded_pdf_dd])
853
 
 
879
  api_name=False,
880
  ).then(_list_uploaded_pdfs, inputs=[selected_notebook_id], outputs=[uploaded_pdf_dd])
881
 
 
 
 
 
 
 
882
 
 
 
 
 
 
 
 
 
 
 
 
 
 
883
  # Text Input Section
884
  gr.Markdown("---")
885
  gr.Markdown("## Add Text")
 
902
 
903
  upload_status = gr.Markdown("", elem_classes=["status"])
904
 
905
+ # Podcast Section
906
+ gr.Markdown("---")
907
+ gr.Markdown("## Podcast")
908
+ gr.Markdown("Generate a podcast script for the selected notebook using all ingested content.")
909
+ with gr.Row():
910
+ podcast_btn = gr.Button("Generate Podcast", variant="primary")
911
+ podcast_audio_btn = gr.Button("Generate Podcast Audio", variant="secondary")
912
+ podcast_status = gr.Markdown("", elem_classes=["status"])
913
+ podcast_script = gr.Markdown("")
914
+ podcast_audio = gr.Audio(label="Podcast Audio", type="filepath")
915
+
916
  # Quiz Section
917
  gr.Markdown("---")
918
  gr.Markdown("## Generate Quiz")
 
989
  outputs=[upload_status],
990
  )
991
 
992
+ report_btn.click(
993
+ _generate_report,
994
+ inputs=[report_scope_dd, selected_notebook_id],
995
+ outputs=[report_status, report_output],
996
+ api_name=False,
997
+ )
998
+
999
+ selected_notebook_id.change(
1000
+ _load_sources,
1001
+ inputs=[selected_notebook_id],
1002
+ outputs=[podcast_status, podcast_script],
1003
+ api_name=False,
1004
+ )
1005
+
1006
+ podcast_btn.click(
1007
+ _safe_generate_podcast,
1008
+ inputs=[selected_notebook_id],
1009
+ outputs=[podcast_status, podcast_script],
1010
+ api_name=False,
1011
+ )
1012
+
1013
+ podcast_audio_btn.click(
1014
+ _safe_generate_podcast_audio,
1015
+ inputs=[selected_notebook_id, podcast_script],
1016
+ outputs=[podcast_status, podcast_audio],
1017
+ api_name=False,
1018
+ )
1019
+
1020
  quiz_source_type.change(
1021
  _quiz_pdf_dropdown_update,
1022
  inputs=[quiz_source_type, selected_notebook_id],
 
1062
  api_name=False,
1063
  )
1064
 
1065
+ chat_submit_btn.click(
1066
+ _on_chat_submit,
1067
+ inputs=[chat_input, selected_notebook_id, chat_history_state],
1068
+ outputs=[chat_input, chat_history_state, chat_status],
1069
+ api_name=False,
1070
+ ).then(
1071
+ lambda h: (h, h),
1072
+ inputs=[chat_history_state],
1073
+ outputs=[chat_history_state, chatbot],
1074
+ )
1075
 
1076
+ if __name__ == "__main__":
1077
+ _log("5. Launching Gradio...")
1078
+ demo.launch()
backend/embedding_service.py ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Shared embedding service - 384-dim vectors for RAG (ingestion + retrieval). Uses MiniLM for low memory."""
2
+
3
+ from sentence_transformers import SentenceTransformer
4
+
5
+ _MODEL_NAME = "sentence-transformers/all-MiniLM-L6-v2"
6
+ _model = None
7
+
8
+
9
+ def _get_model() -> SentenceTransformer:
10
+ """Lazy-load the embedding model."""
11
+ global _model
12
+ if _model is None:
13
+ _model = SentenceTransformer(_MODEL_NAME)
14
+ return _model
15
+
16
+
17
+ def encode(texts: list[str], task: str = "search_document") -> list[list[float]]:
18
+ """
19
+ Embed texts. Returns list of 384-dim vectors.
20
+
21
+ Args:
22
+ texts: List of strings to embed.
23
+ task: Unused (MiniLM doesn't need prefix); kept for API compatibility.
24
+ """
25
+ if not texts:
26
+ return []
27
+
28
+ model = _get_model()
29
+ embeddings = model.encode(texts, show_progress_bar=False)
30
+ return [e.tolist() for e in embeddings]
backend/ingestion_service.py CHANGED
@@ -1,10 +1,11 @@
1
- """PDF ingestion for RAG: extract text, chunk, and persist to chunks table."""
2
 
3
  from pathlib import Path
4
 
5
  from pypdf import PdfReader
6
 
7
  from backend.db import supabase
 
8
 
9
  import requests
10
  from bs4 import BeautifulSoup
@@ -39,7 +40,7 @@ def _chunk_text(text: str, chunk_size: int = DEFAULT_CHUNK_SIZE, overlap: int =
39
 
40
 
41
  def ingest_pdf_chunks(notebook_id: str, source_id: str, pdf_path: Path) -> int:
42
- """Extract and store chunks for a single PDF. Returns number of chunks inserted."""
43
  text = _extract_pdf_text(pdf_path)
44
  chunks = _chunk_text(text)
45
 
@@ -48,11 +49,14 @@ def ingest_pdf_chunks(notebook_id: str, source_id: str, pdf_path: Path) -> int:
48
  if not chunks:
49
  return 0
50
 
 
 
51
  rows = [
52
  {
53
  "notebook_id": notebook_id,
54
  "source_id": source_id,
55
  "content": chunk,
 
56
  "metadata": {
57
  "file_name": source_id,
58
  "file_path": str(pdf_path),
@@ -88,6 +92,7 @@ def _extract_url_text(url: str) -> str:
88
  return " ".join(text.split()).strip()
89
 
90
  def ingest_url_chunks(notebook_id: str, source_id: str, url: str) -> int:
 
91
  text = _extract_url_text(url)
92
  chunks = _chunk_text(text)
93
 
@@ -96,11 +101,14 @@ def ingest_url_chunks(notebook_id: str, source_id: str, url: str) -> int:
96
  if not chunks:
97
  return 0
98
 
 
 
99
  rows = [
100
  {
101
  "notebook_id": notebook_id,
102
  "source_id": source_id,
103
  "content": chunk,
 
104
  "metadata": {
105
  "url": url,
106
  "chunk_index": index,
 
1
+ """PDF ingestion for RAG: extract text, chunk, embed, and persist to chunks table."""
2
 
3
  from pathlib import Path
4
 
5
  from pypdf import PdfReader
6
 
7
  from backend.db import supabase
8
+ from backend.embedding_service import encode as embed_texts
9
 
10
  import requests
11
  from bs4 import BeautifulSoup
 
40
 
41
 
42
  def ingest_pdf_chunks(notebook_id: str, source_id: str, pdf_path: Path) -> int:
43
+ """Extract, embed, and store chunks for a single PDF. Returns number of chunks inserted."""
44
  text = _extract_pdf_text(pdf_path)
45
  chunks = _chunk_text(text)
46
 
 
49
  if not chunks:
50
  return 0
51
 
52
+ embeddings = embed_texts(chunks, task="search_document")
53
+
54
  rows = [
55
  {
56
  "notebook_id": notebook_id,
57
  "source_id": source_id,
58
  "content": chunk,
59
+ "embedding": embeddings[index],
60
  "metadata": {
61
  "file_name": source_id,
62
  "file_path": str(pdf_path),
 
92
  return " ".join(text.split()).strip()
93
 
94
  def ingest_url_chunks(notebook_id: str, source_id: str, url: str) -> int:
95
+ """Extract, embed, and store chunks for a URL. Returns number of chunks inserted."""
96
  text = _extract_url_text(url)
97
  chunks = _chunk_text(text)
98
 
 
101
  if not chunks:
102
  return 0
103
 
104
+ embeddings = embed_texts(chunks, task="search_document")
105
+
106
  rows = [
107
  {
108
  "notebook_id": notebook_id,
109
  "source_id": source_id,
110
  "content": chunk,
111
+ "embedding": embeddings[index],
112
  "metadata": {
113
  "url": url,
114
  "chunk_index": index,
backend/ingestion_txt.py CHANGED
@@ -17,6 +17,9 @@ from sentence_transformers import SentenceTransformer
17
  # Load model once at module level (not on every call)
18
  _model = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2")
19
  # Constants
 
 
 
20
  MAX_FILE_SIZE = 10 * 1024 * 1024 # 10MB
21
 
22
 
@@ -114,16 +117,14 @@ def chunk_text(text: str, source_id: str, notebook_id: str, filename: str = "")
114
  # Embed + Store
115
  def embed_and_store_chunks(chunks: list[dict]) -> None:
116
  """
117
- Embed chunks using sentence-transformers and store in pgvector.
118
  """
119
  if not chunks:
120
  return
121
 
122
- # Embed all chunks in one batch
123
  texts = [c["content"] for c in chunks]
124
- embeddings = _model.encode(texts, show_progress_bar=False)
125
 
126
- # Build rows for Supabase insert
127
  rows = []
128
  for chunk, embedding in zip(chunks, embeddings):
129
  rows.append({
@@ -131,7 +132,7 @@ def embed_and_store_chunks(chunks: list[dict]) -> None:
131
  "source_id": str(chunk["source_id"]),
132
  "notebook_id": str(chunk["notebook_id"]),
133
  "content": chunk["content"],
134
- "embedding": embedding.tolist(),
135
  "metadata": chunk["metadata"]
136
  })
137
 
 
17
  # Load model once at module level (not on every call)
18
  _model = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2")
19
  # Constants
20
+ from backend.embedding_service import encode as embed_texts
21
+ # ── Constants ────────────────────────────────────────────────
22
+
23
  MAX_FILE_SIZE = 10 * 1024 * 1024 # 10MB
24
 
25
 
 
117
  # Embed + Store
118
  def embed_and_store_chunks(chunks: list[dict]) -> None:
119
  """
120
+ Embed chunks using shared 1536-dim model and store in pgvector.
121
  """
122
  if not chunks:
123
  return
124
 
 
125
  texts = [c["content"] for c in chunks]
126
+ embeddings = embed_texts(texts, task="search_document")
127
 
 
128
  rows = []
129
  for chunk, embedding in zip(chunks, embeddings):
130
  rows.append({
 
132
  "source_id": str(chunk["source_id"]),
133
  "notebook_id": str(chunk["notebook_id"]),
134
  "content": chunk["content"],
135
+ "embedding": embedding,
136
  "metadata": chunk["metadata"]
137
  })
138
 
backend/llm_client.py ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Helper for shared LLM-based features."""
2
+
3
+ import os
4
+
5
+ from openai import OpenAI
6
+
7
+ DEFAULT_MODEL = os.getenv("LLM_MODEL", "meta-llama/Llama-3.2-3B-Instruct:together")
8
+
9
+ _client: OpenAI | None = None
10
+
11
+
12
+ def get_llm_client() -> OpenAI:
13
+ """Return a cached OpenAI client configured with the Hugging Face router."""
14
+ global _client
15
+ if _client is None:
16
+ token = os.getenv("HF_TOKEN")
17
+ if not token:
18
+ raise RuntimeError("HF_TOKEN environment variable is required for LLM calls.")
19
+ _client = OpenAI(
20
+ base_url="https://router.huggingface.co/v1",
21
+ api_key=token,
22
+ )
23
+ return _client
backend/podcast_service.py ADDED
@@ -0,0 +1,516 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Podcast generation service using notebook content from chunks table."""
2
+
3
+ import io
4
+ import os
5
+ import re
6
+ import textwrap
7
+ import tempfile
8
+ import wave
9
+ import aifc
10
+ import audioop
11
+ import subprocess
12
+ import shutil
13
+ import numpy as np
14
+ from datetime import datetime
15
+ from pathlib import Path
16
+
17
+ from huggingface_hub import InferenceClient
18
+ import requests
19
+ import pyttsx3
20
+
21
+ try:
22
+ import torch
23
+ from transformers import AutoProcessor, AutoModelForTextToSpectrogram, SpeechT5HifiGan
24
+ except Exception:
25
+ torch = None
26
+ AutoProcessor = None
27
+ AutoModelForTextToSpectrogram = None
28
+ SpeechT5HifiGan = None
29
+
30
+ from backend.artifacts_service import create_artifact
31
+ from backend.db import supabase
32
+ from backend.storage import get_artifacts_path, save_file
33
+
34
+ HF_TOKEN = os.getenv("HF_TOKEN", "")
35
+ HF_MODEL = os.getenv("HF_PODCAST_MODEL", "meta-llama/Llama-3.2-3B-Instruct")
36
+ HF_TTS_MODEL = os.getenv("HF_TTS_MODEL", "microsoft/speecht5_tts")
37
+ HF_TTS_FALLBACK_MODELS = [
38
+ "microsoft/speecht5_tts",
39
+ "facebook/mms-tts",
40
+ "hexgrad/Kokoro-82M",
41
+ "facebook/mms-tts-eng",
42
+ "suno/bark-small",
43
+ ]
44
+
45
+ _SPEECHT5_PROCESSOR = None
46
+ _SPEECHT5_MODEL = None
47
+ _SPEECHT5_VOCODER = None
48
+
49
+
50
+ def _tts_models() -> list[str]:
51
+ candidates = [HF_TTS_MODEL] + HF_TTS_FALLBACK_MODELS
52
+ seen = set()
53
+ ordered = []
54
+ for model in candidates:
55
+ if model and model not in seen:
56
+ seen.add(model)
57
+ ordered.append(model)
58
+ return ordered
59
+
60
+
61
+ def _load_speecht5():
62
+ global _SPEECHT5_PROCESSOR, _SPEECHT5_MODEL, _SPEECHT5_VOCODER
63
+ if AutoProcessor is None or AutoModelForTextToSpectrogram is None or SpeechT5HifiGan is None or torch is None:
64
+ raise ValueError("transformers/torch SpeechT5 dependencies are not available")
65
+
66
+ if _SPEECHT5_PROCESSOR is None:
67
+ _SPEECHT5_PROCESSOR = AutoProcessor.from_pretrained("microsoft/speecht5_tts")
68
+ if _SPEECHT5_MODEL is None:
69
+ _SPEECHT5_MODEL = AutoModelForTextToSpectrogram.from_pretrained("microsoft/speecht5_tts")
70
+ if _SPEECHT5_VOCODER is None:
71
+ _SPEECHT5_VOCODER = SpeechT5HifiGan.from_pretrained("microsoft/speecht5_hifigan")
72
+
73
+ _SPEECHT5_MODEL.eval()
74
+ _SPEECHT5_VOCODER.eval()
75
+
76
+
77
+ def _wav_from_float32(samples: np.ndarray, sample_rate: int = 16000) -> bytes:
78
+ clipped = np.clip(samples, -1.0, 1.0)
79
+ pcm16 = (clipped * 32767.0).astype(np.int16)
80
+ buffer = io.BytesIO()
81
+ with wave.open(buffer, "wb") as wav_file:
82
+ wav_file.setnchannels(1)
83
+ wav_file.setsampwidth(2)
84
+ wav_file.setframerate(sample_rate)
85
+ wav_file.writeframes(pcm16.tobytes())
86
+ return buffer.getvalue()
87
+
88
+
89
+ def _synthesize_local_tts_with_transformers(text: str) -> bytes:
90
+ _load_speecht5()
91
+ chunks = textwrap.wrap(text, width=350, break_long_words=False, break_on_hyphens=False)
92
+ if not chunks:
93
+ raise ValueError("No text provided for SpeechT5 generation")
94
+
95
+ speaker_embeddings = torch.zeros((1, 512), dtype=torch.float32)
96
+ all_parts = []
97
+
98
+ with torch.inference_mode():
99
+ for chunk in chunks:
100
+ inputs = _SPEECHT5_PROCESSOR(text=chunk, return_tensors="pt")
101
+ speech = _SPEECHT5_MODEL.generate_speech(
102
+ inputs["input_ids"],
103
+ speaker_embeddings,
104
+ vocoder=_SPEECHT5_VOCODER,
105
+ )
106
+ samples = speech.cpu().numpy().astype(np.float32)
107
+ all_parts.append(_wav_from_float32(samples, sample_rate=16000))
108
+
109
+ return _concat_wav_bytes(all_parts)
110
+
111
+
112
+ def _get_notebook_chunks(notebook_id: str, limit: int = 300) -> list[dict]:
113
+ result = (
114
+ supabase.table("chunks")
115
+ .select("source_id, content, metadata, created_at")
116
+ .eq("notebook_id", notebook_id)
117
+ .order("created_at")
118
+ .limit(limit)
119
+ .execute()
120
+ )
121
+ return result.data or []
122
+
123
+
124
+ def _sentences(text: str) -> list[str]:
125
+ candidates = re.split(r"(?<=[.!?])\s+", text.strip())
126
+ cleaned = [s.strip() for s in candidates if len(s.strip()) > 40]
127
+ return cleaned
128
+
129
+
130
+ def _fallback_script(notebook_id: str, chunks: list[dict]) -> str:
131
+ grouped: dict[str, list[str]] = {}
132
+ for row in chunks:
133
+ source = row.get("source_id") or "unknown"
134
+ grouped.setdefault(source, []).append(row.get("content") or "")
135
+
136
+ lines = [
137
+ "# Podcast Script",
138
+ "",
139
+ "## Intro",
140
+ "Host 1: Welcome back! Today we're diving into this notebook's key ideas.",
141
+ "Host 2: We'll walk through the main sources and end with practical takeaways.",
142
+ "",
143
+ ]
144
+
145
+ for index, (source, pieces) in enumerate(grouped.items(), start=1):
146
+ merged = " ".join(pieces)
147
+ summary_sentences = _sentences(merged)[:3]
148
+ lines.append(f"## Segment {index}: {source}")
149
+ if summary_sentences:
150
+ for sent in summary_sentences:
151
+ lines.append(f"- {sent}")
152
+ else:
153
+ lines.append("- This source was ingested but had limited extractable text.")
154
+ lines.append("")
155
+
156
+ lines.extend(
157
+ [
158
+ "## Key Takeaways",
159
+ "- The notebook combines multiple sources into a coherent set of ideas.",
160
+ "- Compare perspectives across sources before drawing conclusions.",
161
+ "- Use this script as a draft and tailor it to your audience.",
162
+ "",
163
+ "## Outro",
164
+ f"Host 1: That wraps up our notebook walkthrough for {notebook_id}.",
165
+ "Host 2: Thanks for listening — see you in the next episode!",
166
+ ]
167
+ )
168
+
169
+ return "\n".join(lines)
170
+
171
+
172
+ def _llm_script(notebook_id: str, chunks: list[dict]) -> str:
173
+ if not HF_TOKEN:
174
+ return _fallback_script(notebook_id, chunks)
175
+
176
+ context_parts = []
177
+ total_chars = 0
178
+ char_cap = 18000
179
+ for row in chunks:
180
+ content = (row.get("content") or "").strip()
181
+ source = row.get("source_id") or "unknown"
182
+ snippet = f"[{source}] {content}"
183
+ if total_chars + len(snippet) > char_cap:
184
+ break
185
+ context_parts.append(snippet)
186
+ total_chars += len(snippet)
187
+
188
+ context = "\n\n".join(context_parts)
189
+ prompt = f"""
190
+ You are creating a podcast script from notebook materials.
191
+ Write a clear, engaging script with these sections:
192
+ 1) Intro
193
+ 2) 3-6 discussion segments
194
+ 3) Key Takeaways
195
+ 4) Outro
196
+
197
+ Rules:
198
+ - Keep it factual and grounded in provided content.
199
+ - Use two hosts: Host 1 and Host 2.
200
+ - Mention source names when relevant.
201
+ - Output markdown only.
202
+
203
+ Notebook ID: {notebook_id}
204
+
205
+ Context:
206
+ {context}
207
+ """.strip()
208
+
209
+ try:
210
+ client = InferenceClient(token=HF_TOKEN)
211
+ response = client.chat_completion(
212
+ model=HF_MODEL,
213
+ messages=[{"role": "user", "content": prompt}],
214
+ max_tokens=1400,
215
+ temperature=0.3,
216
+ )
217
+ text = (response.choices[0].message.content or "").strip()
218
+ return text if text else _fallback_script(notebook_id, chunks)
219
+ except Exception:
220
+ return _fallback_script(notebook_id, chunks)
221
+
222
+
223
+ def generate_podcast(notebook_id: str, user_id: str) -> dict:
224
+ """Generate podcast markdown from notebook chunks and persist artifact."""
225
+ chunks = _get_notebook_chunks(notebook_id)
226
+ if not chunks:
227
+ raise ValueError("No notebook content found. Add text/PDF/URL sources first.")
228
+
229
+ script = _llm_script(notebook_id, chunks)
230
+
231
+ stamp = datetime.utcnow().strftime("%Y%m%d_%H%M%S")
232
+ filename = f"podcast_{stamp}.md"
233
+ storage_path = f"{get_artifacts_path(user_id, notebook_id)}/{filename}"
234
+
235
+ save_file(storage_path, script)
236
+ artifact = create_artifact(notebook_id=notebook_id, type="podcast", storage_path=storage_path)
237
+
238
+ return {
239
+ "artifact_id": artifact["id"] if artifact else None,
240
+ "storage_path": storage_path,
241
+ "script": script,
242
+ "sources_count": len({row.get('source_id') for row in chunks}),
243
+ "chunks_used": len(chunks),
244
+ }
245
+
246
+
247
+ def _normalize_script_for_tts(script: str) -> str:
248
+ cleaned = re.sub(r"[#*_`>-]", " ", script)
249
+ cleaned = re.sub(r"\s+", " ", cleaned).strip()
250
+ return cleaned
251
+
252
+
253
+ def _tts_request(text: str) -> bytes:
254
+ if not HF_TOKEN:
255
+ raise ValueError("HF_TOKEN is required for TTS generation.")
256
+
257
+ headers = {
258
+ "Authorization": f"Bearer {HF_TOKEN}",
259
+ "Content-Type": "application/json",
260
+ "Accept": "audio/wav",
261
+ }
262
+ payload_variants = [{"inputs": text}, {"text_inputs": text}]
263
+
264
+ errors = []
265
+ base_urls = [
266
+ "https://router.huggingface.co/hf-inference/models",
267
+ "https://api-inference.huggingface.co/models",
268
+ ]
269
+
270
+ for model_id in _tts_models():
271
+ client = InferenceClient(token=HF_TOKEN)
272
+ try:
273
+ audio = client.text_to_speech(text, model=model_id)
274
+ if isinstance(audio, (bytes, bytearray)) and audio:
275
+ return bytes(audio)
276
+ if hasattr(audio, "read"):
277
+ data = audio.read()
278
+ if data:
279
+ return data
280
+ errors.append(f"{model_id}: client returned empty audio")
281
+ except Exception as exc:
282
+ errors.append(f"{model_id}: InferenceClient {exc}")
283
+
284
+ for base_url in base_urls:
285
+ url = f"{base_url}/{model_id}"
286
+ for payload in payload_variants:
287
+ try:
288
+ response = requests.post(url, headers=headers, json=payload, timeout=180)
289
+ if response.status_code in (404, 410):
290
+ errors.append(f"{model_id} @ {base_url}: HTTP {response.status_code}")
291
+ continue
292
+
293
+ response.raise_for_status()
294
+ if response.content:
295
+ return response.content
296
+
297
+ errors.append(f"{model_id} @ {base_url}: empty audio response")
298
+ except requests.RequestException as exc:
299
+ errors.append(f"{model_id} @ {base_url}: {exc}")
300
+
301
+ raise ValueError(
302
+ "TTS generation failed for all candidate models. "
303
+ f"Tried: {', '.join(_tts_models())}. "
304
+ f"Details: {' | '.join(errors[:3])}"
305
+ )
306
+
307
+
308
+ def _concat_wav_bytes(parts: list[bytes]) -> bytes:
309
+ if not parts:
310
+ raise ValueError("No audio segments returned from TTS model.")
311
+
312
+ output_buffer = io.BytesIO()
313
+ output_wave = None
314
+ params = None
315
+
316
+ try:
317
+ for index, part in enumerate(parts):
318
+ with wave.open(io.BytesIO(part), "rb") as segment:
319
+ if index == 0:
320
+ params = segment.getparams()
321
+ output_wave = wave.open(output_buffer, "wb")
322
+ output_wave.setparams(params)
323
+ target_channels = params.nchannels
324
+ target_width = params.sampwidth
325
+ target_rate = params.framerate
326
+
327
+ input_frames = segment.readframes(segment.getnframes())
328
+ input_width = segment.getsampwidth()
329
+ input_channels = segment.getnchannels()
330
+ input_rate = segment.getframerate()
331
+
332
+ if input_width != target_width:
333
+ input_frames = audioop.lin2lin(input_frames, input_width, target_width)
334
+ input_width = target_width
335
+
336
+ if input_channels != target_channels:
337
+ if input_channels == 2 and target_channels == 1:
338
+ input_frames = audioop.tomono(input_frames, input_width, 0.5, 0.5)
339
+ elif input_channels == 1 and target_channels == 2:
340
+ input_frames = audioop.tostereo(input_frames, input_width, 1.0, 1.0)
341
+ else:
342
+ raise ValueError(
343
+ f"Unsupported channel conversion from {input_channels} to {target_channels}"
344
+ )
345
+ input_channels = target_channels
346
+
347
+ if input_rate != target_rate:
348
+ input_frames, _ = audioop.ratecv(
349
+ input_frames,
350
+ input_width,
351
+ input_channels,
352
+ input_rate,
353
+ target_rate,
354
+ None,
355
+ )
356
+
357
+ output_wave.writeframes(input_frames)
358
+ finally:
359
+ if output_wave is not None:
360
+ output_wave.close()
361
+
362
+ return output_buffer.getvalue()
363
+
364
+
365
+ def _is_valid_wav(audio_bytes: bytes) -> bool:
366
+ try:
367
+ with wave.open(io.BytesIO(audio_bytes), "rb") as wav_file:
368
+ return wav_file.getnframes() > 0 and wav_file.getframerate() > 0
369
+ except Exception:
370
+ return False
371
+
372
+
373
+ def _aiff_to_wav_bytes(aiff_path: Path) -> bytes:
374
+ with aifc.open(str(aiff_path), "rb") as aiff_file:
375
+ nchannels = aiff_file.getnchannels()
376
+ sampwidth = aiff_file.getsampwidth()
377
+ framerate = aiff_file.getframerate()
378
+ frames = aiff_file.readframes(aiff_file.getnframes())
379
+
380
+ buffer = io.BytesIO()
381
+ with wave.open(buffer, "wb") as wav_file:
382
+ wav_file.setnchannels(nchannels)
383
+ wav_file.setsampwidth(sampwidth)
384
+ wav_file.setframerate(framerate)
385
+ wav_file.writeframes(frames)
386
+ return buffer.getvalue()
387
+
388
+
389
+ def _synthesize_local_tts_with_say(text: str) -> tuple[bytes, str]:
390
+ say_bin = shutil.which("say")
391
+ if not say_bin:
392
+ raise ValueError("macOS 'say' command not found")
393
+ afconvert_bin = shutil.which("afconvert")
394
+
395
+ parts: list[bytes] = []
396
+ chunks = textwrap.wrap(text, width=700, break_long_words=False, break_on_hyphens=False)
397
+ if not chunks:
398
+ raise ValueError("No text provided for macOS say generation")
399
+
400
+ with tempfile.TemporaryDirectory() as temp_dir:
401
+ for index, chunk in enumerate(chunks):
402
+ aiff_path = Path(temp_dir) / f"tts_{index}.aiff"
403
+ subprocess.run(
404
+ [say_bin, "-o", str(aiff_path), chunk],
405
+ check=True,
406
+ stdout=subprocess.DEVNULL,
407
+ stderr=subprocess.PIPE,
408
+ text=True,
409
+ )
410
+
411
+ if not aiff_path.exists() or aiff_path.stat().st_size == 0:
412
+ raise ValueError("macOS say produced empty audio")
413
+
414
+ wav_bytes = None
415
+ if afconvert_bin:
416
+ wav_path = Path(temp_dir) / f"tts_{index}.wav"
417
+ subprocess.run(
418
+ [afconvert_bin, "-f", "WAVE", "-d", "LEI16", "-r", "22050", str(aiff_path), str(wav_path)],
419
+ check=True,
420
+ stdout=subprocess.DEVNULL,
421
+ stderr=subprocess.PIPE,
422
+ text=True,
423
+ )
424
+ if wav_path.exists() and wav_path.stat().st_size > 0:
425
+ wav_bytes = wav_path.read_bytes()
426
+
427
+ if not wav_bytes:
428
+ wav_bytes = _aiff_to_wav_bytes(aiff_path)
429
+
430
+ if wav_bytes and _is_valid_wav(wav_bytes):
431
+ parts.append(wav_bytes)
432
+ continue
433
+
434
+ raise ValueError("macOS say WAV conversion failed")
435
+
436
+ combined = _concat_wav_bytes(parts)
437
+ if not _is_valid_wav(combined):
438
+ raise ValueError("macOS say fallback produced invalid WAV")
439
+ return combined, ".wav"
440
+
441
+
442
+ def _synthesize_tts_audio(script: str) -> tuple[bytes, str]:
443
+ normalized = _normalize_script_for_tts(script)
444
+ if not normalized:
445
+ raise ValueError("Podcast script is empty.")
446
+
447
+ chunks = textwrap.wrap(normalized, width=800, break_long_words=False, break_on_hyphens=False)
448
+ try:
449
+ audio_parts = [_tts_request(chunk) for chunk in chunks]
450
+ cloud_audio = _concat_wav_bytes(audio_parts)
451
+ if _is_valid_wav(cloud_audio):
452
+ return cloud_audio, ".wav"
453
+ raise ValueError("Cloud TTS returned invalid/empty WAV")
454
+ except Exception as cloud_error:
455
+ transformers_error = None
456
+ try:
457
+ transformers_audio = _synthesize_local_tts_with_transformers(normalized)
458
+ if _is_valid_wav(transformers_audio):
459
+ return transformers_audio, ".wav"
460
+ raise ValueError("SpeechT5 produced invalid WAV")
461
+ except Exception as exc:
462
+ transformers_error = exc
463
+
464
+ say_error = None
465
+ try:
466
+ say_audio, say_ext = _synthesize_local_tts_with_say(normalized)
467
+ if say_audio:
468
+ return say_audio, say_ext
469
+ raise ValueError("macOS say produced empty audio")
470
+ except Exception as exc:
471
+ say_error = exc
472
+
473
+ try:
474
+ engine = pyttsx3.init()
475
+ with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as temp_audio:
476
+ temp_path = temp_audio.name
477
+
478
+ try:
479
+ engine.save_to_file(normalized, temp_path)
480
+ engine.runAndWait()
481
+ data = Path(temp_path).read_bytes()
482
+ if not _is_valid_wav(data):
483
+ raise ValueError("Local TTS produced invalid/empty WAV.")
484
+ return data, ".wav"
485
+ finally:
486
+ temp_file = Path(temp_path)
487
+ if temp_file.exists():
488
+ temp_file.unlink()
489
+ except Exception as local_error:
490
+ raise ValueError(
491
+ f"Cloud TTS failed ({cloud_error}); transformers fallback failed ({transformers_error}); "
492
+ f"say fallback failed ({say_error}); local fallback failed ({local_error})."
493
+ )
494
+
495
+
496
+ def generate_podcast_audio(notebook_id: str, user_id: str, script: str) -> dict:
497
+ """Generate WAV audio from a podcast script and persist as artifact."""
498
+ audio_bytes, extension = _synthesize_tts_audio(script)
499
+
500
+ stamp = datetime.utcnow().strftime("%Y%m%d_%H%M%S")
501
+ filename = f"podcast_{stamp}{extension}"
502
+ storage_path = f"{get_artifacts_path(user_id, notebook_id)}/{filename}"
503
+
504
+ save_file(storage_path, audio_bytes)
505
+ artifact = create_artifact(notebook_id=notebook_id, type="podcast_audio", storage_path=storage_path)
506
+
507
+ local_cache_dir = Path("data") / "artifacts" / user_id / str(notebook_id)
508
+ local_cache_dir.mkdir(parents=True, exist_ok=True)
509
+ local_path = local_cache_dir / filename
510
+ local_path.write_bytes(audio_bytes)
511
+
512
+ return {
513
+ "artifact_id": artifact["id"] if artifact else None,
514
+ "storage_path": storage_path,
515
+ "audio_path": str(local_path),
516
+ }
backend/rag_service.py ADDED
@@ -0,0 +1,78 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """RAG chat service - retrieve chunks, call LLM, persist messages."""
2
+
3
+ import re
4
+
5
+ from backend.chat_service import save_message, load_chat
6
+ from backend.llm_client import DEFAULT_MODEL, get_llm_client
7
+ from backend.retrieval_service import retrieve_chunks
8
+
9
+ MAX_HISTORY_MESSAGES = 20
10
+ # Together AI - you have recent usage. Or :groq for Groq.
11
+ TOP_K = 5
12
+
13
+
14
+ def _validate_citations(text: str, num_chunks: int) -> str:
15
+ """Strip or fix citation numbers [N] where N > num_chunks."""
16
+ if num_chunks <= 0:
17
+ return text
18
+
19
+ def replace_citation(match):
20
+ n = int(match.group(1))
21
+ if 1 <= n <= num_chunks:
22
+ return match.group(0)
23
+ return ""
24
+
25
+ return re.sub(r"\[(\d+)\]", replace_citation, text)
26
+
27
+
28
+ def rag_chat(notebook_id: str, query: str, chat_history: list) -> tuple[str, list]:
29
+ """
30
+ RAG chat: retrieve chunks, build prompt, call LLM, persist, return answer and updated history.
31
+
32
+ chat_history: list of [user_msg, assistant_msg] pairs (Gradio Chatbot format).
33
+ Returns: (assistant_reply, updated_history).
34
+ """
35
+ save_message(notebook_id, "user", query)
36
+
37
+ chunks = retrieve_chunks(notebook_id, query, top_k=TOP_K)
38
+
39
+ context_parts = []
40
+ for i, c in enumerate(chunks, 1):
41
+ context_parts.append(f"[{i}] {c['content']}")
42
+ context = "\n\n".join(context_parts) if context_parts else "(No relevant sources found.)"
43
+
44
+ system_content = (
45
+ "You are a helpful assistant. Answer ONLY from the provided context. "
46
+ "Cite sources using [1], [2], etc. corresponding to the numbered passages. "
47
+ "If the answer is not in the context, say so clearly.\n\n"
48
+ f"Context:\n{context}"
49
+ )
50
+
51
+ # Truncate history to last MAX_HISTORY_MESSAGES (pairs -> 2*N messages)
52
+ max_pairs = MAX_HISTORY_MESSAGES // 2
53
+ truncated = chat_history[-max_pairs:] if len(chat_history) > max_pairs else chat_history
54
+
55
+ messages = [{"role": "system", "content": system_content}]
56
+ for user_msg, asst_msg in truncated:
57
+ if user_msg:
58
+ messages.append({"role": "user", "content": user_msg})
59
+ if asst_msg:
60
+ messages.append({"role": "assistant", "content": asst_msg})
61
+ messages.append({"role": "user", "content": query})
62
+
63
+ try:
64
+ client = get_llm_client()
65
+ response = client.chat.completions.create(
66
+ model=DEFAULT_MODEL,
67
+ messages=messages,
68
+ max_tokens=512,
69
+ )
70
+ raw_answer = response.choices[0].message.content or ""
71
+ answer = _validate_citations(raw_answer, len(chunks))
72
+ except Exception as e:
73
+ answer = f"Error calling model: {e}"
74
+
75
+ save_message(notebook_id, "assistant", answer)
76
+
77
+ updated_history = chat_history + [[query, answer]]
78
+ return answer, updated_history
backend/report_service.py ADDED
@@ -0,0 +1,81 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Report generation helper that prompts the LLM with retrieved chunks."""
2
+
3
+ from typing import Final
4
+
5
+ from backend.llm_client import DEFAULT_MODEL, get_llm_client
6
+ from backend.retrieval_service import retrieve_chunks
7
+
8
+ MAX_CONTEXT_ITEMS: Final[int] = 8
9
+ REPORT_SCOPE_DESCRIPTIONS = {
10
+ "all": "PDFs, websites, and uploaded text",
11
+ "pdf": "PDF uploads",
12
+ "url": "ingested websites",
13
+ "text": "uploaded text files",
14
+ }
15
+
16
+
17
+ def _is_pdf_metadata(metadata: dict) -> bool:
18
+ name = (metadata.get("file_name") or "").lower()
19
+ path = (metadata.get("file_path") or "").lower()
20
+ if name.endswith(".pdf") or path.endswith(".pdf"):
21
+ return True
22
+ return False
23
+
24
+
25
+ def _is_text_metadata(metadata: dict) -> bool:
26
+ name = (metadata.get("file_name") or "").lower()
27
+ return name.endswith(".txt")
28
+
29
+
30
+ def _matches_scope(chunk: dict, scope: str) -> bool:
31
+ if scope == "all":
32
+ return True
33
+ metadata = chunk.get("metadata") or {}
34
+ if scope == "pdf":
35
+ return _is_pdf_metadata(metadata)
36
+ if scope == "text":
37
+ return _is_text_metadata(metadata)
38
+ if scope == "url":
39
+ return bool(metadata.get("url"))
40
+ return True
41
+
42
+
43
+ def generate_report(notebook_id: str, scope: str = "all") -> str:
44
+ """Return a report string using the provided notebook sources."""
45
+ normalized_scope = scope if scope in REPORT_SCOPE_DESCRIPTIONS else "all"
46
+ desc = REPORT_SCOPE_DESCRIPTIONS[normalized_scope]
47
+ chunks = retrieve_chunks(notebook_id, f"report summary for {desc}", top_k=16)
48
+ filtered = [chunk for chunk in chunks if _matches_scope(chunk, normalized_scope)]
49
+ if not filtered:
50
+ raise ValueError(f"No {desc} chunks are available yet.")
51
+
52
+ context_items = filtered[:MAX_CONTEXT_ITEMS]
53
+ context_lines = [f"[{idx}] {item['content']}" for idx, item in enumerate(context_items, start=1)]
54
+ context_block = "\n\n".join(context_lines)
55
+
56
+ system_content = (
57
+ f"You are NotebookLM's assistant. Using the numbered context items below that come from {desc},"
58
+ "write a concise report that sticks to the facts."
59
+ " Cite every referenced statement with the matching [n]."
60
+ f"\n\nContext:\n{context_block}"
61
+ )
62
+
63
+ user_message = (
64
+ "Create a report with at least the sections Summary, Key Findings, and Recommendations."
65
+ " Bullet points are fine."
66
+ )
67
+
68
+ client = get_llm_client()
69
+ response = client.chat.completions.create(
70
+ model=DEFAULT_MODEL,
71
+ messages=[
72
+ {"role": "system", "content": system_content},
73
+ {"role": "user", "content": user_message},
74
+ ],
75
+ max_tokens=700,
76
+ )
77
+
78
+ report_text = response.choices[0].message.content.strip() if response.choices else ""
79
+ if not report_text:
80
+ raise ValueError("The report generator returned an empty response.")
81
+ return report_text
backend/retrieval_service.py ADDED
@@ -0,0 +1,39 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Retrieval service - vector similarity search for RAG."""
2
+
3
+ from backend.db import supabase
4
+ from backend.embedding_service import encode
5
+
6
+
7
+ def retrieve_chunks(notebook_id: str, query: str, top_k: int = 5) -> list[dict]:
8
+ """
9
+ Retrieve top-k chunks for a query, filtered by notebook_id.
10
+
11
+ Returns list of dicts with keys: id, content, metadata, similarity.
12
+ """
13
+ if not query or not query.strip():
14
+ return []
15
+
16
+ query_embedding = encode([query.strip()], task="search_query")[0]
17
+
18
+ try:
19
+ result = supabase.rpc(
20
+ "match_chunks",
21
+ {
22
+ "query_embedding": query_embedding,
23
+ "match_count": top_k,
24
+ "p_notebook_id": notebook_id,
25
+ },
26
+ ).execute()
27
+
28
+ rows = result.data or []
29
+ return [
30
+ {
31
+ "id": str(r["id"]),
32
+ "content": r["content"],
33
+ "metadata": r.get("metadata") or {},
34
+ "similarity": float(r.get("similarity", 0)),
35
+ }
36
+ for r in rows
37
+ ]
38
+ except Exception:
39
+ return []
db/migrate_to_384.sql ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ -- Migration: Switch from 1536-dim to 384-dim embeddings (MiniLM)
2
+ -- Run this in Supabase SQL Editor if you already have the chunks table with vector(1536)
3
+
4
+ -- 1. Drop the ivfflat index (required before altering column)
5
+ drop index if exists idx_chunks_embedding;
6
+
7
+ -- 2. Clear existing chunks (old 1536-dim embeddings are incompatible)
8
+ truncate table chunks;
9
+
10
+ -- 3. Replace embedding column with 384-dim version
11
+ alter table chunks drop column embedding;
12
+ alter table chunks add column embedding vector(384);
13
+
14
+ -- 4. Recreate the ivfflat index (run AFTER ingesting new PDF/TXT - requires rows)
15
+ -- create index if not exists idx_chunks_embedding on chunks using ivfflat (embedding vector_cosine_ops) with (lists = 100);
16
+
17
+ -- 5. Update match_chunks RPC
18
+ create or replace function match_chunks(
19
+ query_embedding vector(384),
20
+ match_count int,
21
+ p_notebook_id uuid
22
+ )
23
+ returns table (id uuid, content text, metadata jsonb, similarity float)
24
+ language plpgsql as $$
25
+ begin
26
+ return query
27
+ select c.id, c.content, c.metadata,
28
+ 1 - (c.embedding <=> query_embedding) as similarity
29
+ from chunks c
30
+ where c.notebook_id = p_notebook_id
31
+ and c.embedding is not null
32
+ order by c.embedding <=> query_embedding
33
+ limit match_count;
34
+ end;
35
+ $$;
db/schema.sql CHANGED
@@ -33,19 +33,40 @@ create index if not exists idx_artifacts_notebook_id on artifacts(notebook_id);
33
  -- pgvector extension for embeddings
34
  create extension if not exists vector;
35
 
36
- -- chunks with embeddings (for RAG)
37
  create table if not exists chunks (
38
  id uuid primary key default gen_random_uuid(),
39
  notebook_id uuid not null references notebooks(id) on delete cascade,
40
  source_id text,
41
  content text not null,
42
- embedding vector(1536),
43
  metadata jsonb,
44
  created_at timestamptz default now()
45
  );
46
  create index if not exists idx_chunks_notebook_id on chunks(notebook_id);
47
- -- Vector index (run after you have data; ivfflat requires rows):
48
- -- create index idx_chunks_embedding on chunks using ivfflat (embedding vector_cosine_ops) with (lists = 100);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
49
 
50
  -- sources table (ingestion pipeline)
51
  create table if not exists sources (
 
33
  -- pgvector extension for embeddings
34
  create extension if not exists vector;
35
 
36
+ -- chunks with embeddings (for RAG) - 384 dims for MiniLM
37
  create table if not exists chunks (
38
  id uuid primary key default gen_random_uuid(),
39
  notebook_id uuid not null references notebooks(id) on delete cascade,
40
  source_id text,
41
  content text not null,
42
+ embedding vector(384),
43
  metadata jsonb,
44
  created_at timestamptz default now()
45
  );
46
  create index if not exists idx_chunks_notebook_id on chunks(notebook_id);
47
+
48
+ -- Vector index for fast similarity search (run after chunks have data; ivfflat requires rows)
49
+ create index if not exists idx_chunks_embedding on chunks using ivfflat (embedding vector_cosine_ops) with (lists = 100);
50
+
51
+ -- RPC for RAG retrieval: top-k chunks by cosine similarity, filtered by notebook_id
52
+ create or replace function match_chunks(
53
+ query_embedding vector(384),
54
+ match_count int,
55
+ p_notebook_id uuid
56
+ )
57
+ returns table (id uuid, content text, metadata jsonb, similarity float)
58
+ language plpgsql as $$
59
+ begin
60
+ return query
61
+ select c.id, c.content, c.metadata,
62
+ 1 - (c.embedding <=> query_embedding) as similarity
63
+ from chunks c
64
+ where c.notebook_id = p_notebook_id
65
+ and c.embedding is not null
66
+ order by c.embedding <=> query_embedding
67
+ limit match_count;
68
+ end;
69
+ $$;
70
 
71
  -- sources table (ingestion pipeline)
72
  create table if not exists sources (
requirements.txt CHANGED
@@ -1,9 +1,15 @@
1
  gradio[oauth]==4.44.1
2
  huggingface_hub==0.24.7
 
3
  supabase>=2.0.0
4
  python-dotenv>=1.0.0
5
  realtime==2.3.0
6
  chardet>=5.0.0
7
  sentence-transformers>=2.0.0
 
8
  pypdf>=4.2.0
9
  beautifulsoup4>=4.12.3
 
 
 
 
 
1
  gradio[oauth]==4.44.1
2
  huggingface_hub==0.24.7
3
+ openai>=1.0.0
4
  supabase>=2.0.0
5
  python-dotenv>=1.0.0
6
  realtime==2.3.0
7
  chardet>=5.0.0
8
  sentence-transformers>=2.0.0
9
+ einops>=0.7.0
10
  pypdf>=4.2.0
11
  beautifulsoup4>=4.12.3
12
+ pyttsx3>=2.90
13
+ transformers>=4.45.0
14
+ torch>=2.2.0
15
+ sentencepiece>=0.2.0
run.bat ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ @echo off
2
+ REM Gradio has issues with Python 3.13 - use 3.10, 3.11, or 3.12
3
+ echo Checking for Python 3.10/3.11/3.12...
4
+ py -3.10 --version 2>nul && goto run310
5
+ py -3.11 --version 2>nul && goto run311
6
+ py -3.12 --version 2>nul && goto run312
7
+ echo.
8
+ echo Python 3.10, 3.11, or 3.12 not found.
9
+ echo Gradio does NOT work with Python 3.13.
10
+ echo Install Python 3.10 from https://www.python.org/downloads/
11
+ pause
12
+ exit /b 1
13
+
14
+ :run310
15
+ echo Using Python 3.10
16
+ py -3.10 -m pip install -r requirements.txt -q
17
+ py -3.10 app.py
18
+ goto end
19
+
20
+ :run311
21
+ echo Using Python 3.11
22
+ py -3.11 -m pip install -r requirements.txt -q
23
+ py -3.11 app.py
24
+ goto end
25
+
26
+ :run312
27
+ echo Using Python 3.12
28
+ py -3.12 -m pip install -r requirements.txt -q
29
+ py -3.12 app.py
30
+ goto end
31
+
32
+ :end
33
+ pause