Malina Claude Opus 4.8 commited on
Commit
281f5b1
·
1 Parent(s): b67e3ef

LUNA v0.1 — multi-file task → structured Note (map-reduce backend)

Browse files

Drop ≤100 files of any type, give a prompt, get one structured Note
(Title · Agenda · Body · Sources). Parallel two-tier map-reduce over
HF Inference Providers; faster-whisper for audio/video, VLM for images.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Files changed (11) hide show
  1. .gitignore +5 -0
  2. README.md +29 -7
  3. app.py +49 -63
  4. auth.py +53 -0
  5. config.py +53 -0
  6. extract.py +215 -0
  7. models.py +51 -0
  8. packages.txt +1 -0
  9. pipeline.py +139 -0
  10. requirements.txt +22 -0
  11. synthesis.py +245 -0
.gitignore ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ __pycache__/
2
+ *.pyc
3
+ .DS_Store
4
+ .venv/
5
+ venv/
README.md CHANGED
@@ -1,17 +1,39 @@
1
  ---
2
  title: LUNA
3
- emoji: 💬
4
  colorFrom: yellow
5
  colorTo: purple
6
  sdk: gradio
7
- sdk_version: 6.5.1
8
  app_file: app.py
9
  pinned: false
10
- hf_oauth: true
11
- hf_oauth_scopes:
12
- - inference-api
13
  license: apache-2.0
14
- short_description: Latent understanding and narrative assembly
15
  ---
16
 
17
- An example chatbot using [Gradio](https://gradio.app), [`huggingface_hub`](https://huggingface.co/docs/huggingface_hub/v0.22.2/en/index), and the [Hugging Face Inference API](https://huggingface.co/docs/api-inference/index).
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  ---
2
  title: LUNA
3
+ emoji: 🌙
4
  colorFrom: yellow
5
  colorTo: purple
6
  sdk: gradio
7
+ sdk_version: 6.14.0
8
  app_file: app.py
9
  pinned: false
 
 
 
10
  license: apache-2.0
11
+ short_description: Latent understanding & narrative assembly from your files
12
  ---
13
 
14
+ # LUNA — Latent Understanding & Narrative Assembly
15
+
16
+ LUNA is the file-synthesis tool of [Next Neural Labs](https://nnlabs.pl). Drop up to
17
+ **100 files of any type** (documents, spreadsheets, code, images, audio, video) into a
18
+ **task**, tell LUNA *what to build* (e.g. *"Everything about climate change in Italy"*),
19
+ and it returns **one structured Note** — Title, Agenda, themed Body, and a Sources
20
+ section that credits every file it actually used.
21
+
22
+ ## How it works
23
+
24
+ 1. **Categorize** — each upload is sorted by type (text / document / image / audio /
25
+ video / spreadsheet / code / …).
26
+ 2. **Extract in parallel** — content is pulled from every file concurrently to keep cost
27
+ and latency low: documents/sheets/code are parsed to text, images are read by a
28
+ vision-language model (description + on-screen text), audio/video are transcribed with
29
+ Whisper.
30
+ 3. **Map (relevance)** — a fast model judges each file against your prompt and pulls only
31
+ the on-topic passages. A file may feed several themes, or none — irrelevant or
32
+ unreadable files are recognized and set aside.
33
+ 4. **Reduce (synthesis)** — a strong model fuses the kept material into the final Note,
34
+ in the language of your prompt, ending with Sources (used files) and a short *Not used*
35
+ line for what it skipped.
36
+
37
+ The front-end lives at **https://app.nnlabs.pl/luna**; this Space exposes the `build` API
38
+ the website drives. Uploaded files are processed and then purged — only the generated Note
39
+ is stored (in the website's database).
app.py CHANGED
@@ -1,69 +1,55 @@
1
- import gradio as gr
2
- from huggingface_hub import InferenceClient
3
-
4
-
5
- def respond(
6
- message,
7
- history: list[dict[str, str]],
8
- system_message,
9
- max_tokens,
10
- temperature,
11
- top_p,
12
- hf_token: gr.OAuthToken,
13
- ):
14
- """
15
- For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
16
- """
17
- client = InferenceClient(token=hf_token.token, model="openai/gpt-oss-20b")
18
-
19
- messages = [{"role": "system", "content": system_message}]
20
-
21
- messages.extend(history)
22
-
23
- messages.append({"role": "user", "content": message})
24
-
25
- response = ""
26
-
27
- for message in client.chat_completion(
28
- messages,
29
- max_tokens=max_tokens,
30
- stream=True,
31
- temperature=temperature,
32
- top_p=top_p,
33
- ):
34
- choices = message.choices
35
- token = ""
36
- if len(choices) and choices[0].delta.content:
37
- token = choices[0].delta.content
38
-
39
- response += token
40
- yield response
41
 
 
42
 
43
- """
44
- For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
45
- """
46
- chatbot = gr.ChatInterface(
47
- respond,
48
- additional_inputs=[
49
- gr.Textbox(value="You are a friendly Chatbot.", label="System message"),
50
- gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
51
- gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
52
- gr.Slider(
53
- minimum=0.1,
54
- maximum=1.0,
55
- value=0.95,
56
- step=0.05,
57
- label="Top-p (nucleus sampling)",
58
- ),
59
- ],
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
60
  )
61
 
62
- with gr.Blocks() as demo:
63
- with gr.Sidebar():
64
- gr.LoginButton()
65
- chatbot.render()
66
-
67
-
68
  if __name__ == "__main__":
69
  demo.launch()
 
1
+ # ─────────────────────────────────────────────
2
+ # LUNA — Latent Understanding & Narrative Assembly
3
+ # app.py | v0.1 | Made by Malina
4
+ # ─────────────────────────────────────────────
5
+ import logging
6
+ import os
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7
 
8
+ import gradio as gr
9
 
10
+ from pipeline import run
11
+
12
+
13
+ # Quiet benign upload aborts (a browser cancelling a multipart upload mid-stream
14
+ # floods the Space logs with harmless ClientDisconnect tracebacks). Same filter NINA
15
+ # uses — drop only those records.
16
+ class _DropClientDisconnect(logging.Filter):
17
+ def filter(self, record: logging.LogRecord) -> bool:
18
+ exc = record.exc_info[1] if record.exc_info else None
19
+ if exc is not None and type(exc).__name__ == "ClientDisconnect":
20
+ return False
21
+ return "ClientDisconnect" not in record.getMessage()
22
+
23
+
24
+ for _name in ("uvicorn.error", "uvicorn"):
25
+ logging.getLogger(_name).addFilter(_DropClientDisconnect())
26
+
27
+
28
+ # The custom front-end at app.nnlabs.pl drives the `build` endpoint directly (the
29
+ # Gradio UI below is just for direct/testing access). Privacy: purge uploaded files
30
+ # every 15 min so they are processed, not stored.
31
+ with gr.Blocks(title="LUNA — File Synthesis", delete_cache=(900, 900)) as demo:
32
+ gr.Markdown(
33
+ "## LUNA — Latent Understanding & Narrative Assembly\n"
34
+ "API for the app at **https://app.nnlabs.pl/luna**. Drop up to 100 files, say "
35
+ "what to build, and LUNA returns one structured Note. Endpoint: `/build`."
36
+ )
37
+
38
+ with gr.Tab("Build"):
39
+ files = gr.File(label="Files (up to 100, any type)", file_count="multiple")
40
+ prompt = gr.Textbox(label="What should LUNA build?",
41
+ placeholder="e.g. Everything about climate change in Italy")
42
+ token = gr.Textbox(label="access token (set by the website)", value="")
43
+ status = gr.Textbox(label="status", lines=1)
44
+ note = gr.Markdown(label="Note")
45
+ gr.Button("Build").click(
46
+ run, inputs=[files, prompt, token], outputs=[status, note], api_name="build"
47
+ )
48
+
49
+ demo.queue(
50
+ default_concurrency_limit=int(os.getenv("LUNA_CONCURRENCY", "4")),
51
+ max_size=int(os.getenv("LUNA_QUEUE_MAX", "24")),
52
  )
53
 
 
 
 
 
 
 
54
  if __name__ == "__main__":
55
  demo.launch()
auth.py ADDED
@@ -0,0 +1,53 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Per-use signed-token verification — identical scheme to NINA's auth.py. The
2
+ # Cloudflare worker mints an HMAC-SHA256 token per task; LUNA verifies it with the
3
+ # shared SIGNING_SECRET. If no secret is configured the Space is open (the website's
4
+ # login + 3-tasks/day quota is the gate). Single-use: a jti can't be replayed.
5
+ import base64
6
+ import hashlib
7
+ import hmac
8
+ import json
9
+ import threading
10
+ import time
11
+
12
+ from config import SIGNING_SECRET
13
+
14
+ _used_jti: dict = {}
15
+ _jti_lock = threading.Lock()
16
+
17
+
18
+ def _b64url_decode(s: str) -> bytes:
19
+ return base64.urlsafe_b64decode(s + "=" * (-len(s) % 4))
20
+
21
+
22
+ def _b64url_encode(b: bytes) -> str:
23
+ return base64.urlsafe_b64encode(b).decode().rstrip("=")
24
+
25
+
26
+ def verify_usage_token(token: str) -> bool:
27
+ if not SIGNING_SECRET:
28
+ return True
29
+ if not token or "." not in token:
30
+ return False
31
+ try:
32
+ msg, sig = token.split(".", 1)
33
+ expected = _b64url_encode(
34
+ hmac.new(SIGNING_SECRET.encode(), msg.encode(), hashlib.sha256).digest()
35
+ )
36
+ if not hmac.compare_digest(expected, sig):
37
+ return False
38
+ payload = json.loads(_b64url_decode(msg))
39
+ now = int(time.time())
40
+ if int(payload.get("e", 0)) < now:
41
+ return False
42
+ jti = payload.get("j", "")
43
+ if not jti:
44
+ return False
45
+ with _jti_lock:
46
+ for k in [k for k, e in _used_jti.items() if e < now]:
47
+ _used_jti.pop(k, None)
48
+ if jti in _used_jti:
49
+ return False
50
+ _used_jti[jti] = int(payload.get("e", now + 120))
51
+ return True
52
+ except Exception:
53
+ return False
config.py ADDED
@@ -0,0 +1,53 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # ─────────────────────────────────────────────
2
+ # LUNA — Latent Understanding & Narrative Assembly
3
+ # config.py
4
+ # ─────────────────────────────────────────────
5
+ import os
6
+ import tempfile
7
+
8
+ # ── Inference models (HF Inference Providers) ─────────────────────────────────
9
+ # Two-tier map-reduce keeps cost down: a cheap/fast model reads every file and
10
+ # pulls only the prompt-relevant material (MAP), then one strong-model call fuses
11
+ # the kept extracts into the final Note (REDUCE). Images are read by a VLM.
12
+ MAP_MODEL = os.getenv("LUNA_MAP_MODEL", "Qwen/Qwen2.5-7B-Instruct")
13
+ REDUCE_MODEL = os.getenv("LUNA_REDUCE_MODEL", "Qwen/Qwen2.5-72B-Instruct")
14
+ VLM_MODEL = os.getenv("LUNA_VLM_MODEL", "Qwen/Qwen3-VL-30B-A3B-Instruct")
15
+
16
+ MAP_MAX_TOKENS = 900 # per-file relevance + extraction (cheap model)
17
+ REDUCE_MAX_TOKENS = 4000 # the final Note can be long
18
+ MAP_TEMPERATURE = 0.0
19
+ REDUCE_TEMPERATURE = 0.15
20
+ LLM_TIMEOUT = 180
21
+
22
+ VLM_FRAME_MAX_PX = 1024 # downscale an image's long side before sending (caps tokens)
23
+
24
+ # ── Upload limits (re-checked here; the website also enforces them) ───────────
25
+ MAX_FILES = 100
26
+ MAX_FILE_MB = 25 # per non-media file
27
+ MAX_MEDIA_MB = 200 # per audio/video file
28
+ MAX_TOTAL_MB = 500 # whole task
29
+
30
+ # ── Extraction bounds (token/cost control) ────────────────────────────────────
31
+ PER_FILE_CHARS = 16000 # cap raw text pulled from one file before MAP
32
+ MAP_NOTE_CHARS = 4000 # cap the notes MAP keeps from one file
33
+ REDUCE_BUDGET = 60000 # total chars of kept notes fed to REDUCE
34
+ TRANSCRIBE_MAX_SECONDS = 1800 # don't transcribe media longer than 30 min
35
+
36
+ # ── Parallelism ───────────────────────────────────────────────────────────────
37
+ # Extraction + MAP run in a thread pool. LLM/VLM calls are IO-bound, so high
38
+ # concurrency cuts wall-time at no extra token cost. Whisper is CPU-bound, so it's
39
+ # throttled separately to avoid thrashing the Space's cores.
40
+ MAP_WORKERS = int(os.getenv("LUNA_MAP_WORKERS", "8"))
41
+ TRANSCRIBE_WORKERS = int(os.getenv("LUNA_TRANSCRIBE_WORKERS", "2"))
42
+
43
+ # ── Whisper ─────────────────────────────────────────────────────────────────
44
+ WHISPER_MODEL = os.getenv("LUNA_WHISPER_MODEL", "base")
45
+ WHISPER_CPU_THREADS = int(os.getenv("WHISPER_CPU_THREADS", "4"))
46
+
47
+ # ── Misc ───────────────────────────────────────────────────────────────────
48
+ CACHE_DIR = os.path.join(tempfile.gettempdir(), "luna_cache")
49
+ DEBUG = bool(os.getenv("LUNA_DEBUG", ""))
50
+
51
+ # Shared with the Cloudflare worker that mints per-use tokens. If unset, the Space
52
+ # accepts any token (the website's login + 3/day quota is then the only gate).
53
+ SIGNING_SECRET = os.getenv("LUNA_SIGNING_SECRET", os.getenv("NINA_SIGNING_SECRET", ""))
extract.py ADDED
@@ -0,0 +1,215 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # File categorization + content extraction. Each file resolves to one of:
2
+ # kind="text" → payload is extracted plain text (docs, sheets, code, data, html)
3
+ # kind="image" → payload is a base64 data URL (read later by the VLM)
4
+ # kind="media" → payload is a filesystem path (transcribed later by Whisper)
5
+ # kind="error" → payload is a short human reason it couldn't be read
6
+ # Everything here is best-effort and never raises: a single bad file shouldn't sink
7
+ # the whole task.
8
+ import base64
9
+ import io
10
+ import os
11
+ import zipfile
12
+
13
+ from config import PER_FILE_CHARS
14
+
15
+ # ── Extension → category map (category is also what the UI shows) ──────────────
16
+ _EXT = {
17
+ # documents
18
+ "pdf": "document", "docx": "document", "doc": "document", "pptx": "document",
19
+ "ppt": "document", "rtf": "document", "odt": "document", "epub": "document",
20
+ "html": "document", "htm": "document",
21
+ # spreadsheets
22
+ "xlsx": "spreadsheet", "xls": "spreadsheet", "ods": "spreadsheet",
23
+ # plain text / data
24
+ "txt": "text", "text": "text", "log": "text", "md": "text", "markdown": "text",
25
+ "csv": "data", "tsv": "data", "json": "data", "yaml": "data", "yml": "data",
26
+ "xml": "data", "ini": "data", "toml": "data",
27
+ # code
28
+ "py": "code", "js": "code", "ts": "code", "jsx": "code", "tsx": "code",
29
+ "java": "code", "c": "code", "h": "code", "cpp": "code", "cc": "code",
30
+ "cs": "code", "go": "code", "rs": "code", "rb": "code", "php": "code",
31
+ "swift": "code", "kt": "code", "sh": "code", "bash": "code", "sql": "code",
32
+ "css": "code", "scss": "code", "r": "code", "m": "code", "lua": "code",
33
+ # images
34
+ "png": "image", "jpg": "image", "jpeg": "image", "gif": "image",
35
+ "webp": "image", "bmp": "image", "tif": "image", "tiff": "image",
36
+ # audio / video
37
+ "mp3": "audio", "wav": "audio", "m4a": "audio", "flac": "audio",
38
+ "ogg": "audio", "aac": "audio", "opus": "audio", "wma": "audio",
39
+ "mp4": "video", "mov": "video", "mkv": "video", "webm": "video",
40
+ "avi": "video", "m4v": "video", "wmv": "video", "flv": "video",
41
+ # archives
42
+ "zip": "archive",
43
+ }
44
+
45
+ _TEXTLIKE = {"text", "data", "code"}
46
+
47
+
48
+ def category_of(name: str) -> str:
49
+ ext = name.rsplit(".", 1)[-1].lower() if "." in name else ""
50
+ return _EXT.get(ext, "other")
51
+
52
+
53
+ def _truncate(s: str) -> str:
54
+ s = (s or "").strip()
55
+ return s[:PER_FILE_CHARS] if len(s) > PER_FILE_CHARS else s
56
+
57
+
58
+ def _read_textfile(path: str) -> str:
59
+ with open(path, "r", encoding="utf-8", errors="replace") as f:
60
+ return f.read(PER_FILE_CHARS + 1)
61
+
62
+
63
+ def _read_pdf(path: str) -> str:
64
+ from pypdf import PdfReader
65
+ reader = PdfReader(path)
66
+ out, total = [], 0
67
+ for page in reader.pages:
68
+ t = page.extract_text() or ""
69
+ out.append(t)
70
+ total += len(t)
71
+ if total > PER_FILE_CHARS:
72
+ break
73
+ return "\n".join(out)
74
+
75
+
76
+ def _read_docx(path: str) -> str:
77
+ import docx
78
+ d = docx.Document(path)
79
+ parts = [p.text for p in d.paragraphs if p.text.strip()]
80
+ for tbl in d.tables:
81
+ for row in tbl.rows:
82
+ cells = [c.text.strip() for c in row.cells]
83
+ if any(cells):
84
+ parts.append(" | ".join(cells))
85
+ return "\n".join(parts)
86
+
87
+
88
+ def _read_pptx(path: str) -> str:
89
+ from pptx import Presentation
90
+ prs = Presentation(path)
91
+ parts = []
92
+ for i, slide in enumerate(prs.slides, 1):
93
+ parts.append(f"[Slide {i}]")
94
+ for shape in slide.shapes:
95
+ if shape.has_text_frame and shape.text_frame.text.strip():
96
+ parts.append(shape.text_frame.text)
97
+ return "\n".join(parts)
98
+
99
+
100
+ def _read_xlsx(path: str) -> str:
101
+ from openpyxl import load_workbook
102
+ wb = load_workbook(path, read_only=True, data_only=True)
103
+ parts, total = [], 0
104
+ for ws in wb.worksheets:
105
+ parts.append(f"[Sheet: {ws.title}]")
106
+ for row in ws.iter_rows(values_only=True):
107
+ cells = [str(c) for c in row if c is not None]
108
+ if cells:
109
+ line = " | ".join(cells)
110
+ parts.append(line)
111
+ total += len(line)
112
+ if total > PER_FILE_CHARS:
113
+ break
114
+ if total > PER_FILE_CHARS:
115
+ break
116
+ wb.close()
117
+ return "\n".join(parts)
118
+
119
+
120
+ def _read_html(path: str) -> str:
121
+ from bs4 import BeautifulSoup
122
+ with open(path, "r", encoding="utf-8", errors="replace") as f:
123
+ soup = BeautifulSoup(f.read(), "html.parser")
124
+ for tag in soup(["script", "style", "noscript"]):
125
+ tag.decompose()
126
+ return soup.get_text(separator="\n")
127
+
128
+
129
+ def _read_zip(path: str) -> str:
130
+ parts, total = [], 0
131
+ with zipfile.ZipFile(path) as z:
132
+ for info in z.infolist():
133
+ if info.is_dir() or info.file_size == 0:
134
+ continue
135
+ member = info.filename
136
+ cat = category_of(member)
137
+ if cat not in _TEXTLIKE and cat not in ("document",):
138
+ continue # skip media/images/unknown inside archives (best-effort)
139
+ try:
140
+ with z.open(info) as fh:
141
+ raw = fh.read(PER_FILE_CHARS)
142
+ text = raw.decode("utf-8", errors="replace")
143
+ except Exception:
144
+ continue
145
+ parts.append(f"\n=== {member} ===\n{text}")
146
+ total += len(text)
147
+ if total > PER_FILE_CHARS:
148
+ break
149
+ return "\n".join(parts)
150
+
151
+
152
+ def _image_data_url(path: str) -> str:
153
+ from PIL import Image
154
+ from config import VLM_FRAME_MAX_PX
155
+ img = Image.open(path)
156
+ img = img.convert("RGB")
157
+ w, h = img.size
158
+ scale = VLM_FRAME_MAX_PX / max(w, h)
159
+ if scale < 1:
160
+ img = img.resize((max(1, int(w * scale)), max(1, int(h * scale))))
161
+ buf = io.BytesIO()
162
+ img.save(buf, format="JPEG", quality=85)
163
+ b64 = base64.b64encode(buf.getvalue()).decode()
164
+ return f"data:image/jpeg;base64,{b64}"
165
+
166
+
167
+ # Dispatch by category → (kind, extractor)
168
+ _TEXT_EXTRACTORS = {
169
+ "document": None, # resolved below per-extension
170
+ "spreadsheet": _read_xlsx,
171
+ "text": _read_textfile,
172
+ "data": _read_textfile,
173
+ "code": _read_textfile,
174
+ "archive": _read_zip,
175
+ }
176
+
177
+
178
+ def extract(path: str, name: str | None = None) -> dict:
179
+ """Extract one file. Returns {name, category, kind, payload}."""
180
+ name = name or os.path.basename(path)
181
+ cat = category_of(name)
182
+ ext = name.rsplit(".", 1)[-1].lower() if "." in name else ""
183
+
184
+ try:
185
+ if cat == "image":
186
+ return {"name": name, "category": cat, "kind": "image",
187
+ "payload": _image_data_url(path)}
188
+ if cat in ("audio", "video"):
189
+ return {"name": name, "category": cat, "kind": "media", "payload": path}
190
+
191
+ if cat == "document":
192
+ if ext == "pdf":
193
+ text = _read_pdf(path)
194
+ elif ext in ("docx",):
195
+ text = _read_docx(path)
196
+ elif ext in ("pptx",):
197
+ text = _read_pptx(path)
198
+ elif ext in ("html", "htm"):
199
+ text = _read_html(path)
200
+ else:
201
+ # .doc/.ppt/.rtf/.odt/.epub — fall back to a raw text read (best-effort)
202
+ text = _read_textfile(path)
203
+ elif cat in _TEXT_EXTRACTORS:
204
+ text = _TEXT_EXTRACTORS[cat](path)
205
+ else: # "other" — try reading it as text; many unknowns are plain text
206
+ text = _read_textfile(path)
207
+
208
+ text = _truncate(text)
209
+ if not text.strip():
210
+ return {"name": name, "category": cat, "kind": "error",
211
+ "payload": "no readable text found"}
212
+ return {"name": name, "category": cat, "kind": "text", "payload": text}
213
+ except Exception as e:
214
+ return {"name": name, "category": cat, "kind": "error",
215
+ "payload": f"couldn't read ({type(e).__name__})"}
models.py ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Model clients: one HF InferenceClient for all LLM/VLM calls, and a lazily-loaded
2
+ # faster-whisper model for audio/video transcription. No torch — faster-whisper uses
3
+ # the ctranslate2 backend and PyAV for decoding.
4
+ import os
5
+ import threading
6
+
7
+ from huggingface_hub import InferenceClient
8
+
9
+ from config import LLM_TIMEOUT, WHISPER_CPU_THREADS, WHISPER_MODEL
10
+
11
+ HF_TOKEN = os.getenv("HF_TOKEN")
12
+ if not HF_TOKEN:
13
+ _token_file = os.path.expanduser("~/.cache/huggingface/token")
14
+ if os.path.exists(_token_file):
15
+ HF_TOKEN = open(_token_file).read().strip()
16
+
17
+ if HF_TOKEN:
18
+ print("[LUNA] HF_TOKEN loaded — inference enabled.")
19
+ else:
20
+ print("[LUNA] HF_TOKEN not set — LLM/VLM calls will fail until it is.")
21
+
22
+ llm_client = InferenceClient(token=HF_TOKEN, timeout=LLM_TIMEOUT)
23
+
24
+
25
+ # ── Lazy Whisper (faster-whisper, CPU int8) ───────────────────────────────────
26
+ _whisper = None
27
+ _whisper_lock = threading.Lock()
28
+ _whisper_failed = False
29
+
30
+
31
+ def get_whisper():
32
+ """Lazy-load the Whisper model. Returns None (never raises) if it can't load, so
33
+ a task with media degrades gracefully instead of crashing."""
34
+ global _whisper, _whisper_failed
35
+ with _whisper_lock:
36
+ if _whisper_failed:
37
+ return None
38
+ if _whisper is None:
39
+ try:
40
+ from faster_whisper import WhisperModel
41
+ print(f"[LUNA] Loading Whisper {WHISPER_MODEL} (faster-whisper, CPU int8)...")
42
+ _whisper = WhisperModel(
43
+ WHISPER_MODEL, device="cpu", compute_type="int8",
44
+ cpu_threads=WHISPER_CPU_THREADS,
45
+ )
46
+ print(f"[LUNA] Whisper {WHISPER_MODEL} ready.")
47
+ except Exception as e:
48
+ _whisper_failed = True
49
+ print(f"[LUNA] Whisper failed to load: {e} — audio/video skipped.")
50
+ return None
51
+ return _whisper
packages.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ ffmpeg
pipeline.py ADDED
@@ -0,0 +1,139 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Orchestration: categorize → extract+MAP every file in parallel → REDUCE into the
2
+ # final Note. Exposed as a streaming generator that yields (status, note) for Gradio;
3
+ # the website's worker forwards `status` as a progress marker and accumulates `note`.
4
+ import os
5
+ import threading
6
+ from concurrent.futures import ThreadPoolExecutor, as_completed
7
+
8
+ from auth import verify_usage_token
9
+ from config import (
10
+ MAP_NOTE_CHARS, MAP_WORKERS, MAX_FILES, MAX_FILE_MB, MAX_MEDIA_MB, MAX_TOTAL_MB,
11
+ TRANSCRIBE_MAX_SECONDS, TRANSCRIBE_WORKERS,
12
+ )
13
+ from extract import category_of, extract
14
+ from models import get_whisper
15
+ from synthesis import map_image, map_text, reduce_stream
16
+
17
+ _transcribe_sem = threading.Semaphore(TRANSCRIBE_WORKERS)
18
+
19
+
20
+ def _transcribe(path: str) -> str:
21
+ model = get_whisper()
22
+ if model is None:
23
+ return ""
24
+ with _transcribe_sem:
25
+ try:
26
+ segments, _info = model.transcribe(path, beam_size=1, vad_filter=True)
27
+ parts = []
28
+ for seg in segments:
29
+ parts.append(seg.text)
30
+ if seg.end and seg.end > TRANSCRIBE_MAX_SECONDS:
31
+ break
32
+ return " ".join(p.strip() for p in parts).strip()
33
+ except Exception as e:
34
+ print(f"[LUNA] transcribe error on {os.path.basename(path)}: {e}")
35
+ return ""
36
+
37
+
38
+ def _process_one(prompt: str, path: str, name: str) -> dict:
39
+ """Full per-file pipeline (extract → MAP). Returns a 'kept' or 'skipped' dict."""
40
+ info = extract(path, name)
41
+ cat = info["category"]
42
+
43
+ if info["kind"] == "error":
44
+ return {"status": "skipped", "name": name, "category": cat, "reason": info["payload"]}
45
+
46
+ if info["kind"] == "image":
47
+ m = map_image(prompt, name, info["payload"])
48
+ elif info["kind"] == "media":
49
+ text = _transcribe(info["payload"])
50
+ if not text:
51
+ return {"status": "skipped", "name": name, "category": cat,
52
+ "reason": "no speech could be transcribed"}
53
+ m = map_text(prompt, name, cat, text)
54
+ else: # text
55
+ m = map_text(prompt, name, cat, info["payload"])
56
+
57
+ notes = (m.get("notes") or "").strip()[:MAP_NOTE_CHARS]
58
+ if m.get("relevant") and notes:
59
+ return {"status": "kept", "name": name, "category": cat,
60
+ "topics": m.get("topics") or [], "notes": notes}
61
+ return {"status": "skipped", "name": name, "category": cat,
62
+ "reason": "no content relevant to the request"}
63
+
64
+
65
+ def _resolve(files) -> list[tuple[str, str]]:
66
+ """Normalize Gradio's file input → [(path, display_name)]."""
67
+ out = []
68
+ for f in files or []:
69
+ if isinstance(f, str):
70
+ out.append((f, os.path.basename(f)))
71
+ elif isinstance(f, dict):
72
+ p = f.get("path") or f.get("name")
73
+ if p:
74
+ out.append((p, f.get("orig_name") or os.path.basename(p)))
75
+ elif hasattr(f, "name"):
76
+ out.append((f.name, os.path.basename(f.name)))
77
+ return out
78
+
79
+
80
+ def _validate(resolved) -> str | None:
81
+ if not resolved:
82
+ return "Please attach at least one file."
83
+ if len(resolved) > MAX_FILES:
84
+ return f"Too many files — the limit is {MAX_FILES}."
85
+ total = 0
86
+ for path, name in resolved:
87
+ try:
88
+ sz = os.path.getsize(path)
89
+ except OSError:
90
+ continue
91
+ total += sz
92
+ cat = category_of(name)
93
+ cap = MAX_MEDIA_MB if cat in ("audio", "video") else MAX_FILE_MB
94
+ if sz > cap * 1024 * 1024:
95
+ return f"'{name}' is too large ({sz // 1024 // 1024} MB; limit {cap} MB)."
96
+ if total > MAX_TOTAL_MB * 1024 * 1024:
97
+ return f"Total upload too large ({total // 1024 // 1024} MB; limit {MAX_TOTAL_MB} MB)."
98
+ return None
99
+
100
+
101
+ def run(files, prompt, token):
102
+ """Streaming generator → yields (status, note). The final yield carries the Note."""
103
+ prompt = (prompt or "").strip()
104
+ if not verify_usage_token(token):
105
+ yield ("error", "⚠️ This task could not be authorized. Please reload and sign in again.")
106
+ return
107
+ if not prompt:
108
+ yield ("error", "⚠️ Tell LUNA what to build first.")
109
+ return
110
+
111
+ resolved = _resolve(files)
112
+ err = _validate(resolved)
113
+ if err:
114
+ yield ("error", f"⚠️ {err}")
115
+ return
116
+
117
+ n = len(resolved)
118
+ yield (f"⏳ Reading {n} file{'s' if n != 1 else ''}…", "")
119
+
120
+ kept, skipped, done = [], [], 0
121
+ with ThreadPoolExecutor(max_workers=MAP_WORKERS) as pool:
122
+ futures = {pool.submit(_process_one, prompt, p, nm): nm for p, nm in resolved}
123
+ for fut in as_completed(futures):
124
+ try:
125
+ res = fut.result()
126
+ except Exception as e:
127
+ res = {"status": "skipped", "name": futures[fut],
128
+ "category": "other", "reason": f"processing error ({type(e).__name__})"}
129
+ (kept if res["status"] == "kept" else skipped).append(res)
130
+ done += 1
131
+ yield (f"⏳ Read {done}/{n} files — {len(kept)} relevant so far…", "")
132
+
133
+ yield (f"✍️ Writing the Note from {len(kept)} relevant file{'s' if len(kept) != 1 else ''}…", "")
134
+
135
+ last = ""
136
+ for note in reduce_stream(prompt, kept, skipped):
137
+ last = note
138
+ yield ("✍️", note)
139
+ yield ("✅ done", last)
requirements.txt ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # LUNA — pinned for reproducible Hugging Face Space builds
2
+ # (Linux · Python 3.12 · CPU). LUNA needs NO torch: synthesis runs on HF Inference
3
+ # Providers (huggingface-hub InferenceClient) and audio/video transcription uses
4
+ # faster-whisper (ctranslate2 backend, no torch). That keeps the Space small & fast.
5
+
6
+ # ── Core app ──
7
+ gradio==6.14.0
8
+ huggingface-hub==1.15.0
9
+ numpy==2.4.6
10
+ Pillow==12.2.0
11
+ langdetect==1.0.9 # answer in the prompt's language
12
+
13
+ # ── Audio / video transcription ──
14
+ faster-whisper==1.2.1 # pulls `av` + `tokenizers` transitively; decodes media via PyAV
15
+ ctranslate2==4.7.2
16
+
17
+ # ── Document / spreadsheet parsing (pure-python, left unpinned for resilience) ──
18
+ pypdf
19
+ python-docx
20
+ python-pptx
21
+ openpyxl
22
+ beautifulsoup4 # strip HTML to text
synthesis.py ADDED
@@ -0,0 +1,245 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # The two LLM stages of LUNA's map-reduce.
2
+ # MAP — per file: is it relevant to the prompt, and what are the on-topic notes?
3
+ # (cheap model for text; VLM for images). Strict-JSON output.
4
+ # REDUCE — fuse all kept notes into one structured Note (strong model, streamed).
5
+ import json
6
+ import re
7
+
8
+ from config import (
9
+ DEBUG, MAP_MAX_TOKENS, MAP_MODEL, MAP_TEMPERATURE, REDUCE_BUDGET,
10
+ REDUCE_MAX_TOKENS, REDUCE_MODEL, REDUCE_TEMPERATURE, VLM_MODEL,
11
+ )
12
+ from models import llm_client
13
+
14
+ # ── Language of the prompt (answer in it) ─────────────────────────────────────
15
+ _LANG_NAMES = {
16
+ "en": "English", "pl": "Polish", "de": "German", "fr": "French", "es": "Spanish",
17
+ "it": "Italian", "pt": "Portuguese", "nl": "Dutch", "ru": "Russian",
18
+ "uk": "Ukrainian", "cs": "Czech", "sk": "Slovak", "sv": "Swedish",
19
+ "no": "Norwegian", "da": "Danish", "fi": "Finnish", "tr": "Turkish",
20
+ "ro": "Romanian", "hu": "Hungarian", "el": "Greek", "ja": "Japanese",
21
+ "ko": "Korean", "zh": "Chinese", "ar": "Arabic",
22
+ }
23
+ _POLISH_CHARS = frozenset("ąćęłńóśźżĄĆĘŁŃÓŚŹŻ")
24
+
25
+
26
+ def detect_language(text: str) -> str:
27
+ stripped = (text or "").strip()
28
+ if stripped:
29
+ try:
30
+ from langdetect import detect, DetectorFactory
31
+ DetectorFactory.seed = 0
32
+ code = detect(stripped).lower()
33
+ name = _LANG_NAMES.get(code) or _LANG_NAMES.get(code.split("-")[0])
34
+ if name:
35
+ return name
36
+ except Exception:
37
+ pass
38
+ if any(c in text for c in _POLISH_CHARS):
39
+ return "Polish"
40
+ return "English"
41
+
42
+
43
+ # ── MAP ───────────────────────────────────────────────────────────────────────
44
+ _MAP_SYSTEM = (
45
+ "You are LUNA's extraction stage. You are given a user's build request and the "
46
+ "content of ONE source file. Your job: decide whether this file contains anything "
47
+ "relevant to the request, and if so pull out the relevant material faithfully.\n\n"
48
+ "Output STRICT JSON only (no prose, no code fence) with exactly these keys:\n"
49
+ ' "relevant": true or false,\n'
50
+ ' "topics": array of short topic labels (2-5 words each) that THIS file covers '
51
+ "and that relate to the request — a file may legitimately touch several distinct "
52
+ "topics, or none,\n"
53
+ ' "notes": a faithful, concise extraction of ONLY the request-relevant content as '
54
+ "Markdown bullet points; preserve key facts, numbers, names and quotes exactly; omit "
55
+ "anything off-topic; empty string if not relevant.\n\n"
56
+ "Rules: judge relevance generously but honestly — if the file says nothing about the "
57
+ "request, set relevant=false. NEVER invent content that isn't in the file. Keep names, "
58
+ "numbers and quotes verbatim."
59
+ )
60
+
61
+
62
+ def _parse_json(raw: str) -> dict:
63
+ raw = (raw or "").strip()
64
+ raw = re.sub(r"^```(?:json)?\s*|\s*```$", "", raw).strip()
65
+ try:
66
+ obj = json.loads(raw)
67
+ except Exception:
68
+ m = re.search(r"\{.*\}", raw, re.DOTALL)
69
+ if not m:
70
+ return {"relevant": False, "topics": [], "notes": ""}
71
+ try:
72
+ obj = json.loads(m.group(0))
73
+ except Exception:
74
+ return {"relevant": False, "topics": [], "notes": ""}
75
+ return {
76
+ "relevant": bool(obj.get("relevant")),
77
+ "topics": [str(t) for t in (obj.get("topics") or [])][:8],
78
+ "notes": str(obj.get("notes") or "").strip(),
79
+ }
80
+
81
+
82
+ def _map_user(prompt: str, name: str, category: str, content: str) -> str:
83
+ return (
84
+ f'Build request: "{prompt}"\n\n'
85
+ f"File: {name} (type: {category})\n"
86
+ f"--- file content ---\n{content}\n--- end ---\n\n"
87
+ "Return the strict JSON described in the system message."
88
+ )
89
+
90
+
91
+ def map_text(prompt: str, name: str, category: str, content: str) -> dict:
92
+ """Run the cheap MAP model on one text file. Returns {relevant, topics, notes}."""
93
+ try:
94
+ resp = llm_client.chat_completion(
95
+ model=MAP_MODEL,
96
+ messages=[
97
+ {"role": "system", "content": _MAP_SYSTEM},
98
+ {"role": "user", "content": _map_user(prompt, name, category, content)},
99
+ ],
100
+ max_tokens=MAP_MAX_TOKENS,
101
+ temperature=MAP_TEMPERATURE,
102
+ )
103
+ out = _parse_json(resp.choices[0].message.content)
104
+ if DEBUG:
105
+ print(f"[LUNA] MAP {name}: relevant={out['relevant']} topics={out['topics']}")
106
+ return out
107
+ except Exception as e:
108
+ print(f"[LUNA] MAP error on {name}: {e}")
109
+ # On failure, keep the raw content as notes so REDUCE can still use it.
110
+ return {"relevant": True, "topics": [], "notes": content[:2000]}
111
+
112
+
113
+ def map_image(prompt: str, name: str, data_url: str) -> dict:
114
+ """VLM reads the image (on-screen text + scene) AND judges relevance in one call."""
115
+ user = [
116
+ {"type": "text", "text": (
117
+ f'Build request: "{prompt}"\n\n'
118
+ f"This is an image file named {name}. Read any text visible in it verbatim "
119
+ "and describe what it shows, then judge relevance to the request and extract "
120
+ "the relevant content. Return the strict JSON from the system message."
121
+ )},
122
+ {"type": "image_url", "image_url": {"url": data_url}},
123
+ ]
124
+ try:
125
+ resp = llm_client.chat_completion(
126
+ model=VLM_MODEL,
127
+ messages=[
128
+ {"role": "system", "content": _MAP_SYSTEM},
129
+ {"role": "user", "content": user},
130
+ ],
131
+ max_tokens=MAP_MAX_TOKENS,
132
+ temperature=MAP_TEMPERATURE,
133
+ )
134
+ return _parse_json(resp.choices[0].message.content)
135
+ except Exception as e:
136
+ print(f"[LUNA] VLM error on {name}: {e}")
137
+ return {"relevant": False, "topics": [], "notes": ""}
138
+
139
+
140
+ # ── REDUCE ──────────────────────────────────────────────────────────────────
141
+ def _reduce_system(language: str) -> str:
142
+ return (
143
+ "You are LUNA, an expert research synthesizer. From extracted notes drawn from a "
144
+ "set of source files, you write ONE clean, structured Note that fulfils the user's "
145
+ "build request.\n\n"
146
+ f"LANGUAGE — Write the ENTIRE Note in {language} (the language of the request). "
147
+ "Keep names, numbers, brands, titles and direct quotes exactly as they appear.\n\n"
148
+ "GROUNDING — Use ONLY the provided notes. Never invent facts, figures or sources. "
149
+ "If the material is thin or doesn't really cover the request, say so honestly rather "
150
+ "than padding.\n\n"
151
+ "REQUIRED STRUCTURE (Markdown):\n"
152
+ "1. A '# ' title that captures the subject of the request.\n"
153
+ "2. '## Agenda' — a numbered list naming the body sections that follow (a table of "
154
+ "contents).\n"
155
+ "3. The body: one '## ' section per agenda item, in the same order, using clear "
156
+ "prose, bullet lists, and Markdown tables where they help. Synthesize across files; "
157
+ "group related facts even when they came from different files.\n"
158
+ "4. '## Sources' — a bullet per file you ACTUALLY used, formatted "
159
+ "`- **<filename>** — <what it contributed>`. Then, if any files were set aside, one "
160
+ "final line: `_Not used: <comma-separated filenames>_` (with a brief shared reason "
161
+ "if useful). If every file was used, omit the Not-used line.\n\n"
162
+ "Be thorough but do not repeat yourself. Optimize for a reader skimming the Note."
163
+ )
164
+
165
+
166
+ def _reduce_user(prompt: str, kept: list[dict], skipped: list[dict]) -> str:
167
+ blocks, total, truncated = [], 0, False
168
+ for item in kept:
169
+ topics = ", ".join(item.get("topics") or []) or "—"
170
+ block = (f"### File: {item['name']} (type: {item['category']}; topics: {topics})\n"
171
+ f"{item['notes']}\n")
172
+ if total + len(block) > REDUCE_BUDGET:
173
+ truncated = True
174
+ break
175
+ blocks.append(block)
176
+ total += len(block)
177
+
178
+ kept_section = "\n".join(blocks) if blocks else "(no relevant material was extracted)"
179
+ skip_section = ""
180
+ if skipped:
181
+ lines = [f"- {s['name']}: {s.get('reason', 'not relevant')}" for s in skipped]
182
+ skip_section = (
183
+ "\n\n--- Files that were set aside (irrelevant or unreadable) ---\n"
184
+ + "\n".join(lines)
185
+ )
186
+ trunc_note = "\n\n(Note: some lower-priority material was omitted to fit budget.)" if truncated else ""
187
+
188
+ return (
189
+ f'User build request: "{prompt}"\n\n'
190
+ "Below are the relevant extractions from the uploaded files. Synthesize them into "
191
+ "the structured Note defined in the system message.\n\n"
192
+ f"=== Relevant extractions ===\n\n{kept_section}"
193
+ f"{skip_section}{trunc_note}\n\n"
194
+ "Now write the Note. Remember: Title, then '## Agenda', then the body sections, then "
195
+ "'## Sources' (only files you actually used) plus the '_Not used:_' line for the rest."
196
+ )
197
+
198
+
199
+ def reduce_stream(prompt: str, kept: list[dict], skipped: list[dict]):
200
+ """Stream the final Note from the strong model. Yields the accumulated Markdown."""
201
+ language = detect_language(prompt)
202
+ if DEBUG:
203
+ print(f"[LUNA] REDUCE — {len(kept)} kept, {len(skipped)} skipped, lang={language}")
204
+ if not kept:
205
+ yield (
206
+ f"# {prompt.strip()[:80]}\n\n"
207
+ "## Agenda\n1. No relevant material found\n\n"
208
+ "## No relevant material found\n\n"
209
+ "None of the uploaded files contained content relevant to this request. "
210
+ "Try a broader prompt, or check that the right files were attached.\n\n"
211
+ "## Sources\n"
212
+ + ("\n".join(f"- **{s['name']}** — {s.get('reason', 'not relevant')}" for s in skipped)
213
+ if skipped else "- (none)")
214
+ )
215
+ return
216
+
217
+ acc = ""
218
+ try:
219
+ stream = llm_client.chat_completion(
220
+ model=REDUCE_MODEL,
221
+ messages=[
222
+ {"role": "system", "content": _reduce_system(language)},
223
+ {"role": "user", "content": _reduce_user(prompt, kept, skipped)},
224
+ ],
225
+ max_tokens=REDUCE_MAX_TOKENS,
226
+ temperature=REDUCE_TEMPERATURE,
227
+ stream=True,
228
+ )
229
+ for ev in stream:
230
+ if not ev.choices:
231
+ continue
232
+ delta = ev.choices[0].delta.content or ""
233
+ if delta:
234
+ acc += delta
235
+ yield acc
236
+ if not acc.strip():
237
+ raise RuntimeError("empty REDUCE response")
238
+ except Exception as e:
239
+ print(f"[LUNA] REDUCE error: {e}")
240
+ # Fallback: stitch the raw notes so the user still gets something usable.
241
+ body = "\n\n".join(f"## {k['name']}\n{k['notes']}" for k in kept)
242
+ srcs = "\n".join(f"- **{k['name']}**" for k in kept)
243
+ yield (f"# {prompt.strip()[:80]}\n\n## Agenda\n"
244
+ + "\n".join(f"{i}. {k['name']}" for i, k in enumerate(kept, 1))
245
+ + f"\n\n{body}\n\n## Sources\n{srcs}")