# Orchestration: categorize → extract+MAP every file in parallel → REDUCE into the # final Note. Exposed as a streaming generator that yields (status, note) for Gradio; # the website's worker forwards `status` as a progress marker and accumulates `note`. import os import threading import time from concurrent.futures import ThreadPoolExecutor, TimeoutError as FutTimeout, as_completed import cache from auth import check_usage_token from config import ( BUILD_MAP_DEADLINE_SEC, MAP_CHUNK_BUDGET, MAP_NOTE_CHARS, MAP_WORKERS, MAX_CHUNKS_PER_FILE, MAX_FILES, MAX_FILE_MB, MAX_MEDIA_MB, MAX_TOTAL_MB, PER_FILE_CHARS, REDUCE_BUDGET, TRANSCRIBE_MAX_SECONDS, TRANSCRIBE_WORKERS, ) from extract import category_of, extract from frames import sample_frames from models import get_whisper from subtitles import embedded_transcript from synthesis import (MAP_PROMPT_ID, map_image, map_pages, map_text, map_video, reduce_stream) _transcribe_sem = threading.Semaphore(TRANSCRIBE_WORKERS) def _transcribe(path: str) -> str: # A subtitle track already in the container beats anything we would transcribe: # it is free, it is faster, and the spelling of names and numbers is right. subs = embedded_transcript(path) if subs: return subs # Keyed by the file alone, not by the prompt: rebuilding the same video with a # reworded prompt used to run Whisper over it again from scratch. cached = cache.load_raw(path, "transcript") if isinstance(cached, str): print(f"[LUNA] transcript cache hit for {os.path.basename(path)}") return cached model = get_whisper() if model is None: return "" with _transcribe_sem: try: segments, _info = model.transcribe(path, beam_size=1, vad_filter=True) parts, cut = [], False for seg in segments: parts.append(seg.text) if seg.end and seg.end > TRANSCRIBE_MAX_SECONDS: cut = True break text = " ".join(p.strip() for p in parts).strip() if text and cut: # Said in the transcript itself, so MAP and REDUCE both know the # recording goes on. It used to stop silently at the half hour. text += transcript_cut_note() # Stored even when empty: a silent clip stays silent, and re-proving it # costs another full pass. cache.store_raw(path, "transcript", text) return text except Exception as e: print(f"[LUNA] transcribe error on {os.path.basename(path)}: {e}") return "" def transcript_cut_note() -> str: m = TRANSCRIBE_MAX_SECONDS // 60 return (f"\n\n[The transcript stops at {m} minutes. The recording continues beyond " f"this point and the rest was not transcribed.]") def _skip(path, prompt, name, cat, reason, cacheable=True): """A skipped file, optionally remembered. Only the verdicts that cannot change are stored. "This is a binary file" and "no speech could be transcribed" will say the same thing on every retry, and proving it again costs another extraction, or another full Whisper pass. A failure that came from our side - a model that could not be reached, a missing dependency - must NOT be remembered, or one outage would mark a perfectly good file unreadable for the rest of the cache's life.""" out = {"status": "skipped", "name": name, "category": cat, "reason": reason, "notes": ""} if cacheable: cache.store(path, prompt, out) return out def _split(content: str, parts: int) -> list[str]: """Cut a long extraction into `parts` pieces of at most PER_FILE_CHARS. Prefers a line break near the boundary: a cut through the middle of a sentence costs the notes on both sides of it, and documents break at lines for free.""" out, i, n = [], 0, len(content) for k in range(parts): end = min(i + PER_FILE_CHARS, n) if end < n: window = content.rfind("\n", i + PER_FILE_CHARS - 800, end) if window > i: end = window out.append(content[i:end]) i = end if i >= n: break return out def _map_long_text(prompt: str, name: str, cat: str, content: str, allowance: int) -> dict: """MAP one text file, in parts when it is longer than a single call can read. Before this, a file was whatever its first PER_FILE_CHARS said and the rest did not exist: a 143-page report reached synthesis as its own cover and table of contents. Parts are noted concurrently and their notes concatenated in order, so the file still arrives at REDUCE as one block.""" total = len(content) needed = max(1, -(-total // PER_FILE_CHARS)) # ceil parts = max(1, min(needed, allowance, MAX_CHUNKS_PER_FILE)) if parts == 1: dropped = max(0, total - PER_FILE_CHARS) out = map_text(prompt, name, cat, content[:PER_FILE_CHARS], 1, 1, dropped) if dropped: out["coverage"] = (f"only the first {PER_FILE_CHARS} of {total} characters " f"were read") return out chunks = _split(content, parts) read = sum(len(c) for c in chunks) dropped = max(0, total - read) print(f"[LUNA] «{name}» is {total} chars: reading it in {len(chunks)} parts" + (f", {dropped} chars beyond the budget" if dropped else "")) results = [None] * len(chunks) with ThreadPoolExecutor(max_workers=min(MAP_WORKERS, len(chunks))) as pool: futs = {pool.submit(map_text, prompt, name, cat, ch, i + 1, len(chunks), dropped): i for i, ch in enumerate(chunks)} for fut in as_completed(futs): i = futs[fut] try: results[i] = fut.result() except Exception as e: print(f"[LUNA] part {i+1} of «{name}» failed: {type(e).__name__}: {e}") notes = [r["notes"].strip() for r in results if r and (r.get("notes") or "").strip()] topics: list[str] = [] for r in results: for t in (r or {}).get("topics") or []: if t not in topics: topics.append(t) if not notes: err = next((r.get("error") for r in results if r and r.get("error")), None) return {"topics": [], "notes": "", "error": err} out = {"topics": topics[:8], "notes": "\n".join(notes)} # Say what was actually read, and say it separately from what was actually noted: # a part whose model call came back empty is not the same as a part nobody read, # and rolling the two together is how "read 421110 of 421110 characters" ended up # printed over notes that were missing one part of twenty-seven. cov = [] if dropped: cov.append(f"read {read} of {total} characters") if len(notes) < len(chunks): cov.append(f"notes cover {len(notes)} of {len(chunks)} parts") if cov: out["coverage"] = "; ".join(cov) return out def _note_cap(n_files: int) -> int: """One file may keep notes worth the whole REDUCE budget; many share it. The flat 24000 cap used to cut a lone 27-part report to about eight parts, silently.""" return max(MAP_NOTE_CHARS, REDUCE_BUDGET // max(1, n_files)) def _cap_notes(notes: str, cap: int) -> tuple[str, str]: """Cut notes to `cap` at a line boundary and say what was lost. Returns (notes, coverage).""" notes = (notes or "").strip() if len(notes) <= cap: return notes, "" cut = notes.rfind("\n", 0, cap) if cut < cap // 2: cut = cap kept = notes[:cut].rstrip() return kept, f"notes cut to {len(kept)} of {len(notes)} characters" def _process_one(prompt: str, path: str, name: str, allowance: int = 1, info: dict | None = None, note_cap: int = MAP_NOTE_CHARS) -> dict: """Full per-file pipeline (extract → MAP). Returns a 'kept' or 'skipped' dict. Every readable file is 'kept' WITH its extracted notes; the REDUCE stage (which sees all files together) makes the include/exclude call. Only genuinely unreadable files are 'skipped', so a shaky per-file call can never silently destroy a file's content.""" if info is None: info = extract(path, name) ckey = _ckey(prompt) cat = info["category"] print(f"[LUNA] processing «{name}» ({cat}, kind={info['kind']})") if info["kind"] == "error": print(f"[LUNA] «{name}» unreadable: {info['payload']}") # "couldn't read (SomeError)" means an exception on our side, which a later # run may not hit; everything else is a property of the file itself. return _skip(path, ckey, name, cat, info["payload"], cacheable=not str(info["payload"]).startswith("couldn't read (")) if info["kind"] == "pages": m = map_pages(prompt, name, info["payload"]) elif info["kind"] == "image": m = map_image(prompt, name, info["payload"]) elif info["kind"] == "media": if cat == "video": frames = sample_frames(info["payload"]) transcript = _transcribe(info["payload"]) print(f"[LUNA] video «{name}»: {len(frames)} frame(s), " f"{len(transcript)} transcript chars") if frames: m = map_video(prompt, name, transcript, frames) if len(transcript) > PER_FILE_CHARS and allowance > 1: # The vision call reads the frames with the FIRST window of speech; # the rest of a long recording goes through the text path with the # file's remaining allowance, so a 30-minute talk is not reduced to # its first 16000 characters just because it has a picture track. rest = _map_long_text(prompt, name, cat, transcript[PER_FILE_CHARS:], allowance - 1) if (rest.get("notes") or "").strip(): m = {"topics": list(dict.fromkeys((m.get("topics") or []) + (rest.get("topics") or [])))[:8], "notes": ((m.get("notes") or "").strip() + "\n" + rest["notes"].strip()).strip(), "error": m.get("error")} if rest.get("coverage"): m["coverage"] = rest["coverage"] elif transcript: m = _map_long_text(prompt, name, cat, transcript, allowance) else: # NOT remembered. "No frames or speech" is exactly what a missing # PyAV or a Whisper that failed to load also looks like from here, # and caching that would mark a perfectly good recording unreadable # for the life of the entry. Nothing is lost by re-checking: the # transcript cache above already holds the expensive half. return _skip(path, ckey, name, cat, "the video couldn't be read (no frames or speech)", cacheable=False) else: # audio transcript = _transcribe(info["payload"]) if not transcript: # Same reasoning: an empty transcript can mean a silent file or a # Whisper that never loaded, and only one of those is the file's # fault. The transcript cache already saves the second pass. return _skip(path, ckey, name, cat, "no speech could be transcribed", cacheable=False) m = _map_long_text(prompt, name, cat, transcript, allowance) else: # text m = _map_long_text(prompt, name, cat, info["payload"], allowance) notes, cut_note = _cap_notes(m.get("notes") or "", note_cap) if not notes: # MAP produced no extraction. If the stage said why (a model call that could not # be reached), pass that through: blaming the file for an outage on our side sends # the user off checking a file that was perfectly fine. return _skip(path, ckey, name, cat, m.get("error") or "no readable content could be extracted", cacheable=not m.get("error")) result = {"status": "kept", "name": name, "category": cat, "topics": m.get("topics") or [], "notes": notes} cov = "; ".join(x for x in (m.get("coverage"), cut_note) if x) if cov: result["coverage"] = cov cache.store(path, ckey, result) return result def _ckey(prompt: str) -> str: """Cache key for one file's notes: the request plus the identity of the brief that produced them. Without the second half, fixing the note-taking prompt changes nothing for any file already in the cache.""" return f"{prompt}\x00map={MAP_PROMPT_ID}" def _read_one(prompt: str, path: str, name: str): """Cache lookup, then extraction. No model is called here. Extraction is separated from note-taking so the chunk budget can be shared out once the real sizes are known. Sharing it by file size on disk does not work: the same 9 MB is 421000 characters of report text or a single photograph. Returns ("cached", result) or ("info", info).""" hit = cache.load(path, _ckey(prompt)) if hit is not None: print(f"[LUNA] cache hit for «{name}» - extraction and MAP skipped") return "cached", hit info = extract(path, name) print(f"[LUNA] read «{name}» ({info['category']}, kind={info['kind']}" + (f", {len(info['payload'])} chars" if info["kind"] == "text" else "") + ")") return "info", info _MEDIA_CHUNK_ESTIMATE = 4 # 30 minutes of speech, the transcription cap def _allocate(infos: dict, budget: int) -> dict: """Share MAP_CHUNK_BUDGET out between the files that are longer than one call. One chunk each first, so no file is skipped, then the rest handed round in turns to whoever still wants more. A single long document therefore gets the whole budget, and a hundred short files are unaffected.""" need = {} for nm, info in infos.items(): if info.get("kind") == "text": need[nm] = max(1, -(-len(info["payload"]) // PER_FILE_CHARS)) elif info.get("kind") == "media": # A transcript's length is unknown until Whisper has run, and it is the one # input that arrives long: TRANSCRIBE_MAX_SECONDS is 30 minutes of speech, # roughly 4 calls' worth. Reserved up front, and unused reservations cost # nothing because _map_long_text only splits what is actually there. need[nm] = _MEDIA_CHUNK_ESTIMATE give = {nm: 1 for nm in need} left = budget - len(give) while left > 0 and any(give[nm] < min(need[nm], MAX_CHUNKS_PER_FILE) for nm in need): for nm in need: if left <= 0: break if give[nm] < min(need[nm], MAX_CHUNKS_PER_FILE): give[nm] += 1 left -= 1 short = [nm for nm in need if give[nm] < need[nm]] if short: print(f"[LUNA] chunk budget {budget} does not cover {len(short)} long file(s); " f"they are read in part: " + ", ".join(f"{nm} {give[nm]}/{need[nm]}" for nm in short[:5])) return give def _resolve(files) -> list[tuple[str, str]]: """Normalize Gradio's file input → [(path, display_name)].""" out = [] for f in files or []: if isinstance(f, str): out.append((f, os.path.basename(f))) elif isinstance(f, dict): p = f.get("path") or f.get("name") if p: out.append((p, f.get("orig_name") or os.path.basename(p))) elif hasattr(f, "name"): out.append((f.name, os.path.basename(f.name))) return out def _validate(resolved) -> str | None: if not resolved: return "Please attach at least one file." if len(resolved) > MAX_FILES: return f"Too many files - the limit is {MAX_FILES}." total = 0 missing = [] for path, name in resolved: try: sz = os.path.getsize(path) except FileNotFoundError: # The Space purges uploads after an hour (app.py delete_cache). The website # uploads at wizard step 1 and builds at step 4, so an idle user arrives here # with paths that no longer exist. Skipping them silently let validation pass, # every file then failed to read, and REDUCE produced a polite "nothing # relevant was found" Note. That Note is not empty, so the worker stored it # and charged one of the user's three daily builds. Fail loudly instead: the # worker only charges once a Note comes back. missing.append(name) continue except OSError: continue total += sz cat = category_of(name) cap = MAX_MEDIA_MB if cat in ("audio", "video") else MAX_FILE_MB if sz > cap * 1024 * 1024: return f"'{name}' is too large ({sz // 1024 // 1024} MB; limit {cap} MB)." if missing: n = len(missing) shown = ", ".join(missing[:3]) + (" and more" if n > 3 else "") return (f"{n} file{'s' if n != 1 else ''} expired before the build started " f"({shown}). Uploads are kept for one hour, please attach them again.") if total > MAX_TOTAL_MB * 1024 * 1024: return f"Total upload too large ({total // 1024 // 1024} MB; limit {MAX_TOTAL_MB} MB)." return None def run(files, prompt, token): """Streaming generator → yields (status, note). The final yield carries the Note.""" # The website may append format directives after a ⟦FORMAT⟧ sentinel. Split them off # so language detection + synthesis see only the user's real words (the # directives are English meta and must not change the Note's language). raw = (prompt or "").strip() base, _, fmt = raw.partition("⟦FORMAT⟧") base, fmt = base.strip(), fmt.strip() print(f"\n[LUNA] ========== NEW JOB REQUEST ==========") print(f"[LUNA] prompt={len(base)} chars" + (f" fmt={len(fmt)} chars" if fmt else "")) why = check_usage_token(token, "build") if why != "ok": print(f"[LUNA] job REJECTED - token {why}") if why == "expired": yield ("error", "⚠️ This task waited too long before LUNA could start it. " "Nothing was used up - please try again.") elif why == "unconfigured": # Ours, not theirs: the Space has no signing secret, so it cannot tell a # real request from anyone else's. Say so rather than blame their session. yield ("error", "⚠️ LUNA is misconfigured on the server and cannot accept " "tasks right now. Nothing was used up.") else: yield ("error", "⚠️ This task could not be authorized. Please reload and sign in again.") return if not base: print("[LUNA] job REJECTED - empty prompt") yield ("error", "⚠️ Tell LUNA what to build first.") return resolved = _resolve(files) err = _validate(resolved) if err: print(f"[LUNA] job REJECTED - {err}") yield ("error", f"⚠️ {err}") return n = len(resolved) print(f"[LUNA] {n} file(s): " + ", ".join(nm for _, nm in resolved[:25]) + (" …" if n > 25 else "")) cache.evict_stale() yield (f"⏳ Reading {n} file{'s' if n != 1 else ''}…", "") deadline = time.monotonic() + BUILD_MAP_DEADLINE_SEC t_start = time.monotonic() def _timed_out(nm): return {"status": "skipped", "name": nm, "category": "other", "reason": "the build ran out of time before this file was noted", "notes": ""} # Phase 1: read every file (cache, then extraction). No model is called yet, so the # real sizes are known before a single token is spent. kept, skipped, done = [], [], 0 infos, cached = {}, {} pool = ThreadPoolExecutor(max_workers=MAP_WORKERS) futures = {pool.submit(_read_one, base, p, nm): (p, nm) for p, nm in resolved} try: for fut in as_completed(futures, timeout=max(1.0, deadline - time.monotonic())): path, nm = futures[fut] try: kind, payload = fut.result() except Exception as e: print(f"[LUNA] file «{nm}» failed to read: {type(e).__name__}: {e}") skipped.append({"status": "skipped", "name": nm, "category": "other", "reason": f"processing error ({type(e).__name__})", "notes": ""}) done += 1 continue if kind == "cached": cached[nm] = payload else: infos[nm] = payload done += 1 yield (f"⏳ Read {done}/{n} files…", "") except FutTimeout: late = [nm for fut, (_, nm) in futures.items() if not fut.done()] print(f"[LUNA] reading ran past {BUILD_MAP_DEADLINE_SEC}s: {len(late)} file(s) abandoned") skipped.extend(_timed_out(nm) for nm in late) finally: # Never wait: a reader stuck in a parser must not hold the build. Abandoned # threads finish on their own and their results are simply not used. pool.shutdown(wait=False, cancel_futures=True) print(f"[LUNA] stage=read ms={int((time.monotonic() - t_start) * 1000)}") # Phase 2: share the chunk budget out, then take notes on everything at once. give = _allocate(infos, MAP_CHUNK_BUDGET) for res in cached.values(): (kept if res.get("status") == "kept" else skipped).append(res) done = len(cached) + len(skipped) note_cap = _note_cap(n) if infos: yield (f"⏳ Taking notes on {len(infos)} file{'s' if len(infos) != 1 else ''}…", "") by_name = {nm: p for p, nm in resolved} t_map = time.monotonic() pool = ThreadPoolExecutor(max_workers=MAP_WORKERS) futures = {pool.submit(_process_one, base, by_name[nm], nm, give.get(nm, 1), info, note_cap): nm for nm, info in infos.items()} try: for fut in as_completed(futures, timeout=max(1.0, deadline - time.monotonic())): nm = futures[fut] try: res = fut.result() except Exception as e: print(f"[LUNA] file «{nm}» crashed: {type(e).__name__}: {e}") res = {"status": "skipped", "name": nm, "category": "other", "reason": f"processing error ({type(e).__name__})", "notes": ""} (kept if res["status"] == "kept" else skipped).append(res) done += 1 yield (f"⏳ Noted {done}/{n} files - {len(kept)} readable so far…", "") except FutTimeout: late = [nm for fut, nm in futures.items() if not fut.done()] print(f"[LUNA] note-taking ran past the {BUILD_MAP_DEADLINE_SEC}s deadline: " f"{len(late)} file(s) abandoned, writing the Note from the rest") skipped.extend(_timed_out(nm) for nm in late) yield (f"⏳ Out of time: writing the Note from {len(kept)} of {n} files…", "") finally: pool.shutdown(wait=False, cancel_futures=True) print(f"[LUNA] stage=map ms={int((time.monotonic() - t_map) * 1000)}") print(f"[LUNA] extraction complete - {len(kept)} readable, {len(skipped)} unreadable") # Every readable file (with its notes) is now in `kept`; REDUCE decides what to # actually use. `kept` is empty only when no file could be read at all - then REDUCE # produces the honest "nothing could be read" Note from the `skipped` reasons. yield (f"✍️ Writing the Note from {len(kept)} file{'s' if len(kept) != 1 else ''}…", "") # `status` carries the one fact the website could never see before: that synthesis # failed and what the user is holding is not a Note. The final marker says so, and # the worker gives the daily build back instead of charging for raw material. status: dict = {} last = "" t_reduce = time.monotonic() for note in reduce_stream(base, kept, skipped, fmt, status): last = note yield ("✍️", note) print(f"[LUNA] stage=reduce ms={int((time.monotonic() - t_reduce) * 1000)} " f"total ms={int((time.monotonic() - t_start) * 1000)}") if status.get("degraded"): reason = str(status.get("reason") or "synthesis failed")[:200] print(f"[LUNA] ========== JOB DEGRADED - {reason} ==========\n") yield (f"⚠️ degraded: {reason}", last) return print(f"[LUNA] ========== JOB DONE - Note {len(last)} chars ==========\n") yield ("✅ done", last)