LUNA / pipeline.py
Malina
Claude Fable 5.1
Scoped tokens, bounded archives and frames, build deadline, circuit breaker, honest note caps
da9de03
Raw History Blame Contribute Delete
25 kB
# Orchestration: categorize → extract+MAP every file in parallel → REDUCE into the
# final Note. Exposed as a streaming generator that yields (status, note) for Gradio;
# the website's worker forwards `status` as a progress marker and accumulates `note`.
import os
import threading
import time
from concurrent.futures import ThreadPoolExecutor, TimeoutError as FutTimeout, as_completed
import cache
from auth import check_usage_token
from config import (
BUILD_MAP_DEADLINE_SEC, MAP_CHUNK_BUDGET, MAP_NOTE_CHARS, MAP_WORKERS,
MAX_CHUNKS_PER_FILE, MAX_FILES, MAX_FILE_MB, MAX_MEDIA_MB, MAX_TOTAL_MB,
PER_FILE_CHARS, REDUCE_BUDGET, TRANSCRIBE_MAX_SECONDS, TRANSCRIBE_WORKERS,
)
from extract import category_of, extract
from frames import sample_frames
from models import get_whisper
from subtitles import embedded_transcript
from synthesis import (MAP_PROMPT_ID, map_image, map_pages, map_text, map_video,
reduce_stream)
_transcribe_sem = threading.Semaphore(TRANSCRIBE_WORKERS)
def _transcribe(path: str) -> str:
# A subtitle track already in the container beats anything we would transcribe:
# it is free, it is faster, and the spelling of names and numbers is right.
subs = embedded_transcript(path)
if subs:
return subs
# Keyed by the file alone, not by the prompt: rebuilding the same video with a
# reworded prompt used to run Whisper over it again from scratch.
cached = cache.load_raw(path, "transcript")
if isinstance(cached, str):
print(f"[LUNA] transcript cache hit for {os.path.basename(path)}")
return cached
model = get_whisper()
if model is None:
return ""
with _transcribe_sem:
try:
segments, _info = model.transcribe(path, beam_size=1, vad_filter=True)
parts, cut = [], False
for seg in segments:
parts.append(seg.text)
if seg.end and seg.end > TRANSCRIBE_MAX_SECONDS:
cut = True
break
text = " ".join(p.strip() for p in parts).strip()
if text and cut:
# Said in the transcript itself, so MAP and REDUCE both know the
# recording goes on. It used to stop silently at the half hour.
text += transcript_cut_note()
# Stored even when empty: a silent clip stays silent, and re-proving it
# costs another full pass.
cache.store_raw(path, "transcript", text)
return text
except Exception as e:
print(f"[LUNA] transcribe error on {os.path.basename(path)}: {e}")
return ""
def transcript_cut_note() -> str:
m = TRANSCRIBE_MAX_SECONDS // 60
return (f"\n\n[The transcript stops at {m} minutes. The recording continues beyond "
f"this point and the rest was not transcribed.]")
def _skip(path, prompt, name, cat, reason, cacheable=True):
"""A skipped file, optionally remembered.
Only the verdicts that cannot change are stored. "This is a binary file" and
"no speech could be transcribed" will say the same thing on every retry, and
proving it again costs another extraction, or another full Whisper pass. A
failure that came from our side - a model that could not be reached, a missing
dependency - must NOT be remembered, or one outage would mark a perfectly good
file unreadable for the rest of the cache's life."""
out = {"status": "skipped", "name": name, "category": cat,
"reason": reason, "notes": ""}
if cacheable:
cache.store(path, prompt, out)
return out
def _split(content: str, parts: int) -> list[str]:
"""Cut a long extraction into `parts` pieces of at most PER_FILE_CHARS.
Prefers a line break near the boundary: a cut through the middle of a sentence
costs the notes on both sides of it, and documents break at lines for free."""
out, i, n = [], 0, len(content)
for k in range(parts):
end = min(i + PER_FILE_CHARS, n)
if end < n:
window = content.rfind("\n", i + PER_FILE_CHARS - 800, end)
if window > i:
end = window
out.append(content[i:end])
i = end
if i >= n:
break
return out
def _map_long_text(prompt: str, name: str, cat: str, content: str, allowance: int) -> dict:
"""MAP one text file, in parts when it is longer than a single call can read.
Before this, a file was whatever its first PER_FILE_CHARS said and the rest did not
exist: a 143-page report reached synthesis as its own cover and table of contents.
Parts are noted concurrently and their notes concatenated in order, so the file
still arrives at REDUCE as one block."""
total = len(content)
needed = max(1, -(-total // PER_FILE_CHARS)) # ceil
parts = max(1, min(needed, allowance, MAX_CHUNKS_PER_FILE))
if parts == 1:
dropped = max(0, total - PER_FILE_CHARS)
out = map_text(prompt, name, cat, content[:PER_FILE_CHARS], 1, 1, dropped)
if dropped:
out["coverage"] = (f"only the first {PER_FILE_CHARS} of {total} characters "
f"were read")
return out
chunks = _split(content, parts)
read = sum(len(c) for c in chunks)
dropped = max(0, total - read)
print(f"[LUNA] «{name}» is {total} chars: reading it in {len(chunks)} parts"
+ (f", {dropped} chars beyond the budget" if dropped else ""))
results = [None] * len(chunks)
with ThreadPoolExecutor(max_workers=min(MAP_WORKERS, len(chunks))) as pool:
futs = {pool.submit(map_text, prompt, name, cat, ch, i + 1, len(chunks), dropped): i
for i, ch in enumerate(chunks)}
for fut in as_completed(futs):
i = futs[fut]
try:
results[i] = fut.result()
except Exception as e:
print(f"[LUNA] part {i+1} of «{name}» failed: {type(e).__name__}: {e}")
notes = [r["notes"].strip() for r in results if r and (r.get("notes") or "").strip()]
topics: list[str] = []
for r in results:
for t in (r or {}).get("topics") or []:
if t not in topics:
topics.append(t)
if not notes:
err = next((r.get("error") for r in results if r and r.get("error")), None)
return {"topics": [], "notes": "", "error": err}
out = {"topics": topics[:8], "notes": "\n".join(notes)}
# Say what was actually read, and say it separately from what was actually noted:
# a part whose model call came back empty is not the same as a part nobody read,
# and rolling the two together is how "read 421110 of 421110 characters" ended up
# printed over notes that were missing one part of twenty-seven.
cov = []
if dropped:
cov.append(f"read {read} of {total} characters")
if len(notes) < len(chunks):
cov.append(f"notes cover {len(notes)} of {len(chunks)} parts")
if cov:
out["coverage"] = "; ".join(cov)
return out
def _note_cap(n_files: int) -> int:
"""One file may keep notes worth the whole REDUCE budget; many share it. The flat
24000 cap used to cut a lone 27-part report to about eight parts, silently."""
return max(MAP_NOTE_CHARS, REDUCE_BUDGET // max(1, n_files))
def _cap_notes(notes: str, cap: int) -> tuple[str, str]:
"""Cut notes to `cap` at a line boundary and say what was lost. Returns (notes, coverage)."""
notes = (notes or "").strip()
if len(notes) <= cap:
return notes, ""
cut = notes.rfind("\n", 0, cap)
if cut < cap // 2:
cut = cap
kept = notes[:cut].rstrip()
return kept, f"notes cut to {len(kept)} of {len(notes)} characters"
def _process_one(prompt: str, path: str, name: str, allowance: int = 1,
info: dict | None = None, note_cap: int = MAP_NOTE_CHARS) -> dict:
"""Full per-file pipeline (extract → MAP). Returns a 'kept' or 'skipped' dict.
Every readable file is 'kept' WITH its extracted notes; the REDUCE stage (which sees
all files together) makes the include/exclude call. Only genuinely unreadable files are
'skipped', so a shaky per-file call can never silently destroy a file's content."""
if info is None:
info = extract(path, name)
ckey = _ckey(prompt)
cat = info["category"]
print(f"[LUNA] processing «{name}» ({cat}, kind={info['kind']})")
if info["kind"] == "error":
print(f"[LUNA] «{name}» unreadable: {info['payload']}")
# "couldn't read (SomeError)" means an exception on our side, which a later
# run may not hit; everything else is a property of the file itself.
return _skip(path, ckey, name, cat, info["payload"],
cacheable=not str(info["payload"]).startswith("couldn't read ("))
if info["kind"] == "pages":
m = map_pages(prompt, name, info["payload"])
elif info["kind"] == "image":
m = map_image(prompt, name, info["payload"])
elif info["kind"] == "media":
if cat == "video":
frames = sample_frames(info["payload"])
transcript = _transcribe(info["payload"])
print(f"[LUNA] video «{name}»: {len(frames)} frame(s), "
f"{len(transcript)} transcript chars")
if frames:
m = map_video(prompt, name, transcript, frames)
if len(transcript) > PER_FILE_CHARS and allowance > 1:
# The vision call reads the frames with the FIRST window of speech;
# the rest of a long recording goes through the text path with the
# file's remaining allowance, so a 30-minute talk is not reduced to
# its first 16000 characters just because it has a picture track.
rest = _map_long_text(prompt, name, cat, transcript[PER_FILE_CHARS:],
allowance - 1)
if (rest.get("notes") or "").strip():
m = {"topics": list(dict.fromkeys((m.get("topics") or []) + (rest.get("topics") or [])))[:8],
"notes": ((m.get("notes") or "").strip() + "\n" + rest["notes"].strip()).strip(),
"error": m.get("error")}
if rest.get("coverage"):
m["coverage"] = rest["coverage"]
elif transcript:
m = _map_long_text(prompt, name, cat, transcript, allowance)
else:
# NOT remembered. "No frames or speech" is exactly what a missing
# PyAV or a Whisper that failed to load also looks like from here,
# and caching that would mark a perfectly good recording unreadable
# for the life of the entry. Nothing is lost by re-checking: the
# transcript cache above already holds the expensive half.
return _skip(path, ckey, name, cat,
"the video couldn't be read (no frames or speech)",
cacheable=False)
else: # audio
transcript = _transcribe(info["payload"])
if not transcript:
# Same reasoning: an empty transcript can mean a silent file or a
# Whisper that never loaded, and only one of those is the file's
# fault. The transcript cache already saves the second pass.
return _skip(path, ckey, name, cat, "no speech could be transcribed",
cacheable=False)
m = _map_long_text(prompt, name, cat, transcript, allowance)
else: # text
m = _map_long_text(prompt, name, cat, info["payload"], allowance)
notes, cut_note = _cap_notes(m.get("notes") or "", note_cap)
if not notes:
# MAP produced no extraction. If the stage said why (a model call that could not
# be reached), pass that through: blaming the file for an outage on our side sends
# the user off checking a file that was perfectly fine.
return _skip(path, ckey, name, cat,
m.get("error") or "no readable content could be extracted",
cacheable=not m.get("error"))
result = {"status": "kept", "name": name, "category": cat,
"topics": m.get("topics") or [], "notes": notes}
cov = "; ".join(x for x in (m.get("coverage"), cut_note) if x)
if cov:
result["coverage"] = cov
cache.store(path, ckey, result)
return result
def _ckey(prompt: str) -> str:
"""Cache key for one file's notes: the request plus the identity of the brief that
produced them. Without the second half, fixing the note-taking prompt changes
nothing for any file already in the cache."""
return f"{prompt}\x00map={MAP_PROMPT_ID}"
def _read_one(prompt: str, path: str, name: str):
"""Cache lookup, then extraction. No model is called here.
Extraction is separated from note-taking so the chunk budget can be shared out
once the real sizes are known. Sharing it by file size on disk does not work: the
same 9 MB is 421000 characters of report text or a single photograph.
Returns ("cached", result) or ("info", info)."""
hit = cache.load(path, _ckey(prompt))
if hit is not None:
print(f"[LUNA] cache hit for «{name}» - extraction and MAP skipped")
return "cached", hit
info = extract(path, name)
print(f"[LUNA] read «{name}» ({info['category']}, kind={info['kind']}"
+ (f", {len(info['payload'])} chars" if info["kind"] == "text" else "") + ")")
return "info", info
_MEDIA_CHUNK_ESTIMATE = 4 # 30 minutes of speech, the transcription cap
def _allocate(infos: dict, budget: int) -> dict:
"""Share MAP_CHUNK_BUDGET out between the files that are longer than one call.
One chunk each first, so no file is skipped, then the rest handed round in turns to
whoever still wants more. A single long document therefore gets the whole budget,
and a hundred short files are unaffected."""
need = {}
for nm, info in infos.items():
if info.get("kind") == "text":
need[nm] = max(1, -(-len(info["payload"]) // PER_FILE_CHARS))
elif info.get("kind") == "media":
# A transcript's length is unknown until Whisper has run, and it is the one
# input that arrives long: TRANSCRIBE_MAX_SECONDS is 30 minutes of speech,
# roughly 4 calls' worth. Reserved up front, and unused reservations cost
# nothing because _map_long_text only splits what is actually there.
need[nm] = _MEDIA_CHUNK_ESTIMATE
give = {nm: 1 for nm in need}
left = budget - len(give)
while left > 0 and any(give[nm] < min(need[nm], MAX_CHUNKS_PER_FILE) for nm in need):
for nm in need:
if left <= 0:
break
if give[nm] < min(need[nm], MAX_CHUNKS_PER_FILE):
give[nm] += 1
left -= 1
short = [nm for nm in need if give[nm] < need[nm]]
if short:
print(f"[LUNA] chunk budget {budget} does not cover {len(short)} long file(s); "
f"they are read in part: "
+ ", ".join(f"{nm} {give[nm]}/{need[nm]}" for nm in short[:5]))
return give
def _resolve(files) -> list[tuple[str, str]]:
"""Normalize Gradio's file input → [(path, display_name)]."""
out = []
for f in files or []:
if isinstance(f, str):
out.append((f, os.path.basename(f)))
elif isinstance(f, dict):
p = f.get("path") or f.get("name")
if p:
out.append((p, f.get("orig_name") or os.path.basename(p)))
elif hasattr(f, "name"):
out.append((f.name, os.path.basename(f.name)))
return out
def _validate(resolved) -> str | None:
if not resolved:
return "Please attach at least one file."
if len(resolved) > MAX_FILES:
return f"Too many files - the limit is {MAX_FILES}."
total = 0
missing = []
for path, name in resolved:
try:
sz = os.path.getsize(path)
except FileNotFoundError:
# The Space purges uploads after an hour (app.py delete_cache). The website
# uploads at wizard step 1 and builds at step 4, so an idle user arrives here
# with paths that no longer exist. Skipping them silently let validation pass,
# every file then failed to read, and REDUCE produced a polite "nothing
# relevant was found" Note. That Note is not empty, so the worker stored it
# and charged one of the user's three daily builds. Fail loudly instead: the
# worker only charges once a Note comes back.
missing.append(name)
continue
except OSError:
continue
total += sz
cat = category_of(name)
cap = MAX_MEDIA_MB if cat in ("audio", "video") else MAX_FILE_MB
if sz > cap * 1024 * 1024:
return f"'{name}' is too large ({sz // 1024 // 1024} MB; limit {cap} MB)."
if missing:
n = len(missing)
shown = ", ".join(missing[:3]) + (" and more" if n > 3 else "")
return (f"{n} file{'s' if n != 1 else ''} expired before the build started "
f"({shown}). Uploads are kept for one hour, please attach them again.")
if total > MAX_TOTAL_MB * 1024 * 1024:
return f"Total upload too large ({total // 1024 // 1024} MB; limit {MAX_TOTAL_MB} MB)."
return None
def run(files, prompt, token):
"""Streaming generator → yields (status, note). The final yield carries the Note."""
# The website may append format directives after a ⟦FORMAT⟧ sentinel. Split them off
# so language detection + synthesis see only the user's real words (the
# directives are English meta and must not change the Note's language).
raw = (prompt or "").strip()
base, _, fmt = raw.partition("⟦FORMAT⟧")
base, fmt = base.strip(), fmt.strip()
print(f"\n[LUNA] ========== NEW JOB REQUEST ==========")
print(f"[LUNA] prompt={len(base)} chars" + (f" fmt={len(fmt)} chars" if fmt else ""))
why = check_usage_token(token, "build")
if why != "ok":
print(f"[LUNA] job REJECTED - token {why}")
if why == "expired":
yield ("error", "⚠️ This task waited too long before LUNA could start it. "
"Nothing was used up - please try again.")
elif why == "unconfigured":
# Ours, not theirs: the Space has no signing secret, so it cannot tell a
# real request from anyone else's. Say so rather than blame their session.
yield ("error", "⚠️ LUNA is misconfigured on the server and cannot accept "
"tasks right now. Nothing was used up.")
else:
yield ("error", "⚠️ This task could not be authorized. Please reload and sign in again.")
return
if not base:
print("[LUNA] job REJECTED - empty prompt")
yield ("error", "⚠️ Tell LUNA what to build first.")
return
resolved = _resolve(files)
err = _validate(resolved)
if err:
print(f"[LUNA] job REJECTED - {err}")
yield ("error", f"⚠️ {err}")
return
n = len(resolved)
print(f"[LUNA] {n} file(s): " + ", ".join(nm for _, nm in resolved[:25])
+ (" …" if n > 25 else ""))
cache.evict_stale()
yield (f"⏳ Reading {n} file{'s' if n != 1 else ''}…", "")
deadline = time.monotonic() + BUILD_MAP_DEADLINE_SEC
t_start = time.monotonic()
def _timed_out(nm):
return {"status": "skipped", "name": nm, "category": "other",
"reason": "the build ran out of time before this file was noted", "notes": ""}
# Phase 1: read every file (cache, then extraction). No model is called yet, so the
# real sizes are known before a single token is spent.
kept, skipped, done = [], [], 0
infos, cached = {}, {}
pool = ThreadPoolExecutor(max_workers=MAP_WORKERS)
futures = {pool.submit(_read_one, base, p, nm): (p, nm) for p, nm in resolved}
try:
for fut in as_completed(futures, timeout=max(1.0, deadline - time.monotonic())):
path, nm = futures[fut]
try:
kind, payload = fut.result()
except Exception as e:
print(f"[LUNA] file «{nm}» failed to read: {type(e).__name__}: {e}")
skipped.append({"status": "skipped", "name": nm, "category": "other",
"reason": f"processing error ({type(e).__name__})", "notes": ""})
done += 1
continue
if kind == "cached":
cached[nm] = payload
else:
infos[nm] = payload
done += 1
yield (f"⏳ Read {done}/{n} files…", "")
except FutTimeout:
late = [nm for fut, (_, nm) in futures.items() if not fut.done()]
print(f"[LUNA] reading ran past {BUILD_MAP_DEADLINE_SEC}s: {len(late)} file(s) abandoned")
skipped.extend(_timed_out(nm) for nm in late)
finally:
# Never wait: a reader stuck in a parser must not hold the build. Abandoned
# threads finish on their own and their results are simply not used.
pool.shutdown(wait=False, cancel_futures=True)
print(f"[LUNA] stage=read ms={int((time.monotonic() - t_start) * 1000)}")
# Phase 2: share the chunk budget out, then take notes on everything at once.
give = _allocate(infos, MAP_CHUNK_BUDGET)
for res in cached.values():
(kept if res.get("status") == "kept" else skipped).append(res)
done = len(cached) + len(skipped)
note_cap = _note_cap(n)
if infos:
yield (f"⏳ Taking notes on {len(infos)} file{'s' if len(infos) != 1 else ''}…", "")
by_name = {nm: p for p, nm in resolved}
t_map = time.monotonic()
pool = ThreadPoolExecutor(max_workers=MAP_WORKERS)
futures = {pool.submit(_process_one, base, by_name[nm], nm,
give.get(nm, 1), info, note_cap): nm
for nm, info in infos.items()}
try:
for fut in as_completed(futures, timeout=max(1.0, deadline - time.monotonic())):
nm = futures[fut]
try:
res = fut.result()
except Exception as e:
print(f"[LUNA] file «{nm}» crashed: {type(e).__name__}: {e}")
res = {"status": "skipped", "name": nm, "category": "other",
"reason": f"processing error ({type(e).__name__})", "notes": ""}
(kept if res["status"] == "kept" else skipped).append(res)
done += 1
yield (f"⏳ Noted {done}/{n} files - {len(kept)} readable so far…", "")
except FutTimeout:
late = [nm for fut, nm in futures.items() if not fut.done()]
print(f"[LUNA] note-taking ran past the {BUILD_MAP_DEADLINE_SEC}s deadline: "
f"{len(late)} file(s) abandoned, writing the Note from the rest")
skipped.extend(_timed_out(nm) for nm in late)
yield (f"⏳ Out of time: writing the Note from {len(kept)} of {n} files…", "")
finally:
pool.shutdown(wait=False, cancel_futures=True)
print(f"[LUNA] stage=map ms={int((time.monotonic() - t_map) * 1000)}")
print(f"[LUNA] extraction complete - {len(kept)} readable, {len(skipped)} unreadable")
# Every readable file (with its notes) is now in `kept`; REDUCE decides what to
# actually use. `kept` is empty only when no file could be read at all - then REDUCE
# produces the honest "nothing could be read" Note from the `skipped` reasons.
yield (f"✍️ Writing the Note from {len(kept)} file{'s' if len(kept) != 1 else ''}…", "")
# `status` carries the one fact the website could never see before: that synthesis
# failed and what the user is holding is not a Note. The final marker says so, and
# the worker gives the daily build back instead of charging for raw material.
status: dict = {}
last = ""
t_reduce = time.monotonic()
for note in reduce_stream(base, kept, skipped, fmt, status):
last = note
yield ("✍️", note)
print(f"[LUNA] stage=reduce ms={int((time.monotonic() - t_reduce) * 1000)} "
f"total ms={int((time.monotonic() - t_start) * 1000)}")
if status.get("degraded"):
reason = str(status.get("reason") or "synthesis failed")[:200]
print(f"[LUNA] ========== JOB DEGRADED - {reason} ==========\n")
yield (f"⚠️ degraded: {reason}", last)
return
print(f"[LUNA] ========== JOB DONE - Note {len(last)} chars ==========\n")
yield ("✅ done", last)