Spaces:
Paused
Paused
Malina
Claude Fable 5.1
Scoped tokens, bounded archives and frames, build deadline, circuit breaker, honest note caps
da9de03 Download pipeline.py from NextNeural/LUNA: direct link, hf CLI and curl.
- Browser
- Download file 25 kB
-
https://huggingface.co/spaces/NextNeural/LUNA/resolve/main/pipeline.py
- Command line
-
hf download hf://spaces/NextNeural/LUNA/pipeline.py
-
curl -L -o pipeline.py https://huggingface.co/spaces/NextNeural/LUNA/resolve/main/pipeline.py
25 kB
| # Orchestration: categorize → extract+MAP every file in parallel → REDUCE into the | |
| # final Note. Exposed as a streaming generator that yields (status, note) for Gradio; | |
| # the website's worker forwards `status` as a progress marker and accumulates `note`. | |
| import os | |
| import threading | |
| import time | |
| from concurrent.futures import ThreadPoolExecutor, TimeoutError as FutTimeout, as_completed | |
| import cache | |
| from auth import check_usage_token | |
| from config import ( | |
| BUILD_MAP_DEADLINE_SEC, MAP_CHUNK_BUDGET, MAP_NOTE_CHARS, MAP_WORKERS, | |
| MAX_CHUNKS_PER_FILE, MAX_FILES, MAX_FILE_MB, MAX_MEDIA_MB, MAX_TOTAL_MB, | |
| PER_FILE_CHARS, REDUCE_BUDGET, TRANSCRIBE_MAX_SECONDS, TRANSCRIBE_WORKERS, | |
| ) | |
| from extract import category_of, extract | |
| from frames import sample_frames | |
| from models import get_whisper | |
| from subtitles import embedded_transcript | |
| from synthesis import (MAP_PROMPT_ID, map_image, map_pages, map_text, map_video, | |
| reduce_stream) | |
| _transcribe_sem = threading.Semaphore(TRANSCRIBE_WORKERS) | |
| def _transcribe(path: str) -> str: | |
| # A subtitle track already in the container beats anything we would transcribe: | |
| # it is free, it is faster, and the spelling of names and numbers is right. | |
| subs = embedded_transcript(path) | |
| if subs: | |
| return subs | |
| # Keyed by the file alone, not by the prompt: rebuilding the same video with a | |
| # reworded prompt used to run Whisper over it again from scratch. | |
| cached = cache.load_raw(path, "transcript") | |
| if isinstance(cached, str): | |
| print(f"[LUNA] transcript cache hit for {os.path.basename(path)}") | |
| return cached | |
| model = get_whisper() | |
| if model is None: | |
| return "" | |
| with _transcribe_sem: | |
| try: | |
| segments, _info = model.transcribe(path, beam_size=1, vad_filter=True) | |
| parts, cut = [], False | |
| for seg in segments: | |
| parts.append(seg.text) | |
| if seg.end and seg.end > TRANSCRIBE_MAX_SECONDS: | |
| cut = True | |
| break | |
| text = " ".join(p.strip() for p in parts).strip() | |
| if text and cut: | |
| # Said in the transcript itself, so MAP and REDUCE both know the | |
| # recording goes on. It used to stop silently at the half hour. | |
| text += transcript_cut_note() | |
| # Stored even when empty: a silent clip stays silent, and re-proving it | |
| # costs another full pass. | |
| cache.store_raw(path, "transcript", text) | |
| return text | |
| except Exception as e: | |
| print(f"[LUNA] transcribe error on {os.path.basename(path)}: {e}") | |
| return "" | |
| def transcript_cut_note() -> str: | |
| m = TRANSCRIBE_MAX_SECONDS // 60 | |
| return (f"\n\n[The transcript stops at {m} minutes. The recording continues beyond " | |
| f"this point and the rest was not transcribed.]") | |
| def _skip(path, prompt, name, cat, reason, cacheable=True): | |
| """A skipped file, optionally remembered. | |
| Only the verdicts that cannot change are stored. "This is a binary file" and | |
| "no speech could be transcribed" will say the same thing on every retry, and | |
| proving it again costs another extraction, or another full Whisper pass. A | |
| failure that came from our side - a model that could not be reached, a missing | |
| dependency - must NOT be remembered, or one outage would mark a perfectly good | |
| file unreadable for the rest of the cache's life.""" | |
| out = {"status": "skipped", "name": name, "category": cat, | |
| "reason": reason, "notes": ""} | |
| if cacheable: | |
| cache.store(path, prompt, out) | |
| return out | |
| def _split(content: str, parts: int) -> list[str]: | |
| """Cut a long extraction into `parts` pieces of at most PER_FILE_CHARS. | |
| Prefers a line break near the boundary: a cut through the middle of a sentence | |
| costs the notes on both sides of it, and documents break at lines for free.""" | |
| out, i, n = [], 0, len(content) | |
| for k in range(parts): | |
| end = min(i + PER_FILE_CHARS, n) | |
| if end < n: | |
| window = content.rfind("\n", i + PER_FILE_CHARS - 800, end) | |
| if window > i: | |
| end = window | |
| out.append(content[i:end]) | |
| i = end | |
| if i >= n: | |
| break | |
| return out | |
| def _map_long_text(prompt: str, name: str, cat: str, content: str, allowance: int) -> dict: | |
| """MAP one text file, in parts when it is longer than a single call can read. | |
| Before this, a file was whatever its first PER_FILE_CHARS said and the rest did not | |
| exist: a 143-page report reached synthesis as its own cover and table of contents. | |
| Parts are noted concurrently and their notes concatenated in order, so the file | |
| still arrives at REDUCE as one block.""" | |
| total = len(content) | |
| needed = max(1, -(-total // PER_FILE_CHARS)) # ceil | |
| parts = max(1, min(needed, allowance, MAX_CHUNKS_PER_FILE)) | |
| if parts == 1: | |
| dropped = max(0, total - PER_FILE_CHARS) | |
| out = map_text(prompt, name, cat, content[:PER_FILE_CHARS], 1, 1, dropped) | |
| if dropped: | |
| out["coverage"] = (f"only the first {PER_FILE_CHARS} of {total} characters " | |
| f"were read") | |
| return out | |
| chunks = _split(content, parts) | |
| read = sum(len(c) for c in chunks) | |
| dropped = max(0, total - read) | |
| print(f"[LUNA] «{name}» is {total} chars: reading it in {len(chunks)} parts" | |
| + (f", {dropped} chars beyond the budget" if dropped else "")) | |
| results = [None] * len(chunks) | |
| with ThreadPoolExecutor(max_workers=min(MAP_WORKERS, len(chunks))) as pool: | |
| futs = {pool.submit(map_text, prompt, name, cat, ch, i + 1, len(chunks), dropped): i | |
| for i, ch in enumerate(chunks)} | |
| for fut in as_completed(futs): | |
| i = futs[fut] | |
| try: | |
| results[i] = fut.result() | |
| except Exception as e: | |
| print(f"[LUNA] part {i+1} of «{name}» failed: {type(e).__name__}: {e}") | |
| notes = [r["notes"].strip() for r in results if r and (r.get("notes") or "").strip()] | |
| topics: list[str] = [] | |
| for r in results: | |
| for t in (r or {}).get("topics") or []: | |
| if t not in topics: | |
| topics.append(t) | |
| if not notes: | |
| err = next((r.get("error") for r in results if r and r.get("error")), None) | |
| return {"topics": [], "notes": "", "error": err} | |
| out = {"topics": topics[:8], "notes": "\n".join(notes)} | |
| # Say what was actually read, and say it separately from what was actually noted: | |
| # a part whose model call came back empty is not the same as a part nobody read, | |
| # and rolling the two together is how "read 421110 of 421110 characters" ended up | |
| # printed over notes that were missing one part of twenty-seven. | |
| cov = [] | |
| if dropped: | |
| cov.append(f"read {read} of {total} characters") | |
| if len(notes) < len(chunks): | |
| cov.append(f"notes cover {len(notes)} of {len(chunks)} parts") | |
| if cov: | |
| out["coverage"] = "; ".join(cov) | |
| return out | |
| def _note_cap(n_files: int) -> int: | |
| """One file may keep notes worth the whole REDUCE budget; many share it. The flat | |
| 24000 cap used to cut a lone 27-part report to about eight parts, silently.""" | |
| return max(MAP_NOTE_CHARS, REDUCE_BUDGET // max(1, n_files)) | |
| def _cap_notes(notes: str, cap: int) -> tuple[str, str]: | |
| """Cut notes to `cap` at a line boundary and say what was lost. Returns (notes, coverage).""" | |
| notes = (notes or "").strip() | |
| if len(notes) <= cap: | |
| return notes, "" | |
| cut = notes.rfind("\n", 0, cap) | |
| if cut < cap // 2: | |
| cut = cap | |
| kept = notes[:cut].rstrip() | |
| return kept, f"notes cut to {len(kept)} of {len(notes)} characters" | |
| def _process_one(prompt: str, path: str, name: str, allowance: int = 1, | |
| info: dict | None = None, note_cap: int = MAP_NOTE_CHARS) -> dict: | |
| """Full per-file pipeline (extract → MAP). Returns a 'kept' or 'skipped' dict. | |
| Every readable file is 'kept' WITH its extracted notes; the REDUCE stage (which sees | |
| all files together) makes the include/exclude call. Only genuinely unreadable files are | |
| 'skipped', so a shaky per-file call can never silently destroy a file's content.""" | |
| if info is None: | |
| info = extract(path, name) | |
| ckey = _ckey(prompt) | |
| cat = info["category"] | |
| print(f"[LUNA] processing «{name}» ({cat}, kind={info['kind']})") | |
| if info["kind"] == "error": | |
| print(f"[LUNA] «{name}» unreadable: {info['payload']}") | |
| # "couldn't read (SomeError)" means an exception on our side, which a later | |
| # run may not hit; everything else is a property of the file itself. | |
| return _skip(path, ckey, name, cat, info["payload"], | |
| cacheable=not str(info["payload"]).startswith("couldn't read (")) | |
| if info["kind"] == "pages": | |
| m = map_pages(prompt, name, info["payload"]) | |
| elif info["kind"] == "image": | |
| m = map_image(prompt, name, info["payload"]) | |
| elif info["kind"] == "media": | |
| if cat == "video": | |
| frames = sample_frames(info["payload"]) | |
| transcript = _transcribe(info["payload"]) | |
| print(f"[LUNA] video «{name}»: {len(frames)} frame(s), " | |
| f"{len(transcript)} transcript chars") | |
| if frames: | |
| m = map_video(prompt, name, transcript, frames) | |
| if len(transcript) > PER_FILE_CHARS and allowance > 1: | |
| # The vision call reads the frames with the FIRST window of speech; | |
| # the rest of a long recording goes through the text path with the | |
| # file's remaining allowance, so a 30-minute talk is not reduced to | |
| # its first 16000 characters just because it has a picture track. | |
| rest = _map_long_text(prompt, name, cat, transcript[PER_FILE_CHARS:], | |
| allowance - 1) | |
| if (rest.get("notes") or "").strip(): | |
| m = {"topics": list(dict.fromkeys((m.get("topics") or []) + (rest.get("topics") or [])))[:8], | |
| "notes": ((m.get("notes") or "").strip() + "\n" + rest["notes"].strip()).strip(), | |
| "error": m.get("error")} | |
| if rest.get("coverage"): | |
| m["coverage"] = rest["coverage"] | |
| elif transcript: | |
| m = _map_long_text(prompt, name, cat, transcript, allowance) | |
| else: | |
| # NOT remembered. "No frames or speech" is exactly what a missing | |
| # PyAV or a Whisper that failed to load also looks like from here, | |
| # and caching that would mark a perfectly good recording unreadable | |
| # for the life of the entry. Nothing is lost by re-checking: the | |
| # transcript cache above already holds the expensive half. | |
| return _skip(path, ckey, name, cat, | |
| "the video couldn't be read (no frames or speech)", | |
| cacheable=False) | |
| else: # audio | |
| transcript = _transcribe(info["payload"]) | |
| if not transcript: | |
| # Same reasoning: an empty transcript can mean a silent file or a | |
| # Whisper that never loaded, and only one of those is the file's | |
| # fault. The transcript cache already saves the second pass. | |
| return _skip(path, ckey, name, cat, "no speech could be transcribed", | |
| cacheable=False) | |
| m = _map_long_text(prompt, name, cat, transcript, allowance) | |
| else: # text | |
| m = _map_long_text(prompt, name, cat, info["payload"], allowance) | |
| notes, cut_note = _cap_notes(m.get("notes") or "", note_cap) | |
| if not notes: | |
| # MAP produced no extraction. If the stage said why (a model call that could not | |
| # be reached), pass that through: blaming the file for an outage on our side sends | |
| # the user off checking a file that was perfectly fine. | |
| return _skip(path, ckey, name, cat, | |
| m.get("error") or "no readable content could be extracted", | |
| cacheable=not m.get("error")) | |
| result = {"status": "kept", "name": name, "category": cat, | |
| "topics": m.get("topics") or [], "notes": notes} | |
| cov = "; ".join(x for x in (m.get("coverage"), cut_note) if x) | |
| if cov: | |
| result["coverage"] = cov | |
| cache.store(path, ckey, result) | |
| return result | |
| def _ckey(prompt: str) -> str: | |
| """Cache key for one file's notes: the request plus the identity of the brief that | |
| produced them. Without the second half, fixing the note-taking prompt changes | |
| nothing for any file already in the cache.""" | |
| return f"{prompt}\x00map={MAP_PROMPT_ID}" | |
| def _read_one(prompt: str, path: str, name: str): | |
| """Cache lookup, then extraction. No model is called here. | |
| Extraction is separated from note-taking so the chunk budget can be shared out | |
| once the real sizes are known. Sharing it by file size on disk does not work: the | |
| same 9 MB is 421000 characters of report text or a single photograph. | |
| Returns ("cached", result) or ("info", info).""" | |
| hit = cache.load(path, _ckey(prompt)) | |
| if hit is not None: | |
| print(f"[LUNA] cache hit for «{name}» - extraction and MAP skipped") | |
| return "cached", hit | |
| info = extract(path, name) | |
| print(f"[LUNA] read «{name}» ({info['category']}, kind={info['kind']}" | |
| + (f", {len(info['payload'])} chars" if info["kind"] == "text" else "") + ")") | |
| return "info", info | |
| _MEDIA_CHUNK_ESTIMATE = 4 # 30 minutes of speech, the transcription cap | |
| def _allocate(infos: dict, budget: int) -> dict: | |
| """Share MAP_CHUNK_BUDGET out between the files that are longer than one call. | |
| One chunk each first, so no file is skipped, then the rest handed round in turns to | |
| whoever still wants more. A single long document therefore gets the whole budget, | |
| and a hundred short files are unaffected.""" | |
| need = {} | |
| for nm, info in infos.items(): | |
| if info.get("kind") == "text": | |
| need[nm] = max(1, -(-len(info["payload"]) // PER_FILE_CHARS)) | |
| elif info.get("kind") == "media": | |
| # A transcript's length is unknown until Whisper has run, and it is the one | |
| # input that arrives long: TRANSCRIBE_MAX_SECONDS is 30 minutes of speech, | |
| # roughly 4 calls' worth. Reserved up front, and unused reservations cost | |
| # nothing because _map_long_text only splits what is actually there. | |
| need[nm] = _MEDIA_CHUNK_ESTIMATE | |
| give = {nm: 1 for nm in need} | |
| left = budget - len(give) | |
| while left > 0 and any(give[nm] < min(need[nm], MAX_CHUNKS_PER_FILE) for nm in need): | |
| for nm in need: | |
| if left <= 0: | |
| break | |
| if give[nm] < min(need[nm], MAX_CHUNKS_PER_FILE): | |
| give[nm] += 1 | |
| left -= 1 | |
| short = [nm for nm in need if give[nm] < need[nm]] | |
| if short: | |
| print(f"[LUNA] chunk budget {budget} does not cover {len(short)} long file(s); " | |
| f"they are read in part: " | |
| + ", ".join(f"{nm} {give[nm]}/{need[nm]}" for nm in short[:5])) | |
| return give | |
| def _resolve(files) -> list[tuple[str, str]]: | |
| """Normalize Gradio's file input → [(path, display_name)].""" | |
| out = [] | |
| for f in files or []: | |
| if isinstance(f, str): | |
| out.append((f, os.path.basename(f))) | |
| elif isinstance(f, dict): | |
| p = f.get("path") or f.get("name") | |
| if p: | |
| out.append((p, f.get("orig_name") or os.path.basename(p))) | |
| elif hasattr(f, "name"): | |
| out.append((f.name, os.path.basename(f.name))) | |
| return out | |
| def _validate(resolved) -> str | None: | |
| if not resolved: | |
| return "Please attach at least one file." | |
| if len(resolved) > MAX_FILES: | |
| return f"Too many files - the limit is {MAX_FILES}." | |
| total = 0 | |
| missing = [] | |
| for path, name in resolved: | |
| try: | |
| sz = os.path.getsize(path) | |
| except FileNotFoundError: | |
| # The Space purges uploads after an hour (app.py delete_cache). The website | |
| # uploads at wizard step 1 and builds at step 4, so an idle user arrives here | |
| # with paths that no longer exist. Skipping them silently let validation pass, | |
| # every file then failed to read, and REDUCE produced a polite "nothing | |
| # relevant was found" Note. That Note is not empty, so the worker stored it | |
| # and charged one of the user's three daily builds. Fail loudly instead: the | |
| # worker only charges once a Note comes back. | |
| missing.append(name) | |
| continue | |
| except OSError: | |
| continue | |
| total += sz | |
| cat = category_of(name) | |
| cap = MAX_MEDIA_MB if cat in ("audio", "video") else MAX_FILE_MB | |
| if sz > cap * 1024 * 1024: | |
| return f"'{name}' is too large ({sz // 1024 // 1024} MB; limit {cap} MB)." | |
| if missing: | |
| n = len(missing) | |
| shown = ", ".join(missing[:3]) + (" and more" if n > 3 else "") | |
| return (f"{n} file{'s' if n != 1 else ''} expired before the build started " | |
| f"({shown}). Uploads are kept for one hour, please attach them again.") | |
| if total > MAX_TOTAL_MB * 1024 * 1024: | |
| return f"Total upload too large ({total // 1024 // 1024} MB; limit {MAX_TOTAL_MB} MB)." | |
| return None | |
| def run(files, prompt, token): | |
| """Streaming generator → yields (status, note). The final yield carries the Note.""" | |
| # The website may append format directives after a ⟦FORMAT⟧ sentinel. Split them off | |
| # so language detection + synthesis see only the user's real words (the | |
| # directives are English meta and must not change the Note's language). | |
| raw = (prompt or "").strip() | |
| base, _, fmt = raw.partition("⟦FORMAT⟧") | |
| base, fmt = base.strip(), fmt.strip() | |
| print(f"\n[LUNA] ========== NEW JOB REQUEST ==========") | |
| print(f"[LUNA] prompt={len(base)} chars" + (f" fmt={len(fmt)} chars" if fmt else "")) | |
| why = check_usage_token(token, "build") | |
| if why != "ok": | |
| print(f"[LUNA] job REJECTED - token {why}") | |
| if why == "expired": | |
| yield ("error", "⚠️ This task waited too long before LUNA could start it. " | |
| "Nothing was used up - please try again.") | |
| elif why == "unconfigured": | |
| # Ours, not theirs: the Space has no signing secret, so it cannot tell a | |
| # real request from anyone else's. Say so rather than blame their session. | |
| yield ("error", "⚠️ LUNA is misconfigured on the server and cannot accept " | |
| "tasks right now. Nothing was used up.") | |
| else: | |
| yield ("error", "⚠️ This task could not be authorized. Please reload and sign in again.") | |
| return | |
| if not base: | |
| print("[LUNA] job REJECTED - empty prompt") | |
| yield ("error", "⚠️ Tell LUNA what to build first.") | |
| return | |
| resolved = _resolve(files) | |
| err = _validate(resolved) | |
| if err: | |
| print(f"[LUNA] job REJECTED - {err}") | |
| yield ("error", f"⚠️ {err}") | |
| return | |
| n = len(resolved) | |
| print(f"[LUNA] {n} file(s): " + ", ".join(nm for _, nm in resolved[:25]) | |
| + (" …" if n > 25 else "")) | |
| cache.evict_stale() | |
| yield (f"⏳ Reading {n} file{'s' if n != 1 else ''}…", "") | |
| deadline = time.monotonic() + BUILD_MAP_DEADLINE_SEC | |
| t_start = time.monotonic() | |
| def _timed_out(nm): | |
| return {"status": "skipped", "name": nm, "category": "other", | |
| "reason": "the build ran out of time before this file was noted", "notes": ""} | |
| # Phase 1: read every file (cache, then extraction). No model is called yet, so the | |
| # real sizes are known before a single token is spent. | |
| kept, skipped, done = [], [], 0 | |
| infos, cached = {}, {} | |
| pool = ThreadPoolExecutor(max_workers=MAP_WORKERS) | |
| futures = {pool.submit(_read_one, base, p, nm): (p, nm) for p, nm in resolved} | |
| try: | |
| for fut in as_completed(futures, timeout=max(1.0, deadline - time.monotonic())): | |
| path, nm = futures[fut] | |
| try: | |
| kind, payload = fut.result() | |
| except Exception as e: | |
| print(f"[LUNA] file «{nm}» failed to read: {type(e).__name__}: {e}") | |
| skipped.append({"status": "skipped", "name": nm, "category": "other", | |
| "reason": f"processing error ({type(e).__name__})", "notes": ""}) | |
| done += 1 | |
| continue | |
| if kind == "cached": | |
| cached[nm] = payload | |
| else: | |
| infos[nm] = payload | |
| done += 1 | |
| yield (f"⏳ Read {done}/{n} files…", "") | |
| except FutTimeout: | |
| late = [nm for fut, (_, nm) in futures.items() if not fut.done()] | |
| print(f"[LUNA] reading ran past {BUILD_MAP_DEADLINE_SEC}s: {len(late)} file(s) abandoned") | |
| skipped.extend(_timed_out(nm) for nm in late) | |
| finally: | |
| # Never wait: a reader stuck in a parser must not hold the build. Abandoned | |
| # threads finish on their own and their results are simply not used. | |
| pool.shutdown(wait=False, cancel_futures=True) | |
| print(f"[LUNA] stage=read ms={int((time.monotonic() - t_start) * 1000)}") | |
| # Phase 2: share the chunk budget out, then take notes on everything at once. | |
| give = _allocate(infos, MAP_CHUNK_BUDGET) | |
| for res in cached.values(): | |
| (kept if res.get("status") == "kept" else skipped).append(res) | |
| done = len(cached) + len(skipped) | |
| note_cap = _note_cap(n) | |
| if infos: | |
| yield (f"⏳ Taking notes on {len(infos)} file{'s' if len(infos) != 1 else ''}…", "") | |
| by_name = {nm: p for p, nm in resolved} | |
| t_map = time.monotonic() | |
| pool = ThreadPoolExecutor(max_workers=MAP_WORKERS) | |
| futures = {pool.submit(_process_one, base, by_name[nm], nm, | |
| give.get(nm, 1), info, note_cap): nm | |
| for nm, info in infos.items()} | |
| try: | |
| for fut in as_completed(futures, timeout=max(1.0, deadline - time.monotonic())): | |
| nm = futures[fut] | |
| try: | |
| res = fut.result() | |
| except Exception as e: | |
| print(f"[LUNA] file «{nm}» crashed: {type(e).__name__}: {e}") | |
| res = {"status": "skipped", "name": nm, "category": "other", | |
| "reason": f"processing error ({type(e).__name__})", "notes": ""} | |
| (kept if res["status"] == "kept" else skipped).append(res) | |
| done += 1 | |
| yield (f"⏳ Noted {done}/{n} files - {len(kept)} readable so far…", "") | |
| except FutTimeout: | |
| late = [nm for fut, nm in futures.items() if not fut.done()] | |
| print(f"[LUNA] note-taking ran past the {BUILD_MAP_DEADLINE_SEC}s deadline: " | |
| f"{len(late)} file(s) abandoned, writing the Note from the rest") | |
| skipped.extend(_timed_out(nm) for nm in late) | |
| yield (f"⏳ Out of time: writing the Note from {len(kept)} of {n} files…", "") | |
| finally: | |
| pool.shutdown(wait=False, cancel_futures=True) | |
| print(f"[LUNA] stage=map ms={int((time.monotonic() - t_map) * 1000)}") | |
| print(f"[LUNA] extraction complete - {len(kept)} readable, {len(skipped)} unreadable") | |
| # Every readable file (with its notes) is now in `kept`; REDUCE decides what to | |
| # actually use. `kept` is empty only when no file could be read at all - then REDUCE | |
| # produces the honest "nothing could be read" Note from the `skipped` reasons. | |
| yield (f"✍️ Writing the Note from {len(kept)} file{'s' if len(kept) != 1 else ''}…", "") | |
| # `status` carries the one fact the website could never see before: that synthesis | |
| # failed and what the user is holding is not a Note. The final marker says so, and | |
| # the worker gives the daily build back instead of charging for raw material. | |
| status: dict = {} | |
| last = "" | |
| t_reduce = time.monotonic() | |
| for note in reduce_stream(base, kept, skipped, fmt, status): | |
| last = note | |
| yield ("✍️", note) | |
| print(f"[LUNA] stage=reduce ms={int((time.monotonic() - t_reduce) * 1000)} " | |
| f"total ms={int((time.monotonic() - t_start) * 1000)}") | |
| if status.get("degraded"): | |
| reason = str(status.get("reason") or "synthesis failed")[:200] | |
| print(f"[LUNA] ========== JOB DEGRADED - {reason} ==========\n") | |
| yield (f"⚠️ degraded: {reason}", last) | |
| return | |
| print(f"[LUNA] ========== JOB DONE - Note {len(last)} chars ==========\n") | |
| yield ("✅ done", last) | |