LUNA / config.py
Malina
Claude Fable 5.1
Scoped tokens, bounded archives and frames, build deadline, circuit breaker, honest note caps
da9de03
Raw History Blame Contribute Delete
11.9 kB
# ─────────────────────────────────────────────
# LUNA - Latent Understanding & Narrative Assembly
# config.py
# ─────────────────────────────────────────────
import os
import tempfile
# ── Inference models (HF Inference Providers) ─────────────────────────────────
# Two-tier map-reduce keeps cost down: a cheap/fast model reads every file and
# pulls only the prompt-relevant material (MAP), then one strong-model call fuses
# the kept extracts into the final Note (REDUCE). Images are read by a VLM.
# Qwen2.5-7B-Instruct was configured here and is NOT served by any provider on our
# account any more, so every MAP call was failing. The failure is silent by design
# (map_text keeps the raw text on error), so the whole extraction stage had quietly
# degraded to "first 2000 raw characters" with nobody noticing. models.py now checks
# these names at startup and says so loudly.
#
# Not a reasoning model, deliberately. Qwen3-8B was tried and emits its thinking as
# tokens: at clarify's 160-token budget the whole budget went to the thinking and the
# answer came back empty, and at MAP's 900 it was merely unreliable. Llama-3.1-8B is
# cheap and available but hallucinated - it graded a costs spreadsheet as highly
# relevant to a lease question and wrote "Koszty w umowie najmu" over data that says
# nothing of the sort. For a tool whose whole promise is staying inside the files,
# that disqualifies it.
#
# A ROLE IS A LIST, not a name. On 2026-09-21 a build produced a raw stitch instead of
# a Note because the router answered "The requested model 'Qwen/Qwen2.5-72B-Instruct'
# is not supported by any provider you have enabled" - for a model that was, at that
# same hour, served live by deepinfra, novita and featherless-ai. The auto router keeps
# its own catalogue (router.huggingface.co/v1/models, 136 entries) and membership of it
# changes without notice. models.py used to classify that message as a permanent error,
# so one catalogue wobble killed the whole build with no second attempt and no
# alternative. Now each role has candidates, each candidate has providers to pin when
# the router refuses, and nothing about a build depends on one name being in favour.
def _role(env_name: str, default: list[str]) -> list[str]:
"""Env override wins and is tried first; the defaults stay as the fallback chain."""
override = os.getenv(env_name, "").strip()
names = [n.strip() for n in override.split(",") if n.strip()]
return names + [d for d in default if d not in names] if names else list(default)
# Text extraction. Qwen3-VL is the same model the vision path already uses: one name
# fewer to keep alive, 2 to 4 seconds per file against gemma's measured 25 to 63, and
# non-reasoning. gemma stays last: it extracts faithfully, it is merely slow.
MAP_MODELS = _role("LUNA_MAP_MODEL", [
"Qwen/Qwen3-VL-30B-A3B-Instruct",
"Qwen/Qwen3-Next-80B-A3B-Instruct",
"google/gemma-3-27b-it",
])
# Synthesis. Measured on LUNA's own REDUCE payload: GLM-4.7 writes real notes, Kimi is
# the closest second, Qwen3-Next is the cheap insurance. Qwen2.5-72B, the incumbent, is
# deliberately NOT here: slowest of the candidates, prose where notes were asked for,
# and it padded the Note with facts the files did not contain.
# NEVER put a reasoning model in this list. They spend the whole 4000-token budget on
# thinking and return zero visible characters, which looks exactly like an outage.
REDUCE_MODELS = _role("LUNA_REDUCE_MODEL", [
"zai-org/GLM-4.7",
"moonshotai/Kimi-K2.6",
"Qwen/Qwen3-Next-80B-A3B-Instruct",
])
VLM_MODELS = _role("LUNA_VLM_MODEL", [
"Qwen/Qwen3-VL-30B-A3B-Instruct",
])
# Providers to pin, in order, when the auto router says it cannot route a model. All
# verified live for these names on 2026-09-21; a name missing here simply gets no
# second chance at the auto router's expense.
MODEL_PROVIDERS = {
"zai-org/GLM-4.7": ["deepinfra", "novita", "featherless-ai"],
"moonshotai/Kimi-K2.6": ["deepinfra", "novita", "featherless-ai"],
"Qwen/Qwen3-Next-80B-A3B-Instruct": ["deepinfra", "novita", "featherless-ai"],
"Qwen/Qwen3-VL-30B-A3B-Instruct": ["deepinfra", "novita", "featherless-ai"],
"Qwen/Qwen2.5-72B-Instruct": ["deepinfra", "novita", "featherless-ai"],
"google/gemma-3-27b-it": ["deepinfra", "novita"],
}
# First entry of each role, kept for the call sites and the startup probe.
MAP_MODEL = MAP_MODELS[0]
REDUCE_MODEL = REDUCE_MODELS[0]
VLM_MODEL = VLM_MODELS[0]
# 900 was too small and failed silently. On a 16k-character document the model writes
# roughly 2850 characters of extraction, ran past the budget, and the JSON came back
# unterminated - which parsed as "relevance: none, notes: empty", so the file was
# reported to the user as unreadable. Deterministic, so retrying changed nothing.
MAP_MAX_TOKENS = int(os.getenv("LUNA_MAP_MAX_TOKENS", "2000"))
MAP_MAX_TOKENS_RETRY = int(os.getenv("LUNA_MAP_MAX_TOKENS_RETRY", "3200"))
REDUCE_MAX_TOKENS = 4000 # the final Note can be long
MAP_TEMPERATURE = 0.0
REDUCE_TEMPERATURE = 0.15
# Guards on the streaming reply. LLM_TIMEOUT covers reaching the provider, not a
# stream that opens and then goes quiet: one such stall sat for sixteen minutes in
# testing, which on the website would have burned the whole eight-minute budget and
# delivered nothing. STALL is the longest silence allowed between two pieces of text,
# MAX the longest the whole reply may take. Both stay under the website's own cap.
REDUCE_STALL_SECONDS = int(os.getenv("LUNA_REDUCE_STALL", "45"))
REDUCE_MAX_SECONDS = int(os.getenv("LUNA_REDUCE_MAX_SECONDS", "300"))
LLM_TIMEOUT = 180
VLM_FRAME_MAX_PX = 1024 # downscale an image's long side before sending (caps tokens)
# A PDF with no text layer is a scan. Its pages are handed to the vision model, which
# reads them far better than an OCR dependency would (NINA turned its OCR off for
# exactly that reason). Capped, because each page costs a picture worth of tokens.
PDF_SCAN_MAX_PAGES = int(os.getenv("LUNA_PDF_SCAN_MAX_PAGES", "6"))
# ── Video understanding (real frames, like NINA) ──────────────────────────────
# Videos are understood the way NINA does it: we sample keyframes and send them to
# the vision model together with any speech transcript - so even a silent screen
# recording is understood from what's on screen.
VIDEO_MAX_FRAMES = 8
VIDEO_FRAME_MAX_PX = 768
# ── Upload limits (re-checked here; the website also enforces them) ───────────
MAX_FILES = 100
MAX_FILE_MB = 25 # per non-media file
MAX_MEDIA_MB = 500 # per audio/video file
MAX_TOTAL_MB = 500 # whole task
# ── Extraction bounds (token/cost control) ────────────────────────────────────
PER_FILE_CHARS = 16000 # one MAP call reads at most this much text
# How much text may be pulled out of ONE file in total. This used to be PER_FILE_CHARS,
# i.e. a long document was read only as far as its first 16000 characters and the rest
# never existed. Measured on a 143-page report (420968 extractable characters): LUNA
# read 7 pages, shipped 4, and its entire view of the document was the cover, the
# copyright page and the table of contents. A long file is now read whole and handed to
# MAP in PER_FILE_CHARS chunks.
EXTRACT_MAX_CHARS = int(os.getenv("LUNA_EXTRACT_MAX_CHARS", "800000"))
# Ceiling on MAP calls spent on chunks in one build, shared out between the long files.
# 40 chunks is the 143-page report read end to end (27) with room to spare, and at 2 to
# 4 seconds per chunk over 8 workers it is well inside the website's 8-minute budget.
MAP_CHUNK_BUDGET = int(os.getenv("LUNA_MAP_CHUNK_BUDGET", "40"))
MAX_CHUNKS_PER_FILE = int(os.getenv("LUNA_MAX_CHUNKS_PER_FILE", "30"))
# Hard ceiling on one file's notes, a memory guard only. The real per-file limit is the
# share _reduce_user computes from the file count, which is the number that knows how
# much room there actually is: a flat 4000 threw away 80% of the budget REDUCE was
# ready to spend on a three-file task, and cut the notes mid-word doing it.
# pipeline raises it to REDUCE_BUDGET // n_files, so a one-file task keeps the notes
# of all 27 parts of a long report instead of the first eight, and says so when it
# still has to cut.
MAP_NOTE_CHARS = int(os.getenv("LUNA_MAP_NOTE_CHARS", "24000"))
REDUCE_BUDGET = 60000 # total chars of kept notes fed to REDUCE
TRANSCRIBE_MAX_SECONDS = 1800 # don't transcribe media longer than 30 min
# Reading and note-taking must finish inside this, whatever is still running is
# abandoned and listed as "ran out of time", and REDUCE starts on what there is. The
# website cancels the whole build at eight minutes; without this cap one slow provider
# on one file used to hold every other file's notes hostage past that point.
BUILD_MAP_DEADLINE_SEC = int(os.getenv("LUNA_BUILD_MAP_DEADLINE", "360"))
# ── Archive and document bounds ───────────────────────────────────────────────
# A ZIP is read member by member through real extractors; these stop a crafted or
# merely huge archive from filling the disk (a 25 MB upload can hold gigabytes).
ZIP_MEMBER_MAX_BYTES = 50 * 1024 * 1024
ZIP_TOTAL_MAX_BYTES = 200 * 1024 * 1024
ZIP_MAX_MEMBERS = 200
PDF_MAX_PAGES = int(os.getenv("LUNA_PDF_MAX_PAGES", "500"))
# ── Provider circuit breaker ──────────────────────────────────────────────────
# After this many consecutive transport failures (timeouts, 5xx, connection errors)
# every call fails fast for BREAKER_OPEN_SECONDS instead of each of eight MAP workers
# sitting out three retries of its own against a provider that is down.
BREAKER_FAILURES = int(os.getenv("LUNA_BREAKER_FAILURES", "5"))
BREAKER_OPEN_SECONDS = float(os.getenv("LUNA_BREAKER_OPEN", "30"))
# ── Parallelism ───────────────────────────────────────────────────────────────
# Extraction + MAP run in a thread pool. LLM/VLM calls are IO-bound, so high
# concurrency cuts wall-time at no extra token cost. Whisper is CPU-bound, so it's
# throttled separately to avoid thrashing the Space's cores.
MAP_WORKERS = int(os.getenv("LUNA_MAP_WORKERS", "8"))
TRANSCRIBE_WORKERS = int(os.getenv("LUNA_TRANSCRIBE_WORKERS", "2"))
# ── Whisper ─────────────────────────────────────────────────────────────────
WHISPER_MODEL = os.getenv("LUNA_WHISPER_MODEL", "base")
WHISPER_CPU_THREADS = int(os.getenv("WHISPER_CPU_THREADS", "4"))
# ── Misc ───────────────────────────────────────────────────────────────────
CACHE_DIR = os.path.join(tempfile.gettempdir(), "luna_cache")
DEBUG = bool(os.getenv("LUNA_DEBUG", ""))
# Shared with the Cloudflare worker that mints per-use tokens. If unset, the Space
# refuses every build - see auth.py. LUNA's own value only: the NINA fallback that
# used to sit here meant one leaked secret opened both Spaces.
SIGNING_SECRET = os.getenv("LUNA_SIGNING_SECRET", "")
SPACE_TAG = "luna"