Spaces:
Paused
Paused
Malina Claude Opus 4.8 commited on
Commit Β·
24ef38b
1
Parent(s): d70b247
LUNA: real video understanding (frames+VLM), generous relevance + fallback, clarify endpoint, console job logs, 1h cache TTL
Browse files- app.py +20 -2
- config.py +7 -0
- frames.py +69 -0
- pipeline.py +56 -13
- synthesis.py +85 -15
app.py
CHANGED
|
@@ -7,7 +7,16 @@ import os
|
|
| 7 |
|
| 8 |
import gradio as gr
|
| 9 |
|
|
|
|
| 10 |
from pipeline import run
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 11 |
|
| 12 |
|
| 13 |
# Quiet benign upload aborts (a browser cancelling a multipart upload mid-stream
|
|
@@ -27,8 +36,9 @@ for _name in ("uvicorn.error", "uvicorn"):
|
|
| 27 |
|
| 28 |
# The custom front-end at app.nnlabs.pl drives the `build` endpoint directly (the
|
| 29 |
# Gradio UI below is just for direct/testing access). Privacy: purge uploaded files
|
| 30 |
-
#
|
| 31 |
-
|
|
|
|
| 32 |
gr.Markdown(
|
| 33 |
"## LUNA β Latent Understanding & Narrative Assembly\n"
|
| 34 |
"API for the app at **https://app.nnlabs.pl/luna**. Drop up to 100 files, say "
|
|
@@ -46,6 +56,14 @@ with gr.Blocks(title="LUNA β File Synthesis", delete_cache=(900, 900)) as demo
|
|
| 46 |
run, inputs=[files, prompt, token], outputs=[status, note], api_name="build"
|
| 47 |
)
|
| 48 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 49 |
demo.queue(
|
| 50 |
default_concurrency_limit=int(os.getenv("LUNA_CONCURRENCY", "4")),
|
| 51 |
max_size=int(os.getenv("LUNA_QUEUE_MAX", "24")),
|
|
|
|
| 7 |
|
| 8 |
import gradio as gr
|
| 9 |
|
| 10 |
+
from auth import verify_usage_token
|
| 11 |
from pipeline import run
|
| 12 |
+
from synthesis import clarify as _clarify
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def clarify(prompt, token):
|
| 16 |
+
"""Wizard 'Am I right?' step β a short restatement of the understood intent."""
|
| 17 |
+
if not verify_usage_token(token):
|
| 18 |
+
return ""
|
| 19 |
+
return _clarify(prompt or "")
|
| 20 |
|
| 21 |
|
| 22 |
# Quiet benign upload aborts (a browser cancelling a multipart upload mid-stream
|
|
|
|
| 36 |
|
| 37 |
# The custom front-end at app.nnlabs.pl drives the `build` endpoint directly (the
|
| 38 |
# Gradio UI below is just for direct/testing access). Privacy: purge uploaded files
|
| 39 |
+
# after an hour β long enough that files uploaded during the wizard survive until the
|
| 40 |
+
# user finishes the steps and clicks Build; short enough to honor "files are not stored".
|
| 41 |
+
with gr.Blocks(title="LUNA β File Synthesis", delete_cache=(3600, 3600)) as demo:
|
| 42 |
gr.Markdown(
|
| 43 |
"## LUNA β Latent Understanding & Narrative Assembly\n"
|
| 44 |
"API for the app at **https://app.nnlabs.pl/luna**. Drop up to 100 files, say "
|
|
|
|
| 56 |
run, inputs=[files, prompt, token], outputs=[status, note], api_name="build"
|
| 57 |
)
|
| 58 |
|
| 59 |
+
with gr.Tab("Clarify"):
|
| 60 |
+
c_prompt = gr.Textbox(label="prompt")
|
| 61 |
+
c_token = gr.Textbox(label="access token", value="")
|
| 62 |
+
c_out = gr.Textbox(label="understanding")
|
| 63 |
+
gr.Button("Clarify").click(
|
| 64 |
+
clarify, inputs=[c_prompt, c_token], outputs=c_out, api_name="clarify"
|
| 65 |
+
)
|
| 66 |
+
|
| 67 |
demo.queue(
|
| 68 |
default_concurrency_limit=int(os.getenv("LUNA_CONCURRENCY", "4")),
|
| 69 |
max_size=int(os.getenv("LUNA_QUEUE_MAX", "24")),
|
config.py
CHANGED
|
@@ -21,6 +21,13 @@ LLM_TIMEOUT = 180
|
|
| 21 |
|
| 22 |
VLM_FRAME_MAX_PX = 1024 # downscale an image's long side before sending (caps tokens)
|
| 23 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 24 |
# ββ Upload limits (re-checked here; the website also enforces them) βββββββββββ
|
| 25 |
MAX_FILES = 100
|
| 26 |
MAX_FILE_MB = 25 # per non-media file
|
|
|
|
| 21 |
|
| 22 |
VLM_FRAME_MAX_PX = 1024 # downscale an image's long side before sending (caps tokens)
|
| 23 |
|
| 24 |
+
# ββ Video understanding (real frames, like NINA) ββββββββββββββββββββββββββββββ
|
| 25 |
+
# Videos are understood the way NINA does it: we sample keyframes and send them to
|
| 26 |
+
# the vision model together with any speech transcript β so even a silent screen
|
| 27 |
+
# recording is understood from what's on screen.
|
| 28 |
+
VIDEO_MAX_FRAMES = 8
|
| 29 |
+
VIDEO_FRAME_MAX_PX = 768
|
| 30 |
+
|
| 31 |
# ββ Upload limits (re-checked here; the website also enforces them) βββββββββββ
|
| 32 |
MAX_FILES = 100
|
| 33 |
MAX_FILE_MB = 25 # per non-media file
|
frames.py
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Keyframe sampling for real video understanding (NINA-style): pull a handful of
|
| 2 |
+
# evenly-spaced frames from a video and return them as base64 JPEG data URLs, ready
|
| 3 |
+
# to hand to the vision model alongside any speech transcript. Uses PyAV (pulled in
|
| 4 |
+
# transitively by faster-whisper) so there's no extra dependency. Best-effort: returns
|
| 5 |
+
# [] on any failure rather than raising.
|
| 6 |
+
import base64
|
| 7 |
+
import io
|
| 8 |
+
|
| 9 |
+
from config import VIDEO_FRAME_MAX_PX, VIDEO_MAX_FRAMES
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
def _to_data_url(image) -> str:
|
| 13 |
+
from PIL import Image # noqa: F401 (image is already a PIL.Image)
|
| 14 |
+
w, h = image.size
|
| 15 |
+
scale = VIDEO_FRAME_MAX_PX / max(w, h)
|
| 16 |
+
if scale < 1:
|
| 17 |
+
image = image.resize((max(1, int(w * scale)), max(1, int(h * scale))))
|
| 18 |
+
if image.mode != "RGB":
|
| 19 |
+
image = image.convert("RGB")
|
| 20 |
+
buf = io.BytesIO()
|
| 21 |
+
image.save(buf, format="JPEG", quality=85)
|
| 22 |
+
return "data:image/jpeg;base64," + base64.b64encode(buf.getvalue()).decode()
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def sample_frames(path: str, max_frames: int = VIDEO_MAX_FRAMES) -> list[tuple[float, str]]:
|
| 26 |
+
"""Return [(timestamp_seconds, data_url), β¦] sampled evenly across the video."""
|
| 27 |
+
try:
|
| 28 |
+
import av
|
| 29 |
+
except Exception as e:
|
| 30 |
+
print(f"[LUNA] frames: PyAV unavailable ({e}) β video frames skipped.")
|
| 31 |
+
return []
|
| 32 |
+
|
| 33 |
+
out: list[tuple[float, str]] = []
|
| 34 |
+
try:
|
| 35 |
+
container = av.open(path)
|
| 36 |
+
vstream = container.streams.video[0]
|
| 37 |
+
vstream.thread_type = "AUTO"
|
| 38 |
+
tb = vstream.time_base
|
| 39 |
+
duration = float(vstream.duration * tb) if vstream.duration else None
|
| 40 |
+
if not duration and container.duration:
|
| 41 |
+
duration = float(container.duration) / 1_000_000.0 # AV_TIME_BASE
|
| 42 |
+
|
| 43 |
+
if duration and duration > 0:
|
| 44 |
+
# Evenly spaced timestamps; seek to each and grab the next decoded frame.
|
| 45 |
+
times = [duration * (i + 1) / (max_frames + 1) for i in range(max_frames)]
|
| 46 |
+
for t in times:
|
| 47 |
+
try:
|
| 48 |
+
container.seek(int(t / tb), stream=vstream, any_frame=False, backward=True)
|
| 49 |
+
for frame in container.decode(vstream):
|
| 50 |
+
out.append((round(t, 1), _to_data_url(frame.to_image())))
|
| 51 |
+
break
|
| 52 |
+
except Exception:
|
| 53 |
+
continue
|
| 54 |
+
else:
|
| 55 |
+
# No duration metadata: decode sequentially and keep every Nth frame.
|
| 56 |
+
step, idx = 30, 0
|
| 57 |
+
for frame in container.decode(vstream):
|
| 58 |
+
if idx % step == 0:
|
| 59 |
+
out.append((round(float(frame.pts * tb) if frame.pts else 0.0, 1),
|
| 60 |
+
_to_data_url(frame.to_image())))
|
| 61 |
+
if len(out) >= max_frames:
|
| 62 |
+
break
|
| 63 |
+
idx += 1
|
| 64 |
+
container.close()
|
| 65 |
+
except Exception as e:
|
| 66 |
+
print(f"[LUNA] frames: sampling failed for {path} ({type(e).__name__}: {e}).")
|
| 67 |
+
return out
|
| 68 |
+
print(f"[LUNA] frames: sampled {len(out)} keyframe(s).")
|
| 69 |
+
return out
|
pipeline.py
CHANGED
|
@@ -11,8 +11,9 @@ from config import (
|
|
| 11 |
TRANSCRIBE_MAX_SECONDS, TRANSCRIBE_WORKERS,
|
| 12 |
)
|
| 13 |
from extract import category_of, extract
|
|
|
|
| 14 |
from models import get_whisper
|
| 15 |
-
from synthesis import is_global_prompt, map_image, map_text, reduce_stream
|
| 16 |
|
| 17 |
_transcribe_sem = threading.Semaphore(TRANSCRIBE_WORKERS)
|
| 18 |
|
|
@@ -36,30 +37,48 @@ def _transcribe(path: str) -> str:
|
|
| 36 |
|
| 37 |
|
| 38 |
def _process_one(prompt: str, path: str, name: str, global_mode: bool = False) -> dict:
|
| 39 |
-
"""Full per-file pipeline (extract β MAP). Returns a 'kept' or 'skipped' dict.
|
|
|
|
|
|
|
| 40 |
info = extract(path, name)
|
| 41 |
cat = info["category"]
|
|
|
|
| 42 |
|
| 43 |
if info["kind"] == "error":
|
| 44 |
-
|
|
|
|
|
|
|
| 45 |
|
| 46 |
if info["kind"] == "image":
|
| 47 |
m = map_image(prompt, name, info["payload"], global_mode)
|
| 48 |
elif info["kind"] == "media":
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 54 |
else: # text
|
| 55 |
m = map_text(prompt, name, cat, info["payload"], global_mode)
|
| 56 |
|
| 57 |
notes = (m.get("notes") or "").strip()[:MAP_NOTE_CHARS]
|
| 58 |
-
if m.get("relevant")
|
| 59 |
return {"status": "kept", "name": name, "category": cat,
|
| 60 |
"topics": m.get("topics") or [], "notes": notes}
|
| 61 |
return {"status": "skipped", "name": name, "category": cat,
|
| 62 |
-
"reason": "
|
| 63 |
|
| 64 |
|
| 65 |
def _resolve(files) -> list[tuple[str, str]]:
|
|
@@ -106,21 +125,29 @@ def run(files, prompt, token):
|
|
| 106 |
raw = (prompt or "").strip()
|
| 107 |
base, _, fmt = raw.partition("β¦FORMATβ§")
|
| 108 |
base, fmt = base.strip(), fmt.strip()
|
|
|
|
|
|
|
| 109 |
if not verify_usage_token(token):
|
|
|
|
| 110 |
yield ("error", "β οΈ This task could not be authorized. Please reload and sign in again.")
|
| 111 |
return
|
| 112 |
if not base:
|
|
|
|
| 113 |
yield ("error", "β οΈ Tell LUNA what to build first.")
|
| 114 |
return
|
| 115 |
|
| 116 |
resolved = _resolve(files)
|
| 117 |
err = _validate(resolved)
|
| 118 |
if err:
|
|
|
|
| 119 |
yield ("error", f"β οΈ {err}")
|
| 120 |
return
|
| 121 |
|
| 122 |
n = len(resolved)
|
| 123 |
glob = is_global_prompt(base)
|
|
|
|
|
|
|
|
|
|
| 124 |
yield (f"β³ Reading {n} file{'s' if n != 1 else ''}β¦", "")
|
| 125 |
|
| 126 |
kept, skipped, done = [], [], 0
|
|
@@ -130,16 +157,32 @@ def run(files, prompt, token):
|
|
| 130 |
try:
|
| 131 |
res = fut.result()
|
| 132 |
except Exception as e:
|
| 133 |
-
|
| 134 |
-
|
|
|
|
| 135 |
(kept if res["status"] == "kept" else skipped).append(res)
|
| 136 |
done += 1
|
| 137 |
yield (f"β³ Read {done}/{n} files β {len(kept)} relevant so farβ¦", "")
|
| 138 |
|
| 139 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 140 |
|
| 141 |
last = ""
|
| 142 |
for note in reduce_stream(base, kept, skipped, fmt):
|
| 143 |
last = note
|
| 144 |
yield ("βοΈ", note)
|
|
|
|
| 145 |
yield ("β
done", last)
|
|
|
|
| 11 |
TRANSCRIBE_MAX_SECONDS, TRANSCRIBE_WORKERS,
|
| 12 |
)
|
| 13 |
from extract import category_of, extract
|
| 14 |
+
from frames import sample_frames
|
| 15 |
from models import get_whisper
|
| 16 |
+
from synthesis import is_global_prompt, map_image, map_text, map_video, reduce_stream
|
| 17 |
|
| 18 |
_transcribe_sem = threading.Semaphore(TRANSCRIBE_WORKERS)
|
| 19 |
|
|
|
|
| 37 |
|
| 38 |
|
| 39 |
def _process_one(prompt: str, path: str, name: str, global_mode: bool = False) -> dict:
|
| 40 |
+
"""Full per-file pipeline (extract β MAP). Returns a 'kept' or 'skipped' dict.
|
| 41 |
+
'skipped' entries still carry any extracted notes so run() can fall back to them
|
| 42 |
+
rather than ever telling the user 'nothing relevant' when content exists."""
|
| 43 |
info = extract(path, name)
|
| 44 |
cat = info["category"]
|
| 45 |
+
print(f"[LUNA] processing Β«{name}Β» ({cat}, kind={info['kind']})")
|
| 46 |
|
| 47 |
if info["kind"] == "error":
|
| 48 |
+
print(f"[LUNA] Β«{name}Β» unreadable: {info['payload']}")
|
| 49 |
+
return {"status": "skipped", "name": name, "category": cat,
|
| 50 |
+
"reason": info["payload"], "notes": ""}
|
| 51 |
|
| 52 |
if info["kind"] == "image":
|
| 53 |
m = map_image(prompt, name, info["payload"], global_mode)
|
| 54 |
elif info["kind"] == "media":
|
| 55 |
+
if cat == "video":
|
| 56 |
+
frames = sample_frames(info["payload"])
|
| 57 |
+
transcript = _transcribe(info["payload"])
|
| 58 |
+
print(f"[LUNA] video Β«{name}Β»: {len(frames)} frame(s), "
|
| 59 |
+
f"{len(transcript)} transcript chars")
|
| 60 |
+
if frames:
|
| 61 |
+
m = map_video(prompt, name, transcript, frames, global_mode)
|
| 62 |
+
elif transcript:
|
| 63 |
+
m = map_text(prompt, name, cat, transcript, global_mode)
|
| 64 |
+
else:
|
| 65 |
+
return {"status": "skipped", "name": name, "category": cat,
|
| 66 |
+
"reason": "the video couldn't be read (no frames or speech)", "notes": ""}
|
| 67 |
+
else: # audio
|
| 68 |
+
transcript = _transcribe(info["payload"])
|
| 69 |
+
if not transcript:
|
| 70 |
+
return {"status": "skipped", "name": name, "category": cat,
|
| 71 |
+
"reason": "no speech could be transcribed", "notes": ""}
|
| 72 |
+
m = map_text(prompt, name, cat, transcript, global_mode)
|
| 73 |
else: # text
|
| 74 |
m = map_text(prompt, name, cat, info["payload"], global_mode)
|
| 75 |
|
| 76 |
notes = (m.get("notes") or "").strip()[:MAP_NOTE_CHARS]
|
| 77 |
+
if notes and (global_mode or m.get("relevant")):
|
| 78 |
return {"status": "kept", "name": name, "category": cat,
|
| 79 |
"topics": m.get("topics") or [], "notes": notes}
|
| 80 |
return {"status": "skipped", "name": name, "category": cat,
|
| 81 |
+
"reason": "not clearly relevant to the request", "notes": notes}
|
| 82 |
|
| 83 |
|
| 84 |
def _resolve(files) -> list[tuple[str, str]]:
|
|
|
|
| 125 |
raw = (prompt or "").strip()
|
| 126 |
base, _, fmt = raw.partition("β¦FORMATβ§")
|
| 127 |
base, fmt = base.strip(), fmt.strip()
|
| 128 |
+
print(f"\n[LUNA] ========== NEW JOB REQUEST ==========")
|
| 129 |
+
print(f"[LUNA] prompt=Β«{base[:160]}Β»" + (f" fmt=Β«{fmt[:80]}Β»" if fmt else ""))
|
| 130 |
if not verify_usage_token(token):
|
| 131 |
+
print("[LUNA] job REJECTED β invalid/expired/replayed token")
|
| 132 |
yield ("error", "β οΈ This task could not be authorized. Please reload and sign in again.")
|
| 133 |
return
|
| 134 |
if not base:
|
| 135 |
+
print("[LUNA] job REJECTED β empty prompt")
|
| 136 |
yield ("error", "β οΈ Tell LUNA what to build first.")
|
| 137 |
return
|
| 138 |
|
| 139 |
resolved = _resolve(files)
|
| 140 |
err = _validate(resolved)
|
| 141 |
if err:
|
| 142 |
+
print(f"[LUNA] job REJECTED β {err}")
|
| 143 |
yield ("error", f"β οΈ {err}")
|
| 144 |
return
|
| 145 |
|
| 146 |
n = len(resolved)
|
| 147 |
glob = is_global_prompt(base)
|
| 148 |
+
print(f"[LUNA] {n} file(s): " + ", ".join(nm for _, nm in resolved[:25])
|
| 149 |
+
+ (" β¦" if n > 25 else ""))
|
| 150 |
+
print(f"[LUNA] global_mode={glob} (broad/describe intent β keep all readable files)")
|
| 151 |
yield (f"β³ Reading {n} file{'s' if n != 1 else ''}β¦", "")
|
| 152 |
|
| 153 |
kept, skipped, done = [], [], 0
|
|
|
|
| 157 |
try:
|
| 158 |
res = fut.result()
|
| 159 |
except Exception as e:
|
| 160 |
+
print(f"[LUNA] file Β«{futures[fut]}Β» crashed: {type(e).__name__}: {e}")
|
| 161 |
+
res = {"status": "skipped", "name": futures[fut], "category": "other",
|
| 162 |
+
"reason": f"processing error ({type(e).__name__})", "notes": ""}
|
| 163 |
(kept if res["status"] == "kept" else skipped).append(res)
|
| 164 |
done += 1
|
| 165 |
yield (f"β³ Read {done}/{n} files β {len(kept)} relevant so farβ¦", "")
|
| 166 |
|
| 167 |
+
print(f"[LUNA] extraction complete β {len(kept)} kept, {len(skipped)} skipped")
|
| 168 |
+
|
| 169 |
+
# Never tell the user "nothing relevant" when readable content exists: fall back to
|
| 170 |
+
# the readable-but-off-topic files (they carry notes) and let REDUCE decide what to use.
|
| 171 |
+
if not kept:
|
| 172 |
+
fallback = [s for s in skipped if (s.get("notes") or "").strip()]
|
| 173 |
+
if fallback:
|
| 174 |
+
print(f"[LUNA] nothing passed the filter β falling back to "
|
| 175 |
+
f"{len(fallback)} readable file(s)")
|
| 176 |
+
for s in fallback:
|
| 177 |
+
kept.append({"status": "kept", "name": s["name"], "category": s["category"],
|
| 178 |
+
"topics": [], "notes": s["notes"]})
|
| 179 |
+
skipped = [s for s in skipped if not (s.get("notes") or "").strip()]
|
| 180 |
+
|
| 181 |
+
yield (f"βοΈ Writing the Note from {len(kept)} file{'s' if len(kept) != 1 else ''}β¦", "")
|
| 182 |
|
| 183 |
last = ""
|
| 184 |
for note in reduce_stream(base, kept, skipped, fmt):
|
| 185 |
last = note
|
| 186 |
yield ("βοΈ", note)
|
| 187 |
+
print(f"[LUNA] ========== JOB DONE β Note {len(last)} chars ==========\n")
|
| 188 |
yield ("β
done", last)
|
synthesis.py
CHANGED
|
@@ -94,19 +94,20 @@ _MAP_SYSTEM_GLOBAL = (
|
|
| 94 |
|
| 95 |
_MAP_SYSTEM = (
|
| 96 |
"You are LUNA's extraction stage. You are given a user's build request and the "
|
| 97 |
-
"content of ONE source file.
|
| 98 |
-
"
|
| 99 |
"Output STRICT JSON only (no prose, no code fence) with exactly these keys:\n"
|
| 100 |
' "relevant": true or false,\n'
|
| 101 |
' "topics": array of short topic labels (2-5 words each) that THIS file covers '
|
| 102 |
-
"and that relate to the request β a file may
|
| 103 |
-
"
|
| 104 |
-
|
| 105 |
-
"
|
| 106 |
-
"
|
| 107 |
-
"
|
| 108 |
-
"
|
| 109 |
-
"
|
|
|
|
| 110 |
)
|
| 111 |
|
| 112 |
|
|
@@ -152,8 +153,8 @@ def map_text(prompt: str, name: str, category: str, content: str, global_mode: b
|
|
| 152 |
temperature=MAP_TEMPERATURE,
|
| 153 |
)
|
| 154 |
out = _parse_json(resp.choices[0].message.content)
|
| 155 |
-
|
| 156 |
-
|
| 157 |
return out
|
| 158 |
except Exception as e:
|
| 159 |
print(f"[LUNA] MAP error on {name}: {e}")
|
|
@@ -182,12 +183,81 @@ def map_image(prompt: str, name: str, data_url: str, global_mode: bool = False)
|
|
| 182 |
max_tokens=MAP_MAX_TOKENS,
|
| 183 |
temperature=MAP_TEMPERATURE,
|
| 184 |
)
|
| 185 |
-
|
|
|
|
|
|
|
| 186 |
except Exception as e:
|
| 187 |
print(f"[LUNA] VLM error on {name}: {e}")
|
| 188 |
return {"relevant": False, "topics": [], "notes": ""}
|
| 189 |
|
| 190 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 191 |
# ββ REDUCE ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 192 |
def _reduce_system(language: str) -> str:
|
| 193 |
return (
|
|
@@ -260,8 +330,8 @@ def reduce_stream(prompt: str, kept: list[dict], skipped: list[dict], fmt: str =
|
|
| 260 |
The model mirrors the request's language itself (more reliable than langdetect,
|
| 261 |
which misfires on short prompts)."""
|
| 262 |
language = detect_language(prompt)
|
| 263 |
-
|
| 264 |
-
|
| 265 |
if not kept:
|
| 266 |
yield (
|
| 267 |
f"# {prompt.strip()[:80]}\n\n"
|
|
|
|
| 94 |
|
| 95 |
_MAP_SYSTEM = (
|
| 96 |
"You are LUNA's extraction stage. You are given a user's build request and the "
|
| 97 |
+
"content of ONE source file. Pull out the material from this file that could help "
|
| 98 |
+
"fulfil the request.\n\n"
|
| 99 |
"Output STRICT JSON only (no prose, no code fence) with exactly these keys:\n"
|
| 100 |
' "relevant": true or false,\n'
|
| 101 |
' "topics": array of short topic labels (2-5 words each) that THIS file covers '
|
| 102 |
+
"and that relate to the request β a file may touch several distinct topics,\n"
|
| 103 |
+
' "notes": a faithful, concise extraction of the file content that could help the '
|
| 104 |
+
"request, as Markdown bullet points; preserve key facts, numbers, names and quotes "
|
| 105 |
+
"exactly.\n\n"
|
| 106 |
+
"RELEVANCE β lean strongly toward relevant=true. Set relevant=false ONLY when the file "
|
| 107 |
+
"is CLEARLY and ENTIRELY unrelated to the request; when in doubt, keep it (relevant=true). "
|
| 108 |
+
"Always populate notes with the file's pertinent content β even if the connection is "
|
| 109 |
+
"loose, give a brief faithful summary rather than an empty string. NEVER invent content "
|
| 110 |
+
"that isn't in the file. Keep names, numbers and quotes verbatim."
|
| 111 |
)
|
| 112 |
|
| 113 |
|
|
|
|
| 153 |
temperature=MAP_TEMPERATURE,
|
| 154 |
)
|
| 155 |
out = _parse_json(resp.choices[0].message.content)
|
| 156 |
+
print(f"[LUNA] MAP text Β«{name}Β» β relevant={out['relevant']} topics={out['topics']} "
|
| 157 |
+
f"notes={len(out['notes'])} chars")
|
| 158 |
return out
|
| 159 |
except Exception as e:
|
| 160 |
print(f"[LUNA] MAP error on {name}: {e}")
|
|
|
|
| 183 |
max_tokens=MAP_MAX_TOKENS,
|
| 184 |
temperature=MAP_TEMPERATURE,
|
| 185 |
)
|
| 186 |
+
out = _parse_json(resp.choices[0].message.content)
|
| 187 |
+
print(f"[LUNA] MAP image Β«{name}Β» β relevant={out['relevant']} topics={out['topics']}")
|
| 188 |
+
return out
|
| 189 |
except Exception as e:
|
| 190 |
print(f"[LUNA] VLM error on {name}: {e}")
|
| 191 |
return {"relevant": False, "topics": [], "notes": ""}
|
| 192 |
|
| 193 |
|
| 194 |
+
def map_video(prompt: str, name: str, transcript: str, frames: list[tuple],
|
| 195 |
+
global_mode: bool = False) -> dict:
|
| 196 |
+
"""Real video understanding (NINA-style): the VLM sees sampled keyframes + the
|
| 197 |
+
speech transcript and extracts the request-relevant content. Works on silent
|
| 198 |
+
videos (screen recordings) because it reads what is on screen."""
|
| 199 |
+
head = (f'Build request: "{prompt}"\n\n'
|
| 200 |
+
f"This is a video file named {name}. Below are {len(frames)} keyframes sampled "
|
| 201 |
+
f"across it" + (" plus its speech transcript" if transcript else " (no speech was "
|
| 202 |
+
"detected, so rely on what is shown)") + ". Read any on-screen text verbatim, "
|
| 203 |
+
"describe what is shown and what happens over time, fuse it with the transcript "
|
| 204 |
+
"if present, and extract the request-relevant content. Return the strict JSON "
|
| 205 |
+
"from the system message.")
|
| 206 |
+
user = [{"type": "text", "text": head}]
|
| 207 |
+
if transcript:
|
| 208 |
+
user.append({"type": "text", "text": "Speech transcript:\n" + transcript[:8000]})
|
| 209 |
+
for ts, url in frames:
|
| 210 |
+
user.append({"type": "text", "text": f"Frame at {int(ts)}s:"})
|
| 211 |
+
user.append({"type": "image_url", "image_url": {"url": url}})
|
| 212 |
+
try:
|
| 213 |
+
resp = llm_client.chat_completion(
|
| 214 |
+
model=VLM_MODEL,
|
| 215 |
+
messages=[
|
| 216 |
+
{"role": "system", "content": _MAP_SYSTEM_GLOBAL if global_mode else _MAP_SYSTEM},
|
| 217 |
+
{"role": "user", "content": user},
|
| 218 |
+
],
|
| 219 |
+
max_tokens=MAP_MAX_TOKENS,
|
| 220 |
+
temperature=MAP_TEMPERATURE,
|
| 221 |
+
)
|
| 222 |
+
out = _parse_json(resp.choices[0].message.content)
|
| 223 |
+
print(f"[LUNA] MAP video Β«{name}Β» β relevant={out['relevant']} topics={out['topics']} "
|
| 224 |
+
f"(frames={len(frames)}, transcript={len(transcript)} chars)")
|
| 225 |
+
return out
|
| 226 |
+
except Exception as e:
|
| 227 |
+
print(f"[LUNA] VLM(video) error on {name}: {e}")
|
| 228 |
+
return {"relevant": False, "topics": [], "notes": ""}
|
| 229 |
+
|
| 230 |
+
|
| 231 |
+
# ββ Clarify (wizard "Am I right?" step) βββββββββββββββββββββββββββββββββββββββ
|
| 232 |
+
def clarify(prompt: str) -> str:
|
| 233 |
+
"""One or two short sentences restating what LUNA understands it should build, in
|
| 234 |
+
the prompt's language. Used by the wizard's confirmation step."""
|
| 235 |
+
base = (prompt or "").partition("β¦FORMATβ§")[0].strip()
|
| 236 |
+
if not base:
|
| 237 |
+
return ""
|
| 238 |
+
lang = detect_language(base)
|
| 239 |
+
system = (
|
| 240 |
+
f"You are LUNA, a file-synthesis assistant. The user has uploaded some files and "
|
| 241 |
+
f"will build a structured Note from them. In ONE or TWO short sentences, written in "
|
| 242 |
+
f"{lang}, restate what you understand they want you to build, so they can confirm. "
|
| 243 |
+
f"Be concrete and warm. Do NOT ask questions and do NOT add caveats β just state your "
|
| 244 |
+
f"understanding, starting naturally (e.g. \"Got it β\" / \"Jasne β\")."
|
| 245 |
+
)
|
| 246 |
+
try:
|
| 247 |
+
resp = llm_client.chat_completion(
|
| 248 |
+
model=MAP_MODEL,
|
| 249 |
+
messages=[{"role": "system", "content": system},
|
| 250 |
+
{"role": "user", "content": base}],
|
| 251 |
+
max_tokens=160, temperature=0.2,
|
| 252 |
+
)
|
| 253 |
+
text = (resp.choices[0].message.content or "").strip()
|
| 254 |
+
print(f"[LUNA] CLARIFY ({lang}) β Β«{base[:80]}Β» β Β«{text[:120]}Β»")
|
| 255 |
+
return text
|
| 256 |
+
except Exception as e:
|
| 257 |
+
print(f"[LUNA] CLARIFY error: {e}")
|
| 258 |
+
return ""
|
| 259 |
+
|
| 260 |
+
|
| 261 |
# ββ REDUCE ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 262 |
def _reduce_system(language: str) -> str:
|
| 263 |
return (
|
|
|
|
| 330 |
The model mirrors the request's language itself (more reliable than langdetect,
|
| 331 |
which misfires on short prompts)."""
|
| 332 |
language = detect_language(prompt)
|
| 333 |
+
print(f"[LUNA] REDUCE start β {len(kept)} kept, {len(skipped)} skipped, lang={language}, "
|
| 334 |
+
f"model={REDUCE_MODEL}")
|
| 335 |
if not kept:
|
| 336 |
yield (
|
| 337 |
f"# {prompt.strip()[:80]}\n\n"
|