Phoenix21's picture
Deploy LectureLens runtime from GitHub 5dca40ef8a57
eabfa32 verified
Raw History Blame Contribute Delete
9.32 kB
"""Visual index building only. No answer-generation provider is required."""
import json
from html import escape
from pathlib import Path
import subprocess
import sys
ROOT = Path(__file__).resolve().parent
if (ROOT.parent / "lecturelens_v2").is_dir():
sys.path.insert(0, str(ROOT.parent))
sys.path.insert(0, str(ROOT.parents[1] / "shared"))
from lecturelens_v2.models import configure
configure()
import gradio as gr
from lecturelens_v2.pipeline import build, clean_session
from lecturelens_v2.visualize import timeline, hierarchy, tree_html
def asset(name):
staged = ROOT / "assets" / name
return staged if staged.exists() else ROOT.parents[1] / "examples" / name
def load_demo(choice):
name = "changes" if choice == "Cuts, fade and repeated scene" else "demo"
return str(asset(name + ".mp4")), str(asset(name + ".srt"))
def build_index(video, subtitles, detector, threshold, minimum, representation, merge, maximum, use_ocr, previous, progress=gr.Progress()):
try:
state = build(video, subtitles, detector, threshold, minimum, representation, merge, maximum, use_ocr,
progress=lambda text: progress(0.4, desc=text))
index = state["index"]; clean_session(previous)
choices = [(f'{n["id"]} · {n["start"]:.2f}–{n["end"]:.2f}s', n["id"]) for n in reversed(index["nodes"])]
summary = (f'**Index ready** · {index["source"]["frame_count"]:,} frames · {len(index["shots"])} shots'
f' · {len(index["groups"])} groups · {len(index["keyframes"])} keyframes · {state["elapsed_seconds"]}s processing')
return summary, timeline(index), hierarchy(index), tree_html(index), gr.Dropdown(choices=choices, value=index["root"]), state["bundle"], str(Path(state["workspace"]) / "index.json"), state
except (ValueError, RuntimeError) as exc:
raise gr.Error(str(exc)[:400]) from None
except Exception as exc:
print("V2 indexing failed: " + type(exc).__name__, flush=True)
raise gr.Error("Indexing could not complete. Try the Adaptive detector with Color baseline, or use a shorter clip.") from None
def inspect_node(ident, state):
if not state or not ident:
return "Build an index to inspect a node.", [], None, None
index = state["index"]; node = next((n for n in index["nodes"] if n["id"] == ident), None)
if node is None:
raise gr.Error("Choose a node from the current index")
evidence = [e for e in index["evidence"] if e["id"] in node["evidence_ids"]]
frames = [f for f in index["keyframes"] if f["id"] in node["keyframes"]]
related = [r for r in index["relations"] if r["source"] == ident or r["target"] == ident]
context = {"schema_version": "lecturelens.context.v1", "time_units": "seconds", "source": index["source"],
"node": {k: node[k] for k in ("id", "start", "end")}, "evidence": evidence, "keyframes": frames,
"limitations": index["limitations"]}
root = Path(state["workspace"]); file = root / (ident + "-context.json")
file.write_text(json.dumps(context, indent=2) + "\n")
preview = root / (ident + "-preview.mp4")
if not preview.exists():
subprocess.run(["ffmpeg", "-v", "error", "-nostdin", "-y", "-ss", str(node["start"]), "-i", state["video"],
"-t", str(min(20, node["end"] - node["start"])), "-c:v", "libx264", "-preset", "veryfast", "-c:a", "aac",
"-movflags", "+faststart", str(preview)], capture_output=True, timeout=60, check=True)
details = (f'<h3>{escape(ident)}</h3><p>{node["start"]:.3f}–{node["end"]:.3f} seconds · '
f'frames [{node["lo"]}, {node["hi"]}) · {len(evidence)} evidence items</p>'
f'<p>{escape(node.get("label", "Chronological hierarchy node"))}</p>'
f'<p>Label source: {escape(node.get("label_source", "structural_identifier"))}. '
f'Merge similarity: {node["merge_similarity"] if node["merge_similarity"] is not None else "shot leaf"}. '
f'{len(related)} repeated-scene links. Scores are not confidence probabilities.</p>')
gallery = [(str(root / f["file"]), f'{f["id"]} · {f["time"]:.3f}s') for f in frames[:24]]
return details, gallery, str(preview), str(file)
def reset_input(state):
clean_session(state)
return (None, "Input changed. Build a new index to inspect this video.", None, None, "",
gr.Dropdown(choices=[], value=None), None, None, "Build an index to inspect a node.", [], None, None)
def create_app():
with gr.Blocks(title="LectureLens V2 · Index explorer", delete_cache=(3600, 3600)) as app:
state = gr.State(None, time_to_live=3600, delete_callback=clean_session)
gr.HTML('<div class="hero"><div class="eyebrow">LECTURELENS / V2</div><h1>See how a video<br>becomes an index.</h1><p>Detect changes. Explore the structure. Export timestamped evidence.</p></div>')
with gr.Row():
with gr.Column(scale=4):
video = gr.Video(value=str(asset("changes.mp4")), label="Video to index", sources=["upload"], height=270)
subtitles = gr.File(value=str(asset("changes.srt")), label="Optional subtitles (SRT / VTT)", file_types=[".srt", ".vtt"], type="filepath")
examples = gr.Dropdown(["Cuts, fade and repeated scene", "Gradient descent slides"], value="Cuts, fade and repeated scene", label="Demo video")
gr.Button("Load demo").click(load_demo, examples, [video, subtitles])
with gr.Accordion("Index settings", open=True):
detector = gr.Dropdown(["TransNet V2", "Adaptive baseline"], value="TransNet V2", label="Boundary detector")
threshold = gr.Slider(0.05, 0.95, value=0.5, step=0.05, label="Boundary threshold (higher = fewer cuts)")
minimum = gr.Slider(0.1, 10, value=0.5, step=0.1, label="Minimum shot length (seconds)")
representation = gr.Dropdown(["CLIP embeddings", "Color baseline"], value="CLIP embeddings", label="Visual representation")
merge = gr.Slider(0, 1, value=0.8, step=0.02, label="Similarity required to keep a group")
maximum = gr.Slider(2, 120, value=40, step=1, label="Preferred maximum group length (seconds)")
use_ocr = gr.Checkbox(True, label="Read visible text from up to 24 keyframes")
button = gr.Button("Build video index", variant="primary")
status = gr.Markdown("Up to 120 seconds / 100 MB. Learned models run on the hosting server; no Groq key is used.")
with gr.Column(scale=7):
with gr.Tab("Timeline"):
plot = gr.Plot(label="Change scores, shots and groups")
gr.Markdown("Zoom into the timeline, then choose a node below to inspect its evidence. The V1 row shows equal frame-halving at a comparable depth.")
with gr.Tab("Hierarchy map"):
graph = gr.Plot(label="Chronological hierarchy")
tree = gr.HTML()
node = gr.Dropdown(choices=[], label="Inspect hierarchy node", interactive=True)
details = gr.HTML("Build the index to inspect a segment.")
gallery = gr.Gallery(label="Original keyframes (first 24 for this node)", columns=3, height=260)
replay = gr.Video(label="Selected node preview (first 20 seconds)", height=230)
selected = gr.File(label="Selected node context for an LLM")
with gr.Row():
bundle = gr.File(label="Complete portable evidence package (ZIP)")
canonical = gr.File(label="Canonical video index (JSON)")
with gr.Accordion("What this index can and cannot tell you", open=False):
gr.Markdown("Shots are visual intervals, not guaranteed topics. CLIP groups visually related content; OCR and supplied subtitles provide literal text. There is no automatic speech recognition or generated visual description here. Text-only LLMs cannot read the exported images. The full package can exceed a model's context limit; use a selected-node context when appropriate. A static slide can contain several spoken topics. The Color baseline avoids model/GPU calls but has weaker meaning.")
event = button.click(build_index, [video, subtitles, detector, threshold, minimum, representation, merge, maximum, use_ocr, state],
[status, plot, graph, tree, node, bundle, canonical, state], concurrency_limit=1, api_name="build_index")
event.then(inspect_node, [node, state], [details, gallery, replay, selected], api_name=False)
node.change(inspect_node, [node, state], [details, gallery, replay, selected], concurrency_limit=1, api_name="inspect_node")
cleared = [state, status, plot, graph, tree, node, bundle, canonical, details, gallery, replay, selected]
video.change(reset_input, state, cleared, api_name=False)
subtitles.change(reset_input, state, cleared, api_name=False)
return app
demo = create_app()
if __name__ == "__main__":
demo.queue(max_size=6, default_concurrency_limit=1).launch(server_name="0.0.0.0", server_port=7860,
max_file_size="100mb", ssr_mode=False, show_error=False, css=(ROOT / "styles.css").read_text())