NINA / retrieval.py
Malina
Claude Sonnet 4.6
NINA v0.91 โ€” 5-part retrieval + accuracy upgrade
8aa6bb8
Raw History Blame
13.8 kB
import re
import numpy as np
from config import (
BI_ENCODER_TOP_K, CHUNK_MAX_DURATION, CHUNK_MIN_DURATION,
CHUNK_SPLIT_THRESHOLD, CHUNK_WINDOW, CROSS_ENCODER_TOP_K,
DEBUG, GLOBAL_MAX_CHUNKS, MERGE_GAP_SEC, REGION_MIN_DURATION,
VISUAL_WEIGHT,
)
from models import bi_encoder, cross_encoder
from utils import _build_chunk, _cosine, _sigmoid
from visual import span_visual_score
# โ”€โ”€ Entity extraction โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
_STOPWORDS = {
"the","a","an","and","or","but","in","on","at","to","for","of","is","are",
"was","were","be","been","that","this","it","he","she","they","we","i",
"w","z","ลผe","na","do","siฤ™","nie","to","jest","sฤ…","by","przez","po",
"le","la","les","un","une","des","du","et","en","ร ","de","est","il","elle",
"ein","eine","der","die","das","und","ist","zu","mit","von","dem",
}
def extract_entities(text: str) -> set[str]:
entities: set[str] = set()
for match in re.finditer(r'"([^"]{2,40})"', text):
entities.add(match.group(1).lower().strip())
for match in re.finditer(
r'\b([A-Zรร‰รร“รšร€ร‚รŠรŽร”ร™ร„ร‹รร–รœฤ„ฤ†ฤ˜ลลƒร“ลšลนลป][a-zรกรฉรญรณรบร รขรชรฎรดรนรครซรฏรถรผฤ…ฤ‡ฤ™ล‚ล„รณล›ลบลผ]+'
r'(?:\s[A-Zรร‰รร“รšร€ร‚รŠรŽร”ร™ร„ร‹รร–รœฤ„ฤ†ฤ˜ลลƒร“ลšลนลป][a-zร€-ลพฤ…ฤ‡ฤ™ล‚ล„รณล›ลบลผ]+)*)\b',
text,
):
token = match.group(1).lower().strip()
if token not in _STOPWORDS and len(token) > 2:
entities.add(token)
for token in re.findall(
r'\b[A-Zรร‰รร“รšร€ร‚รŠรŽร”ร™ร„ร‹รร–รœฤ„ฤ†ฤ˜ลลƒร“ลšลนลป][a-zร€-ลพฤ…ฤ‡ฤ™ล‚ล„รณล›ลบลผ]{2,}\b', text
):
t = token.lower()
if t not in _STOPWORDS:
entities.add(t)
for match in re.finditer(r'\b([A-Za-z]+[-\s]?\d+[\w]*)\b', text):
entities.add(match.group(1).lower())
return entities
def entity_overlap_score(
prompt_entities: set[str], chunk_entities: set[str], chunk_text: str = ""
) -> float:
if not prompt_entities:
return 0.0
chunk_text_lower = chunk_text.lower()
chunk_words = set(re.findall(r'\w+', chunk_text_lower))
matched = 0
for pe in prompt_entities:
# 1. entity-to-entity overlap
if any(pe in ce or ce in pe for ce in chunk_entities):
matched += 1; continue
# 2. substring in raw text (>= 3 chars โ€” catches "won", "who", etc.)
if len(pe) >= 3 and pe in chunk_text_lower:
matched += 1; continue
# 3. exact word-level match (handles ASR spacing differences)
if pe in chunk_words:
matched += 1; continue
# 4. compound-word split: "manhunt" โ†’ check "man" + "hunt" both in text
if len(pe) >= 5:
for split in range(2, len(pe) - 1):
left, right = pe[:split], pe[split:]
if len(left) >= 3 and len(right) >= 3 and left in chunk_words and right in chunk_words:
matched += 1; break
return matched / len(prompt_entities)
def add_entities_to_chunks(chunks: list[dict]) -> list[dict]:
for chunk in chunks:
chunk["entities"] = extract_entities(chunk["text"])
return chunks
# โ”€โ”€ Semantic chunking โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
def semantic_chunk(segments: list[dict]) -> list[dict]:
if not segments:
return []
texts = [s["text"] for s in segments]
embeddings = bi_encoder.encode(texts, convert_to_numpy=True, batch_size=64)
# Adaptive threshold: calibrate to this video's similarity distribution
if len(embeddings) >= 4:
consec_sims = [
max(_cosine(embeddings[i], embeddings[i - 1]), 0.0)
for i in range(1, len(embeddings))
]
mean_sim = float(np.mean(consec_sims))
std_sim = float(np.std(consec_sims))
threshold = float(np.clip(mean_sim - 0.5 * std_sim, 0.30, 0.62))
else:
mean_sim = std_sim = 0.0
threshold = CHUNK_SPLIT_THRESHOLD
if DEBUG:
print(
f"[DEBUG] Adaptive chunk threshold: {threshold:.3f} "
f"(mean_sim={mean_sim:.3f}, std={std_sim:.3f})"
)
chunks, current, cur_emb = [], [segments[0]], [embeddings[0]]
for i in range(1, len(segments)):
chunk_dur = segments[i]["end"] - current[0]["start"]
window_mean = np.mean(cur_emb[-CHUNK_WINDOW:], axis=0)
sim = max(_cosine(embeddings[i], window_mean),
_cosine(embeddings[i], cur_emb[-1]))
long_enough = (current[-1]["end"] - current[0]["start"]) >= CHUNK_MIN_DURATION
if (sim < threshold or chunk_dur >= CHUNK_MAX_DURATION) and long_enough:
chunks.append(_build_chunk(current, cur_emb))
current, cur_emb = [segments[i]], [embeddings[i]]
else:
current.append(segments[i])
cur_emb.append(embeddings[i])
if current:
chunks.append(_build_chunk(current, cur_emb))
print(f"[NINA] {len(segments)} segments โ†’ {len(chunks)} chunks (threshold={threshold:.3f}).")
return chunks
# โ”€โ”€ Query router โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
_GLOBAL_PATTERNS = re.compile(
r'\b('
r'summari[sz]e|summary|overview|recap|tl;?dr|key\s+points?|main\s+points?|'
r'key\s+takeaways?|main\s+idea|gist|'
r'what(?:\'s| is| was)\s+(?:this|the|it)\s+(?:video|recording|clip|all\s+)?\s*about|'
r'what(?:\'s| is| was)\s+(?:this|the)\s+(?:video|recording|clip)|'
r'describe\s+(?:the|this)\s+(?:video|recording|clip)|'
r'streszcz\w*|podsumuj\w*|podsumowanie|'
r'o\s+czym\s+(?:jest|to|opowiada|mรณwi)|czego\s+dotyczy|'
r'gล‚[oรณ]wn\w+\s+(?:punkt\w*|my[sล›]l\w*|tez\w*)|og[oรณ]ln\w+\s+zarys'
r')\b',
re.IGNORECASE,
)
def classify_query(prompt: str) -> str:
"""Pattern-based pass-1 router. Returns 'global' or 'specific'."""
if _GLOBAL_PATTERNS.search(prompt):
return "global"
return "specific"
# โ”€โ”€ Bi-encoder retrieval โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
def bi_encode_retrieve(chunks: list[dict], prompt: str) -> tuple[list[dict], float]:
prompt_emb = bi_encoder.encode([prompt], convert_to_numpy=True)[0]
qn = prompt_emb / (np.linalg.norm(prompt_emb) + 1e-9)
scores = []
for c in chunks:
seg_embs = c.get("seg_embs")
if seg_embs is not None and len(seg_embs) > 1:
# MaxSim: highest cosine among individual segment embeddings
en = seg_embs / (np.linalg.norm(seg_embs, axis=1, keepdims=True) + 1e-9)
scores.append(float(np.max(en @ qn)))
else:
scores.append(_cosine(prompt_emb, c["emb"]))
top_k = min(BI_ENCODER_TOP_K, len(chunks))
top_idx = np.argsort(scores)[::-1][:top_k]
if DEBUG:
print(f"[DEBUG] Bi top-3 scores (MaxSim): " +
", ".join(f"{scores[i]:.3f}" for i in top_idx[:3]))
best = scores[top_idx[0]] if len(top_idx) else 0.0
return (
[{**chunks[i], "bi_score": round(scores[i], 3)} for i in top_idx],
best,
)
# โ”€โ”€ Cross-encoder rerank + audio/visual fusion โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
_LONG_CHUNK_WORDS = 120 # cross-encoder context limit guard
def rerank_and_score(
candidates: list[dict], prompt: str,
times: np.ndarray, norm: np.ndarray,
) -> list[dict]:
# Build a flat pairs list; track per-candidate slice indices
all_pairs: list[list[str]] = []
pair_ranges: list[tuple[int,int]] = []
for c in candidates:
start = len(all_pairs)
seg_texts = c.get("seg_texts", [])
if len(c["text"].split()) > _LONG_CHUNK_WORDS and len(seg_texts) > 1:
for t in seg_texts:
if t.strip():
all_pairs.append([prompt, t])
if len(all_pairs) == start: # short chunk or no seg_texts
all_pairs.append([prompt, c["text"]])
pair_ranges.append((start, len(all_pairs)))
all_logits = cross_encoder.predict(all_pairs)
# Max-segment score per candidate: ensures long chunks aren't hurt by truncation
logits = [float(np.max(all_logits[s:e])) for s, e in pair_ranges]
prompt_entities = extract_entities(prompt)
prompt_words = {
w.lower() for w in re.findall(r'[\wร€-ลพ]{3,}', prompt)
if w.lower() not in _STOPWORDS
}
all_prompt_tokens = prompt_entities | prompt_words
if DEBUG and all_prompt_tokens:
print(f"[DEBUG] Prompt tokens for entity match: {all_prompt_tokens}")
vw = VISUAL_WEIGHT if len(norm) else 0.0
scored = []
for candidate, logit in zip(candidates, logits):
chunk_entities = candidate.get("entities", set())
chunk_text = candidate.get("text", "")
overlap = entity_overlap_score(all_prompt_tokens, chunk_entities, chunk_text)
ce_conf = _sigmoid(float(logit))
audio_score = 0.6 * ce_conf + 0.4 * overlap
visual_score = span_visual_score(times, norm, candidate["start"], candidate["end"])
final = (1.0 - vw) * audio_score + vw * visual_score
scored.append({
**candidate,
"ce_logit": round(float(logit), 3),
"entity_overlap": round(overlap, 3),
"visual_score": round(visual_score, 3),
"confidence": round(min(1.0, final), 3),
})
ranked = sorted(scored, key=lambda x: x["confidence"], reverse=True)
if DEBUG:
print("[DEBUG] Top-3 after reranking (audio+visual):")
for r in ranked[:3]:
print(
f" ce_logit={r['ce_logit']:.2f} "
f"bi={r['bi_score']:.3f} "
f"entity={r['entity_overlap']:.3f} "
f"visual={r['visual_score']:.3f} "
f"conf={r['confidence']:.3f} "
f"\"{r['text'][:60]}\""
)
return ranked[:CROSS_ENCODER_TOP_K]
# โ”€โ”€ Hybrid padding โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
def pad_with_neighbours(results: list[dict], chunks: list[dict]) -> list[dict]:
"""Include immediate neighbours of each retrieved chunk (HYBRID mode)."""
result_starts = {r["start"] for r in results}
padded = list(results)
for r in results:
for i, c in enumerate(chunks):
if abs(c["start"] - r["start"]) < 0.5:
for neighbour_i in [i - 1, i + 1]:
if 0 <= neighbour_i < len(chunks):
nb = chunks[neighbour_i]
if nb["start"] not in result_starts:
padded.append({
**nb,
"bi_score": 0.0,
"ce_logit": -9.0,
"entity_overlap": 0.0,
"visual_score": 0.0,
"confidence": nb.get("confidence", 0.1),
})
result_starts.add(nb["start"])
break
return sorted(padded, key=lambda x: x["start"])
# โ”€โ”€ Global sampling โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
def sample_chunks_for_global(chunks: list[dict], n_target: int) -> list[dict]:
"""Pick n_target chunks evenly distributed across the video."""
n_target = max(3, min(n_target, len(chunks)))
if n_target >= len(chunks):
return chunks
indices = [int(i * (len(chunks) - 1) / (n_target - 1)) for i in range(n_target)]
return [chunks[i] for i in indices]
# โ”€โ”€ Merge regions โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
def merge_regions(results: list[dict]) -> list[dict]:
if not results:
return []
sorted_r = sorted(results, key=lambda x: x["start"])
merged, current = [], sorted_r[0].copy()
current["_frags"] = [sorted_r[0]]
for nxt in sorted_r[1:]:
if nxt["start"] - current["end"] <= MERGE_GAP_SEC:
current["end"] = max(current["end"], nxt["end"])
current["text"] += " " + nxt["text"]
current["_frags"].append(nxt)
else:
merged.append(_finalize_region(current))
current = nxt.copy()
current["_frags"] = [nxt]
merged.append(_finalize_region(current))
return [r for r in merged if (r["end"] - r["start"]) >= REGION_MIN_DURATION]
def _finalize_region(region: dict) -> dict:
frags = region.pop("_frags")
total_dur = sum(max(f["end"] - f["start"], 0.1) for f in frags)
w_conf = sum(
f["confidence"] * max(f["end"] - f["start"], 0.1) for f in frags
) / total_dur
duration = region["end"] - region["start"]
w_conf *= min(1.0, duration / 20.0)
visual = max((f.get("visual_score", 0.0) for f in frags), default=0.0)
return {**region, "confidence": round(w_conf, 3), "visual_score": round(visual, 3)}