import re import numpy as np from config import ( BI_ENCODER_TOP_K, CHUNK_MAX_DURATION, CHUNK_MIN_DURATION, CHUNK_SPLIT_THRESHOLD, CHUNK_WINDOW, CROSS_ENCODER_TOP_K, DEBUG, GLOBAL_MAX_CHUNKS, MERGE_GAP_SEC, REGION_MIN_DURATION, VISUAL_WEIGHT, ) from models import bi_encoder, cross_encoder from utils import _build_chunk, _cosine, _sigmoid from visual import span_visual_score # ── Entity extraction ────────────────────────────────────────────────────────── _STOPWORDS = { "the","a","an","and","or","but","in","on","at","to","for","of","is","are", "was","were","be","been","that","this","it","he","she","they","we","i", "w","z","że","na","do","się","nie","to","jest","są","by","przez","po", "le","la","les","un","une","des","du","et","en","à","de","est","il","elle", "ein","eine","der","die","das","und","ist","zu","mit","von","dem", } def extract_entities(text: str) -> set[str]: entities: set[str] = set() for match in re.finditer(r'"([^"]{2,40})"', text): entities.add(match.group(1).lower().strip()) for match in re.finditer( r'\b([A-ZÁÉÍÓÚÀÂÊÎÔÙÄËÏÖÜĄĆĘŁŃÓŚŹŻ][a-záéíóúàâêîôùäëïöüąćęłńóśźż]+' r'(?:\s[A-ZÁÉÍÓÚÀÂÊÎÔÙÄËÏÖÜĄĆĘŁŃÓŚŹŻ][a-zÀ-žąćęłńóśźż]+)*)\b', text, ): token = match.group(1).lower().strip() if token not in _STOPWORDS and len(token) > 2: entities.add(token) for token in re.findall( r'\b[A-ZÁÉÍÓÚÀÂÊÎÔÙÄËÏÖÜĄĆĘŁŃÓŚŹŻ][a-zÀ-žąćęłńóśźż]{2,}\b', text ): t = token.lower() if t not in _STOPWORDS: entities.add(t) for match in re.finditer(r'\b([A-Za-z]+[-\s]?\d+[\w]*)\b', text): entities.add(match.group(1).lower()) return entities def entity_overlap_score( prompt_entities: set[str], chunk_entities: set[str], chunk_text: str = "" ) -> float: if not prompt_entities: return 0.0 chunk_text_lower = chunk_text.lower() chunk_words = set(re.findall(r'\w+', chunk_text_lower)) matched = 0 for pe in prompt_entities: # 1. entity-to-entity overlap if any(pe in ce or ce in pe for ce in chunk_entities): matched += 1; continue # 2. substring in raw text (>= 3 chars — catches "won", "who", etc.) if len(pe) >= 3 and pe in chunk_text_lower: matched += 1; continue # 3. exact word-level match (handles ASR spacing differences) if pe in chunk_words: matched += 1; continue # 4. compound-word split: "manhunt" → check "man" + "hunt" both in text if len(pe) >= 5: for split in range(2, len(pe) - 1): left, right = pe[:split], pe[split:] if len(left) >= 3 and len(right) >= 3 and left in chunk_words and right in chunk_words: matched += 1; break return matched / len(prompt_entities) def add_entities_to_chunks(chunks: list[dict]) -> list[dict]: for chunk in chunks: chunk["entities"] = extract_entities(chunk["text"]) return chunks # ── Semantic chunking ────────────────────────────────────────────────────────── def semantic_chunk(segments: list[dict]) -> list[dict]: if not segments: return [] texts = [s["text"] for s in segments] embeddings = bi_encoder.encode(texts, convert_to_numpy=True, batch_size=64) # Adaptive threshold: calibrate to this video's similarity distribution if len(embeddings) >= 4: consec_sims = [ max(_cosine(embeddings[i], embeddings[i - 1]), 0.0) for i in range(1, len(embeddings)) ] mean_sim = float(np.mean(consec_sims)) std_sim = float(np.std(consec_sims)) threshold = float(np.clip(mean_sim - 0.5 * std_sim, 0.30, 0.62)) else: mean_sim = std_sim = 0.0 threshold = CHUNK_SPLIT_THRESHOLD if DEBUG: print( f"[DEBUG] Adaptive chunk threshold: {threshold:.3f} " f"(mean_sim={mean_sim:.3f}, std={std_sim:.3f})" ) chunks, current, cur_emb = [], [segments[0]], [embeddings[0]] for i in range(1, len(segments)): chunk_dur = segments[i]["end"] - current[0]["start"] window_mean = np.mean(cur_emb[-CHUNK_WINDOW:], axis=0) sim = max(_cosine(embeddings[i], window_mean), _cosine(embeddings[i], cur_emb[-1])) long_enough = (current[-1]["end"] - current[0]["start"]) >= CHUNK_MIN_DURATION if (sim < threshold or chunk_dur >= CHUNK_MAX_DURATION) and long_enough: chunks.append(_build_chunk(current, cur_emb)) current, cur_emb = [segments[i]], [embeddings[i]] else: current.append(segments[i]) cur_emb.append(embeddings[i]) if current: chunks.append(_build_chunk(current, cur_emb)) print(f"[NINA] {len(segments)} segments → {len(chunks)} chunks (threshold={threshold:.3f}).") return chunks # ── Query router ─────────────────────────────────────────────────────────────── _GLOBAL_PATTERNS = re.compile( r'\b(' r'summari[sz]e|summary|overview|recap|tl;?dr|key\s+points?|main\s+points?|' r'key\s+takeaways?|main\s+idea|gist|' r'what(?:\'s| is| was)\s+(?:this|the|it)\s+(?:video|recording|clip|all\s+)?\s*about|' r'what(?:\'s| is| was)\s+(?:this|the)\s+(?:video|recording|clip)|' r'describe\s+(?:the|this)\s+(?:video|recording|clip)|' r'streszcz\w*|podsumuj\w*|podsumowanie|' r'o\s+czym\s+(?:jest|to|opowiada|mówi)|czego\s+dotyczy|' r'gł[oó]wn\w+\s+(?:punkt\w*|my[sś]l\w*|tez\w*)|og[oó]ln\w+\s+zarys' r')\b', re.IGNORECASE, ) def classify_query(prompt: str) -> str: """Pattern-based pass-1 router. Returns 'global' or 'specific'.""" if _GLOBAL_PATTERNS.search(prompt): return "global" return "specific" # ── Bi-encoder retrieval ─────────────────────────────────────────────────────── def bi_encode_retrieve(chunks: list[dict], prompt: str) -> tuple[list[dict], float]: prompt_emb = bi_encoder.encode([prompt], convert_to_numpy=True)[0] qn = prompt_emb / (np.linalg.norm(prompt_emb) + 1e-9) scores = [] for c in chunks: seg_embs = c.get("seg_embs") if seg_embs is not None and len(seg_embs) > 1: # MaxSim: highest cosine among individual segment embeddings en = seg_embs / (np.linalg.norm(seg_embs, axis=1, keepdims=True) + 1e-9) scores.append(float(np.max(en @ qn))) else: scores.append(_cosine(prompt_emb, c["emb"])) top_k = min(BI_ENCODER_TOP_K, len(chunks)) top_idx = np.argsort(scores)[::-1][:top_k] if DEBUG: print(f"[DEBUG] Bi top-3 scores (MaxSim): " + ", ".join(f"{scores[i]:.3f}" for i in top_idx[:3])) best = scores[top_idx[0]] if len(top_idx) else 0.0 return ( [{**chunks[i], "bi_score": round(scores[i], 3)} for i in top_idx], best, ) # ── Cross-encoder rerank + audio/visual fusion ──────────────────────────────── _LONG_CHUNK_WORDS = 120 # cross-encoder context limit guard def rerank_and_score( candidates: list[dict], prompt: str, times: np.ndarray, norm: np.ndarray, ) -> list[dict]: # Build a flat pairs list; track per-candidate slice indices all_pairs: list[list[str]] = [] pair_ranges: list[tuple[int,int]] = [] for c in candidates: start = len(all_pairs) seg_texts = c.get("seg_texts", []) if len(c["text"].split()) > _LONG_CHUNK_WORDS and len(seg_texts) > 1: for t in seg_texts: if t.strip(): all_pairs.append([prompt, t]) if len(all_pairs) == start: # short chunk or no seg_texts all_pairs.append([prompt, c["text"]]) pair_ranges.append((start, len(all_pairs))) all_logits = cross_encoder.predict(all_pairs) # Max-segment score per candidate: ensures long chunks aren't hurt by truncation logits = [float(np.max(all_logits[s:e])) for s, e in pair_ranges] prompt_entities = extract_entities(prompt) prompt_words = { w.lower() for w in re.findall(r'[\wÀ-ž]{3,}', prompt) if w.lower() not in _STOPWORDS } all_prompt_tokens = prompt_entities | prompt_words if DEBUG and all_prompt_tokens: print(f"[DEBUG] Prompt tokens for entity match: {all_prompt_tokens}") vw = VISUAL_WEIGHT if len(norm) else 0.0 scored = [] for candidate, logit in zip(candidates, logits): chunk_entities = candidate.get("entities", set()) chunk_text = candidate.get("text", "") overlap = entity_overlap_score(all_prompt_tokens, chunk_entities, chunk_text) ce_conf = _sigmoid(float(logit)) audio_score = 0.6 * ce_conf + 0.4 * overlap visual_score = span_visual_score(times, norm, candidate["start"], candidate["end"]) final = (1.0 - vw) * audio_score + vw * visual_score scored.append({ **candidate, "ce_logit": round(float(logit), 3), "entity_overlap": round(overlap, 3), "visual_score": round(visual_score, 3), "confidence": round(min(1.0, final), 3), }) ranked = sorted(scored, key=lambda x: x["confidence"], reverse=True) if DEBUG: print("[DEBUG] Top-3 after reranking (audio+visual):") for r in ranked[:3]: print( f" ce_logit={r['ce_logit']:.2f} " f"bi={r['bi_score']:.3f} " f"entity={r['entity_overlap']:.3f} " f"visual={r['visual_score']:.3f} " f"conf={r['confidence']:.3f} " f"\"{r['text'][:60]}\"" ) return ranked[:CROSS_ENCODER_TOP_K] # ── Hybrid padding ───────────────────────────────────────────────────────────── def pad_with_neighbours(results: list[dict], chunks: list[dict]) -> list[dict]: """Include immediate neighbours of each retrieved chunk (HYBRID mode).""" result_starts = {r["start"] for r in results} padded = list(results) for r in results: for i, c in enumerate(chunks): if abs(c["start"] - r["start"]) < 0.5: for neighbour_i in [i - 1, i + 1]: if 0 <= neighbour_i < len(chunks): nb = chunks[neighbour_i] if nb["start"] not in result_starts: padded.append({ **nb, "bi_score": 0.0, "ce_logit": -9.0, "entity_overlap": 0.0, "visual_score": 0.0, "confidence": nb.get("confidence", 0.1), }) result_starts.add(nb["start"]) break return sorted(padded, key=lambda x: x["start"]) # ── Global sampling ──────────────────────────────────────────────────────────── def sample_chunks_for_global(chunks: list[dict], n_target: int) -> list[dict]: """Pick n_target chunks evenly distributed across the video.""" n_target = max(3, min(n_target, len(chunks))) if n_target >= len(chunks): return chunks indices = [int(i * (len(chunks) - 1) / (n_target - 1)) for i in range(n_target)] return [chunks[i] for i in indices] # ── Merge regions ────────────────────────────────────────────────────────────── def merge_regions(results: list[dict]) -> list[dict]: if not results: return [] sorted_r = sorted(results, key=lambda x: x["start"]) merged, current = [], sorted_r[0].copy() current["_frags"] = [sorted_r[0]] for nxt in sorted_r[1:]: if nxt["start"] - current["end"] <= MERGE_GAP_SEC: current["end"] = max(current["end"], nxt["end"]) current["text"] += " " + nxt["text"] current["_frags"].append(nxt) else: merged.append(_finalize_region(current)) current = nxt.copy() current["_frags"] = [nxt] merged.append(_finalize_region(current)) return [r for r in merged if (r["end"] - r["start"]) >= REGION_MIN_DURATION] def _finalize_region(region: dict) -> dict: frags = region.pop("_frags") total_dur = sum(max(f["end"] - f["start"], 0.1) for f in frags) w_conf = sum( f["confidence"] * max(f["end"] - f["start"], 0.1) for f in frags ) / total_dur duration = region["end"] - region["start"] w_conf *= min(1.0, duration / 20.0) visual = max((f.get("visual_score", 0.0) for f in frags), default=0.0) return {**region, "confidence": round(w_conf, 3), "visual_score": round(visual, 3)}