Spaces:
Paused
Paused
Download retrieval.py from NextNeural/NINA: direct link, hf CLI and curl.
- Browser
- Download file 13.8 kB
-
https://huggingface.co/spaces/NextNeural/NINA/resolve/main/retrieval.py
- Command line
-
hf download hf://spaces/NextNeural/NINA/retrieval.py
-
curl -L -o retrieval.py https://huggingface.co/spaces/NextNeural/NINA/resolve/main/retrieval.py
13.8 kB
| import re | |
| import numpy as np | |
| from config import ( | |
| BI_ENCODER_TOP_K, CHUNK_MAX_DURATION, CHUNK_MIN_DURATION, | |
| CHUNK_SPLIT_THRESHOLD, CHUNK_WINDOW, CROSS_ENCODER_TOP_K, | |
| DEBUG, GLOBAL_MAX_CHUNKS, MERGE_GAP_SEC, REGION_MIN_DURATION, | |
| VISUAL_WEIGHT, | |
| ) | |
| from models import bi_encoder, cross_encoder | |
| from utils import _build_chunk, _cosine, _sigmoid | |
| from visual import span_visual_score | |
| # โโ Entity extraction โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ | |
| _STOPWORDS = { | |
| "the","a","an","and","or","but","in","on","at","to","for","of","is","are", | |
| "was","were","be","been","that","this","it","he","she","they","we","i", | |
| "w","z","ลผe","na","do","siฤ","nie","to","jest","sฤ ","by","przez","po", | |
| "le","la","les","un","une","des","du","et","en","ร ","de","est","il","elle", | |
| "ein","eine","der","die","das","und","ist","zu","mit","von","dem", | |
| } | |
| def extract_entities(text: str) -> set[str]: | |
| entities: set[str] = set() | |
| for match in re.finditer(r'"([^"]{2,40})"', text): | |
| entities.add(match.group(1).lower().strip()) | |
| for match in re.finditer( | |
| r'\b([A-Zรรรรรรรรรรรรรรรรฤฤฤลลรลลนลป][a-zรกรฉรญรณรบร รขรชรฎรดรนรครซรฏรถรผฤ ฤฤลลรณลลบลผ]+' | |
| r'(?:\s[A-Zรรรรรรรรรรรรรรรรฤฤฤลลรลลนลป][a-zร-ลพฤ ฤฤลลรณลลบลผ]+)*)\b', | |
| text, | |
| ): | |
| token = match.group(1).lower().strip() | |
| if token not in _STOPWORDS and len(token) > 2: | |
| entities.add(token) | |
| for token in re.findall( | |
| r'\b[A-Zรรรรรรรรรรรรรรรรฤฤฤลลรลลนลป][a-zร-ลพฤ ฤฤลลรณลลบลผ]{2,}\b', text | |
| ): | |
| t = token.lower() | |
| if t not in _STOPWORDS: | |
| entities.add(t) | |
| for match in re.finditer(r'\b([A-Za-z]+[-\s]?\d+[\w]*)\b', text): | |
| entities.add(match.group(1).lower()) | |
| return entities | |
| def entity_overlap_score( | |
| prompt_entities: set[str], chunk_entities: set[str], chunk_text: str = "" | |
| ) -> float: | |
| if not prompt_entities: | |
| return 0.0 | |
| chunk_text_lower = chunk_text.lower() | |
| chunk_words = set(re.findall(r'\w+', chunk_text_lower)) | |
| matched = 0 | |
| for pe in prompt_entities: | |
| # 1. entity-to-entity overlap | |
| if any(pe in ce or ce in pe for ce in chunk_entities): | |
| matched += 1; continue | |
| # 2. substring in raw text (>= 3 chars โ catches "won", "who", etc.) | |
| if len(pe) >= 3 and pe in chunk_text_lower: | |
| matched += 1; continue | |
| # 3. exact word-level match (handles ASR spacing differences) | |
| if pe in chunk_words: | |
| matched += 1; continue | |
| # 4. compound-word split: "manhunt" โ check "man" + "hunt" both in text | |
| if len(pe) >= 5: | |
| for split in range(2, len(pe) - 1): | |
| left, right = pe[:split], pe[split:] | |
| if len(left) >= 3 and len(right) >= 3 and left in chunk_words and right in chunk_words: | |
| matched += 1; break | |
| return matched / len(prompt_entities) | |
| def add_entities_to_chunks(chunks: list[dict]) -> list[dict]: | |
| for chunk in chunks: | |
| chunk["entities"] = extract_entities(chunk["text"]) | |
| return chunks | |
| # โโ Semantic chunking โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ | |
| def semantic_chunk(segments: list[dict]) -> list[dict]: | |
| if not segments: | |
| return [] | |
| texts = [s["text"] for s in segments] | |
| embeddings = bi_encoder.encode(texts, convert_to_numpy=True, batch_size=64) | |
| # Adaptive threshold: calibrate to this video's similarity distribution | |
| if len(embeddings) >= 4: | |
| consec_sims = [ | |
| max(_cosine(embeddings[i], embeddings[i - 1]), 0.0) | |
| for i in range(1, len(embeddings)) | |
| ] | |
| mean_sim = float(np.mean(consec_sims)) | |
| std_sim = float(np.std(consec_sims)) | |
| threshold = float(np.clip(mean_sim - 0.5 * std_sim, 0.30, 0.62)) | |
| else: | |
| mean_sim = std_sim = 0.0 | |
| threshold = CHUNK_SPLIT_THRESHOLD | |
| if DEBUG: | |
| print( | |
| f"[DEBUG] Adaptive chunk threshold: {threshold:.3f} " | |
| f"(mean_sim={mean_sim:.3f}, std={std_sim:.3f})" | |
| ) | |
| chunks, current, cur_emb = [], [segments[0]], [embeddings[0]] | |
| for i in range(1, len(segments)): | |
| chunk_dur = segments[i]["end"] - current[0]["start"] | |
| window_mean = np.mean(cur_emb[-CHUNK_WINDOW:], axis=0) | |
| sim = max(_cosine(embeddings[i], window_mean), | |
| _cosine(embeddings[i], cur_emb[-1])) | |
| long_enough = (current[-1]["end"] - current[0]["start"]) >= CHUNK_MIN_DURATION | |
| if (sim < threshold or chunk_dur >= CHUNK_MAX_DURATION) and long_enough: | |
| chunks.append(_build_chunk(current, cur_emb)) | |
| current, cur_emb = [segments[i]], [embeddings[i]] | |
| else: | |
| current.append(segments[i]) | |
| cur_emb.append(embeddings[i]) | |
| if current: | |
| chunks.append(_build_chunk(current, cur_emb)) | |
| print(f"[NINA] {len(segments)} segments โ {len(chunks)} chunks (threshold={threshold:.3f}).") | |
| return chunks | |
| # โโ Query router โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ | |
| _GLOBAL_PATTERNS = re.compile( | |
| r'\b(' | |
| r'summari[sz]e|summary|overview|recap|tl;?dr|key\s+points?|main\s+points?|' | |
| r'key\s+takeaways?|main\s+idea|gist|' | |
| r'what(?:\'s| is| was)\s+(?:this|the|it)\s+(?:video|recording|clip|all\s+)?\s*about|' | |
| r'what(?:\'s| is| was)\s+(?:this|the)\s+(?:video|recording|clip)|' | |
| r'describe\s+(?:the|this)\s+(?:video|recording|clip)|' | |
| r'streszcz\w*|podsumuj\w*|podsumowanie|' | |
| r'o\s+czym\s+(?:jest|to|opowiada|mรณwi)|czego\s+dotyczy|' | |
| r'gล[oรณ]wn\w+\s+(?:punkt\w*|my[sล]l\w*|tez\w*)|og[oรณ]ln\w+\s+zarys' | |
| r')\b', | |
| re.IGNORECASE, | |
| ) | |
| def classify_query(prompt: str) -> str: | |
| """Pattern-based pass-1 router. Returns 'global' or 'specific'.""" | |
| if _GLOBAL_PATTERNS.search(prompt): | |
| return "global" | |
| return "specific" | |
| # โโ Bi-encoder retrieval โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ | |
| def bi_encode_retrieve(chunks: list[dict], prompt: str) -> tuple[list[dict], float]: | |
| prompt_emb = bi_encoder.encode([prompt], convert_to_numpy=True)[0] | |
| qn = prompt_emb / (np.linalg.norm(prompt_emb) + 1e-9) | |
| scores = [] | |
| for c in chunks: | |
| seg_embs = c.get("seg_embs") | |
| if seg_embs is not None and len(seg_embs) > 1: | |
| # MaxSim: highest cosine among individual segment embeddings | |
| en = seg_embs / (np.linalg.norm(seg_embs, axis=1, keepdims=True) + 1e-9) | |
| scores.append(float(np.max(en @ qn))) | |
| else: | |
| scores.append(_cosine(prompt_emb, c["emb"])) | |
| top_k = min(BI_ENCODER_TOP_K, len(chunks)) | |
| top_idx = np.argsort(scores)[::-1][:top_k] | |
| if DEBUG: | |
| print(f"[DEBUG] Bi top-3 scores (MaxSim): " + | |
| ", ".join(f"{scores[i]:.3f}" for i in top_idx[:3])) | |
| best = scores[top_idx[0]] if len(top_idx) else 0.0 | |
| return ( | |
| [{**chunks[i], "bi_score": round(scores[i], 3)} for i in top_idx], | |
| best, | |
| ) | |
| # โโ Cross-encoder rerank + audio/visual fusion โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ | |
| _LONG_CHUNK_WORDS = 120 # cross-encoder context limit guard | |
| def rerank_and_score( | |
| candidates: list[dict], prompt: str, | |
| times: np.ndarray, norm: np.ndarray, | |
| ) -> list[dict]: | |
| # Build a flat pairs list; track per-candidate slice indices | |
| all_pairs: list[list[str]] = [] | |
| pair_ranges: list[tuple[int,int]] = [] | |
| for c in candidates: | |
| start = len(all_pairs) | |
| seg_texts = c.get("seg_texts", []) | |
| if len(c["text"].split()) > _LONG_CHUNK_WORDS and len(seg_texts) > 1: | |
| for t in seg_texts: | |
| if t.strip(): | |
| all_pairs.append([prompt, t]) | |
| if len(all_pairs) == start: # short chunk or no seg_texts | |
| all_pairs.append([prompt, c["text"]]) | |
| pair_ranges.append((start, len(all_pairs))) | |
| all_logits = cross_encoder.predict(all_pairs) | |
| # Max-segment score per candidate: ensures long chunks aren't hurt by truncation | |
| logits = [float(np.max(all_logits[s:e])) for s, e in pair_ranges] | |
| prompt_entities = extract_entities(prompt) | |
| prompt_words = { | |
| w.lower() for w in re.findall(r'[\wร-ลพ]{3,}', prompt) | |
| if w.lower() not in _STOPWORDS | |
| } | |
| all_prompt_tokens = prompt_entities | prompt_words | |
| if DEBUG and all_prompt_tokens: | |
| print(f"[DEBUG] Prompt tokens for entity match: {all_prompt_tokens}") | |
| vw = VISUAL_WEIGHT if len(norm) else 0.0 | |
| scored = [] | |
| for candidate, logit in zip(candidates, logits): | |
| chunk_entities = candidate.get("entities", set()) | |
| chunk_text = candidate.get("text", "") | |
| overlap = entity_overlap_score(all_prompt_tokens, chunk_entities, chunk_text) | |
| ce_conf = _sigmoid(float(logit)) | |
| audio_score = 0.6 * ce_conf + 0.4 * overlap | |
| visual_score = span_visual_score(times, norm, candidate["start"], candidate["end"]) | |
| final = (1.0 - vw) * audio_score + vw * visual_score | |
| scored.append({ | |
| **candidate, | |
| "ce_logit": round(float(logit), 3), | |
| "entity_overlap": round(overlap, 3), | |
| "visual_score": round(visual_score, 3), | |
| "confidence": round(min(1.0, final), 3), | |
| }) | |
| ranked = sorted(scored, key=lambda x: x["confidence"], reverse=True) | |
| if DEBUG: | |
| print("[DEBUG] Top-3 after reranking (audio+visual):") | |
| for r in ranked[:3]: | |
| print( | |
| f" ce_logit={r['ce_logit']:.2f} " | |
| f"bi={r['bi_score']:.3f} " | |
| f"entity={r['entity_overlap']:.3f} " | |
| f"visual={r['visual_score']:.3f} " | |
| f"conf={r['confidence']:.3f} " | |
| f"\"{r['text'][:60]}\"" | |
| ) | |
| return ranked[:CROSS_ENCODER_TOP_K] | |
| # โโ Hybrid padding โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ | |
| def pad_with_neighbours(results: list[dict], chunks: list[dict]) -> list[dict]: | |
| """Include immediate neighbours of each retrieved chunk (HYBRID mode).""" | |
| result_starts = {r["start"] for r in results} | |
| padded = list(results) | |
| for r in results: | |
| for i, c in enumerate(chunks): | |
| if abs(c["start"] - r["start"]) < 0.5: | |
| for neighbour_i in [i - 1, i + 1]: | |
| if 0 <= neighbour_i < len(chunks): | |
| nb = chunks[neighbour_i] | |
| if nb["start"] not in result_starts: | |
| padded.append({ | |
| **nb, | |
| "bi_score": 0.0, | |
| "ce_logit": -9.0, | |
| "entity_overlap": 0.0, | |
| "visual_score": 0.0, | |
| "confidence": nb.get("confidence", 0.1), | |
| }) | |
| result_starts.add(nb["start"]) | |
| break | |
| return sorted(padded, key=lambda x: x["start"]) | |
| # โโ Global sampling โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ | |
| def sample_chunks_for_global(chunks: list[dict], n_target: int) -> list[dict]: | |
| """Pick n_target chunks evenly distributed across the video.""" | |
| n_target = max(3, min(n_target, len(chunks))) | |
| if n_target >= len(chunks): | |
| return chunks | |
| indices = [int(i * (len(chunks) - 1) / (n_target - 1)) for i in range(n_target)] | |
| return [chunks[i] for i in indices] | |
| # โโ Merge regions โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ | |
| def merge_regions(results: list[dict]) -> list[dict]: | |
| if not results: | |
| return [] | |
| sorted_r = sorted(results, key=lambda x: x["start"]) | |
| merged, current = [], sorted_r[0].copy() | |
| current["_frags"] = [sorted_r[0]] | |
| for nxt in sorted_r[1:]: | |
| if nxt["start"] - current["end"] <= MERGE_GAP_SEC: | |
| current["end"] = max(current["end"], nxt["end"]) | |
| current["text"] += " " + nxt["text"] | |
| current["_frags"].append(nxt) | |
| else: | |
| merged.append(_finalize_region(current)) | |
| current = nxt.copy() | |
| current["_frags"] = [nxt] | |
| merged.append(_finalize_region(current)) | |
| return [r for r in merged if (r["end"] - r["start"]) >= REGION_MIN_DURATION] | |
| def _finalize_region(region: dict) -> dict: | |
| frags = region.pop("_frags") | |
| total_dur = sum(max(f["end"] - f["start"], 0.1) for f in frags) | |
| w_conf = sum( | |
| f["confidence"] * max(f["end"] - f["start"], 0.1) for f in frags | |
| ) / total_dur | |
| duration = region["end"] - region["start"] | |
| w_conf *= min(1.0, duration / 20.0) | |
| visual = max((f.get("visual_score", 0.0) for f in frags), default=0.0) | |
| return {**region, "confidence": round(w_conf, 3), "visual_score": round(visual, 3)} | |