Spaces:
Runtime error
Runtime error
File size: 1,547 Bytes
f9a9b47 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 | from __future__ import annotations
from collections import Counter
import math
import re
from dataclasses import dataclass, field
from typing import Iterable
TOKEN_RE = re.compile(r"[A-Za-z0-9']+")
def tokenize(text: str) -> list[str]:
return [tok.lower() for tok in TOKEN_RE.findall(text or '')]
def vectorize(text: str) -> Counter[str]:
return Counter(tokenize(text))
def cosine_similarity(left: Counter[str], right: Counter[str]) -> float:
if not left or not right:
return 0.0
keys = set(left) | set(right)
dot = sum(left[k] * right[k] for k in keys)
if dot == 0:
return 0.0
left_norm = math.sqrt(sum(v * v for v in left.values()))
right_norm = math.sqrt(sum(v * v for v in right.values()))
if not left_norm or not right_norm:
return 0.0
return dot / (left_norm * right_norm)
@dataclass
class SimpleEmbeddingIndex:
entries: dict[str, Counter[str]] = field(default_factory=dict)
def add(self, record_id: str, text: str) -> None:
self.entries[record_id] = vectorize(text)
def search(self, query: str, limit: int = 5) -> list[tuple[str, float]]:
qvec = vectorize(query)
scored = [(record_id, cosine_similarity(qvec, vec)) for record_id, vec in self.entries.items()]
return sorted(scored, key=lambda item: item[1], reverse=True)[:limit]
def extract_keywords(text: str, limit: int = 6) -> list[str]:
counts = Counter(tok for tok in tokenize(text) if len(tok) > 2)
return [word for word, _ in counts.most_common(limit)]
|