Spaces:
Runtime error
Runtime error
File size: 2,341 Bytes
169aba7 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 | from nltk.corpus import wordnet
import Preprocessing
_MATH_MAPPINGS = {
"algebra": ["linear", "equation", "variable"],
"calc": ["calculus", "derivative", "integration", "limit"],
"geom": ["geometry", "surface", "area", "shape", "cone", "pyramid"],
"prob": ["probability", "statistics", "simulation", "chance"],
"math": ["mathematics", "mathematical", "thinking"],
"eq": ["equation", "linear"],
"deriv": ["derivative", "differentiation"],
"limit": ["limits", "calculus"],
"area": ["surface", "geometry"],
}
_MATH_OPERATORS = set("+-=/*^<>{}[]()")
def expand_query_wordnet(query: str) -> str:
terms = Preprocessing.preprocess_text(query)
expanded_terms = set(terms)
for term in terms:
synsets = wordnet.synsets(term)
if not synsets:
continue
added = 0
for lemma in synsets[0].lemmas():
syn = lemma.name().replace("_", " ")
if syn.lower() != term.lower():
expanded_terms.add(syn)
added += 1
if added >= 2:
break
return " ".join(expanded_terms)
def expand_query_bert(query: str) -> str:
terms = Preprocessing.preprocess_text(query)
expanded_terms = set(terms)
for term in terms:
if term in _MATH_MAPPINGS:
expanded_terms.update(_MATH_MAPPINGS[term])
return " ".join(expanded_terms)
def pseudo_relevance_feedback(
query: str,
feedback_results: list,
inverted_index: dict,
top_k: int = 2,
) -> str:
if not feedback_results:
return query
top_docs = {doc_id for doc_id, _ in feedback_results[:top_k]}
term_freqs: dict = {}
for term, postings in inverted_index.items():
for doc_id in top_docs.intersection(postings.keys()):
term_freqs[term] = term_freqs.get(term, 0) + postings[doc_id]
query_terms = set(Preprocessing.preprocess_text(query))
new_terms = [
term for term, _ in sorted(term_freqs.items(), key=lambda x: x[1], reverse=True)
if term not in query_terms
and not any(c in _MATH_OPERATORS for c in term)
and not term.isnumeric()
][:2]
return (query + " " + " ".join(new_terms)).strip() if new_terms else query
|