Spaces:
Running on Zero
Running on Zero
Download src/data/query_gen.py from Kagzy/ResearchOS: direct link, hf CLI and curl.
- Browser
- Download file 7.51 kB
-
https://huggingface.co/spaces/Kagzy/ResearchOS/resolve/main/src/data/query_gen.py
- Command line
-
hf download hf://spaces/Kagzy/ResearchOS/src/data/query_gen.py
-
curl -L -o query_gen.py https://huggingface.co/spaces/Kagzy/ResearchOS/resolve/main/src/data/query_gen.py
7.51 kB
| """Generate natural-language evaluation queries from papers (doc2query / InPars). | |
| Why this exists | |
| --------------- | |
| The sentence-based benchmark in scripts/eval_retrieval.py uses verbatim text | |
| from the target paper as the query. That hands BM25 an exact lexical match and | |
| is why it scores 0.89 there. It is not what a user types. | |
| Here the model reads a paper and writes the question a researcher would type to | |
| FIND it. The label is still structural, not an opinion: the question was written | |
| from that paper, so that paper answers it. The model generates the *query*, never | |
| the *verdict* - which is what separates this from LLM-as-judge. | |
| The prompt forbids reusing the paper's title, acronyms and coined terms. Without | |
| that the model paraphrases the title, the lexical gift comes straight back, and | |
| the benchmark measures nothing new. | |
| Known weakness: false negatives. A generated question about paper A may also be | |
| answered by B and C, but only A is marked relevant, so absolute recall reads low. | |
| Every configuration is penalised identically, so comparisons stay valid. | |
| `judge.py` recovers most of the loss by judging a pooled candidate set. | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import random | |
| import re | |
| import threading | |
| import time | |
| from concurrent.futures import ThreadPoolExecutor | |
| from dataclasses import asdict, dataclass | |
| from pathlib import Path | |
| from src.api.llm import _complete | |
| from src.data.arxiv_source import Paper | |
| QUERY_SYSTEM_PROMPT = """You write realistic search queries for a research-paper \ | |
| search engine. | |
| You are shown one paper. Write ONE query that a researcher would type into a search \ | |
| box when they are looking for work like this but have never seen this paper. | |
| Rules: | |
| - 8 to 20 words. Natural language. A question or a topic phrase, not a title. | |
| - Describe the PROBLEM and the APPROACH in generic terms. | |
| - Do NOT reuse the paper's title, its acronym, its model name, or any term the \ | |
| authors coined. If the paper calls its method "FlowMPC", never write "FlowMPC". | |
| - Do NOT copy distinctive multi-word phrases from the text verbatim. | |
| - Write what someone searching would plausibly know to ask, not what the paper calls \ | |
| itself. | |
| Output only the query. No preamble, no quotes, no explanation.""" | |
| class GeneratedQuery: | |
| query_id: str | |
| query: str | |
| gold_paper_id: str | |
| gold_title: str | |
| def _looks_leaky(query: str, title: str, min_overlap: int = 4) -> bool: | |
| """True when the query copies a long run of words from the title.""" | |
| def words(text: str) -> list[str]: | |
| return re.sub(r"[^a-z0-9 ]+", " ", text.lower()).split() | |
| q, t = words(query), words(title) | |
| if len(q) < min_overlap or len(t) < min_overlap: | |
| return False | |
| t_grams = {tuple(t[i : i + min_overlap]) for i in range(len(t) - min_overlap + 1)} | |
| return any(tuple(q[i : i + min_overlap]) in t_grams for i in range(len(q) - min_overlap + 1)) | |
| def _clean(text: str) -> str: | |
| text = text.strip().strip('"').strip("'").strip() | |
| text = re.sub(r"^(query|question|search)\s*[:\-]\s*", "", text, flags=re.IGNORECASE) | |
| return re.sub(r"\s+", " ", text).strip() | |
| def generate_one( | |
| paper: Paper, | |
| context: str, | |
| model: str, | |
| max_tokens: int, | |
| temperature: float, | |
| retries: int = 4, | |
| ) -> tuple[str | None, str]: | |
| """One query for one paper. | |
| Returns (query, reason). reason is "ok", "leak", "malformed" or "api". | |
| These were previously collapsed into a single None, which meant Groq | |
| rate-limit errors were reported as "rejected for copying the title" - the | |
| run looked like a strict filter working when it was actually being throttled. | |
| """ | |
| user = ( | |
| f"Title: {paper.title}\n\n" | |
| f"Abstract: {paper.abstract[:1200]}\n\n" | |
| f"Excerpt: {context[:900]}\n\n" | |
| "Write the search query." | |
| ) | |
| raw = None | |
| for attempt in range(retries): | |
| try: | |
| raw = _complete(QUERY_SYSTEM_PROMPT, user, model, max_tokens, temperature) | |
| break | |
| except Exception as exc: # noqa: BLE001 | |
| message = str(exc).lower() | |
| transient = "rate" in message or "429" in message or "timeout" in message | |
| if attempt == retries - 1 or not transient: | |
| return None, "api" | |
| time.sleep(2.0 * (2**attempt) + random.random()) | |
| if raw is None: | |
| return None, "api" | |
| query = _clean(raw) | |
| if not (5 <= len(query.split()) <= 40): | |
| return None, "malformed" | |
| if _looks_leaky(query, paper.title): | |
| return None, "leak" | |
| return query, "ok" | |
| def generate_queries( | |
| papers: list[Paper], | |
| context_by_paper: dict[str, str], | |
| n_queries: int, | |
| model: str, | |
| out_path: str | Path, | |
| max_tokens: int = 900, | |
| temperature: float = 0.7, | |
| workers: int = 6, | |
| rng: random.Random | None = None, | |
| ) -> list[GeneratedQuery]: | |
| """Generate queries in parallel, writing incrementally so a crash costs nothing.""" | |
| rng = rng or random.Random(0) | |
| target = Path(out_path) | |
| target.parent.mkdir(parents=True, exist_ok=True) | |
| done: dict[str, GeneratedQuery] = {} | |
| if target.exists(): | |
| for line in target.read_text(encoding="utf-8").splitlines(): | |
| if line.strip(): | |
| item = GeneratedQuery(**json.loads(line)) | |
| done[item.gold_paper_id] = item | |
| print(f"[qgen] resuming: {len(done)} already generated") | |
| candidates = [p for p in papers if p.paper_id in context_by_paper and p.paper_id not in done] | |
| rng.shuffle(candidates) | |
| # Oversample: roughly half get rejected for copying the title, so taking | |
| # exactly `n_queries` candidates permanently loses a slot per rejection. | |
| wanted = max(0, n_queries - len(done)) | |
| candidates = candidates[: wanted * 3] | |
| if not candidates: | |
| return list(done.values()) | |
| lock = threading.Lock() | |
| handle = open(target, "a", encoding="utf-8") | |
| counts = {"leak": 0, "api": 0, "malformed": 0} | |
| def work(paper: Paper) -> None: | |
| if len(done) >= n_queries: | |
| return | |
| query, reason = generate_one( | |
| paper, context_by_paper[paper.paper_id], model, max_tokens, temperature | |
| ) | |
| if query is None: | |
| with lock: | |
| counts[reason] = counts.get(reason, 0) + 1 | |
| return | |
| item = GeneratedQuery( | |
| query_id=f"g{paper.paper_id}", | |
| query=query, | |
| gold_paper_id=paper.paper_id, | |
| gold_title=paper.title, | |
| ) | |
| with lock: | |
| done[paper.paper_id] = item | |
| handle.write(json.dumps(asdict(item), ensure_ascii=False) + "\n") | |
| handle.flush() | |
| if len(done) % 25 == 0: | |
| print(f"[qgen] {len(done)}/{n_queries} generated | {counts}") | |
| try: | |
| with ThreadPoolExecutor(max_workers=workers) as pool: | |
| list(pool.map(work, candidates)) | |
| finally: | |
| handle.close() | |
| print( | |
| f"[qgen] {len(done)} queries | {counts['leak']} rejected for copying the title, " | |
| f"{counts['api']} API failures, {counts['malformed']} malformed" | |
| ) | |
| return list(done.values()) | |
| def read_generated(path: str | Path) -> list[GeneratedQuery]: | |
| with open(path, encoding="utf-8") as handle: | |
| return [GeneratedQuery(**json.loads(line)) for line in handle if line.strip()] | |