Spaces:
Running on Zero
Running on Zero
OpenAlex live source, DOI citations, live v2 funnel
Browse files- app.py +6 -2
- configs/ml.cpu.yaml +4 -0
- configs/ml.yaml +4 -0
- src/api/db.py +303 -223
- src/api/ml_app.py +35 -0
- src/api/ml_pipeline.py +114 -19
- src/data/citation.py +86 -10
- src/data/openalex_source.py +307 -0
app.py
CHANGED
|
@@ -152,7 +152,8 @@ def _render_papers(result: dict[str, Any]) -> str:
|
|
| 152 |
lines: list[str] = [diag] if diag else []
|
| 153 |
if result.get("source") == "arxiv_live":
|
| 154 |
lines.append(
|
| 155 |
-
"> Not in the indexed corpus, so these came from a live
|
|
|
|
| 156 |
)
|
| 157 |
elif not result.get("confident"):
|
| 158 |
lines.append("> Weak matches — treat these as leads rather than answers.\n")
|
|
@@ -166,7 +167,10 @@ def _render_papers(result: dict[str, Any]) -> str:
|
|
| 166 |
meta = [m for m in (
|
| 167 |
_authors(paper.get("authors") or []),
|
| 168 |
(paper.get("published") or "")[:10],
|
| 169 |
-
paper.get("
|
|
|
|
|
|
|
|
|
|
| 170 |
f"score {paper.get('score')}",
|
| 171 |
) if m]
|
| 172 |
lines.append(" · ".join(meta))
|
|
|
|
| 152 |
lines: list[str] = [diag] if diag else []
|
| 153 |
if result.get("source") == "arxiv_live":
|
| 154 |
lines.append(
|
| 155 |
+
"> Not in the indexed corpus, so these came from a live search of arXiv and "
|
| 156 |
+
"OpenAlex just now, rescored by the same reranker.\n"
|
| 157 |
)
|
| 158 |
elif not result.get("confident"):
|
| 159 |
lines.append("> Weak matches — treat these as leads rather than answers.\n")
|
|
|
|
| 167 |
meta = [m for m in (
|
| 168 |
_authors(paper.get("authors") or []),
|
| 169 |
(paper.get("published") or "")[:10],
|
| 170 |
+
(paper.get("venue") if paper.get("venue") not in (None, "", "arXiv")
|
| 171 |
+
else paper.get("primary_category")) or "",
|
| 172 |
+
f"{paper['cited_by_count']:,} citations" if paper.get("cited_by_count") else "",
|
| 173 |
+
f"live · {paper['live_source']}" if paper.get("live_source") else "",
|
| 174 |
f"score {paper.get('score')}",
|
| 175 |
) if m]
|
| 176 |
lines.append(" · ".join(meta))
|
configs/ml.cpu.yaml
CHANGED
|
@@ -115,6 +115,10 @@ citation:
|
|
| 115 |
# shared IPs (Hugging Face Spaces) and a throttled call can hang ~50s before
|
| 116 |
# failing; past this budget the answer goes out from the corpus instead.
|
| 117 |
live_budget_s: 5.0
|
|
|
|
|
|
|
|
|
|
|
|
|
| 118 |
fetch_full_text: true # download the PDF for on-demand papers
|
| 119 |
# Cap the on-demand PDF cache. Citation lookup downloads one PDF per
|
| 120 |
# unresolved paper, which grows without bound on a shared deployment.
|
|
|
|
| 115 |
# shared IPs (Hugging Face Spaces) and a throttled call can hang ~50s before
|
| 116 |
# failing; past this budget the answer goes out from the corpus instead.
|
| 117 |
live_budget_s: 5.0
|
| 118 |
+
# Queried in parallel under the one budget above. OpenAlex adds the
|
| 119 |
+
# conference/journal papers with no arXiv preprint, plus DOI lookup; set
|
| 120 |
+
# OPENALEX_API_KEY for a 10x daily budget (keyless works, ~100 searches/day).
|
| 121 |
+
live_sources: [arxiv, openalex]
|
| 122 |
fetch_full_text: true # download the PDF for on-demand papers
|
| 123 |
# Cap the on-demand PDF cache. Citation lookup downloads one PDF per
|
| 124 |
# unresolved paper, which grows without bound on a shared deployment.
|
configs/ml.yaml
CHANGED
|
@@ -105,6 +105,10 @@ citation:
|
|
| 105 |
# shared IPs (Hugging Face Spaces) and a throttled call can hang ~50s before
|
| 106 |
# failing; past this budget the answer goes out from the corpus instead.
|
| 107 |
live_budget_s: 5.0
|
|
|
|
|
|
|
|
|
|
|
|
|
| 108 |
fetch_full_text: true # download the PDF for on-demand papers
|
| 109 |
# Cap the on-demand PDF cache. Citation lookup downloads one PDF per
|
| 110 |
# unresolved paper, which grows without bound on a shared deployment.
|
|
|
|
| 105 |
# shared IPs (Hugging Face Spaces) and a throttled call can hang ~50s before
|
| 106 |
# failing; past this budget the answer goes out from the corpus instead.
|
| 107 |
live_budget_s: 5.0
|
| 108 |
+
# Queried in parallel under the one budget above. OpenAlex adds the
|
| 109 |
+
# conference/journal papers with no arXiv preprint, plus DOI lookup; set
|
| 110 |
+
# OPENALEX_API_KEY for a 10x daily budget (keyless works, ~100 searches/day).
|
| 111 |
+
live_sources: [arxiv, openalex]
|
| 112 |
fetch_full_text: true # download the PDF for on-demand papers
|
| 113 |
# Cap the on-demand PDF cache. Citation lookup downloads one PDF per
|
| 114 |
# unresolved paper, which grows without bound on a shared deployment.
|
src/api/db.py
CHANGED
|
@@ -1,223 +1,303 @@
|
|
| 1 |
-
"""SQLite logging: queries, citations, clicks, votes and failures.
|
| 2 |
-
|
| 3 |
-
This is the instrumentation that makes a small deployment worth running. With
|
| 4 |
-
several people using it, the interesting question is not "did it work" but
|
| 5 |
-
"where is it failing, and on what kind of question" - which needs the failures
|
| 6 |
-
recorded as deliberately as the successes.
|
| 7 |
-
|
| 8 |
-
Four signals, in ascending order of how much they tell you:
|
| 9 |
-
|
| 10 |
-
clicks which citation someone opened. Weak, plentiful, unbiased.
|
| 11 |
-
votes an explicit thumbs up/down. Strong, scarce, self-selected.
|
| 12 |
-
low queries the retriever itself flagged as out-of-corpus. Free, and the
|
| 13 |
-
fastest route to "what do people ask that this cannot answer".
|
| 14 |
-
errors exceptions, with endpoint and query. Silent otherwise.
|
| 15 |
-
"""
|
| 16 |
-
|
| 17 |
-
from __future__ import annotations
|
| 18 |
-
|
| 19 |
-
import sqlite3
|
| 20 |
-
import traceback
|
| 21 |
-
from datetime import datetime, timezone
|
| 22 |
-
from pathlib import Path
|
| 23 |
-
from typing import Any
|
| 24 |
-
|
| 25 |
-
SCHEMA = """
|
| 26 |
-
CREATE TABLE IF NOT EXISTS queries (
|
| 27 |
-
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
| 28 |
-
ts TEXT NOT NULL,
|
| 29 |
-
query TEXT NOT NULL,
|
| 30 |
-
answer TEXT,
|
| 31 |
-
latency_ms REAL,
|
| 32 |
-
backend TEXT
|
| 33 |
-
);
|
| 34 |
-
|
| 35 |
-
CREATE TABLE IF NOT EXISTS citations (
|
| 36 |
-
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
| 37 |
-
query_id INTEGER NOT NULL REFERENCES queries(id),
|
| 38 |
-
doc_id TEXT NOT NULL,
|
| 39 |
-
rank INTEGER NOT NULL,
|
| 40 |
-
score REAL,
|
| 41 |
-
title TEXT
|
| 42 |
-
);
|
| 43 |
-
|
| 44 |
-
CREATE TABLE IF NOT EXISTS citation_clicks (
|
| 45 |
-
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
| 46 |
-
ts TEXT NOT NULL,
|
| 47 |
-
query_id INTEGER NOT NULL REFERENCES queries(id),
|
| 48 |
-
doc_id TEXT NOT NULL
|
| 49 |
-
);
|
| 50 |
-
|
| 51 |
-
CREATE TABLE IF NOT EXISTS votes (
|
| 52 |
-
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
| 53 |
-
ts TEXT NOT NULL,
|
| 54 |
-
query_id INTEGER NOT NULL REFERENCES queries(id),
|
| 55 |
-
helpful INTEGER NOT NULL,
|
| 56 |
-
comment TEXT
|
| 57 |
-
);
|
| 58 |
-
|
| 59 |
-
CREATE TABLE IF NOT EXISTS errors (
|
| 60 |
-
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
| 61 |
-
ts TEXT NOT NULL,
|
| 62 |
-
endpoint TEXT NOT NULL,
|
| 63 |
-
query TEXT,
|
| 64 |
-
error_type TEXT NOT NULL,
|
| 65 |
-
message TEXT,
|
| 66 |
-
stack TEXT
|
| 67 |
-
);
|
| 68 |
-
|
| 69 |
-
CREATE INDEX IF NOT EXISTS idx_citations_query ON citations(query_id);
|
| 70 |
-
CREATE INDEX IF NOT EXISTS idx_clicks_query ON citation_clicks(query_id);
|
| 71 |
-
CREATE INDEX IF NOT EXISTS idx_votes_query ON votes(query_id);
|
| 72 |
-
CREATE INDEX IF NOT EXISTS idx_errors_ts ON errors(ts);
|
| 73 |
-
"""
|
| 74 |
-
|
| 75 |
-
# Columns added after the first release. SQLite has no "ADD COLUMN IF NOT
|
| 76 |
-
# EXISTS", so they are applied one at a time and duplicates ignored.
|
| 77 |
-
MIGRATIONS = [
|
| 78 |
-
("queries", "mode", "TEXT"),
|
| 79 |
-
("queries", "confident", "INTEGER"),
|
| 80 |
-
("queries", "top_score", "REAL"),
|
| 81 |
-
("queries", "mean_top_score", "REAL"),
|
| 82 |
-
("queries", "n_results", "INTEGER"),
|
| 83 |
-
("queries", "generated", "INTEGER"),
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
|
| 96 |
-
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
|
| 100 |
-
|
| 101 |
-
|
| 102 |
-
|
| 103 |
-
|
| 104 |
-
|
| 105 |
-
|
| 106 |
-
|
| 107 |
-
|
| 108 |
-
|
| 109 |
-
|
| 110 |
-
|
| 111 |
-
|
| 112 |
-
|
| 113 |
-
|
| 114 |
-
|
| 115 |
-
|
| 116 |
-
|
| 117 |
-
|
| 118 |
-
|
| 119 |
-
|
| 120 |
-
|
| 121 |
-
|
| 122 |
-
|
| 123 |
-
|
| 124 |
-
|
| 125 |
-
|
| 126 |
-
|
| 127 |
-
|
| 128 |
-
|
| 129 |
-
|
| 130 |
-
|
| 131 |
-
|
| 132 |
-
|
| 133 |
-
|
| 134 |
-
|
| 135 |
-
|
| 136 |
-
|
| 137 |
-
|
| 138 |
-
|
| 139 |
-
|
| 140 |
-
"
|
| 141 |
-
|
| 142 |
-
|
| 143 |
-
|
| 144 |
-
|
| 145 |
-
|
| 146 |
-
|
| 147 |
-
|
| 148 |
-
|
| 149 |
-
|
| 150 |
-
|
| 151 |
-
|
| 152 |
-
|
| 153 |
-
)
|
| 154 |
-
|
| 155 |
-
|
| 156 |
-
|
| 157 |
-
|
| 158 |
-
|
| 159 |
-
|
| 160 |
-
|
| 161 |
-
)
|
| 162 |
-
|
| 163 |
-
|
| 164 |
-
|
| 165 |
-
|
| 166 |
-
|
| 167 |
-
|
| 168 |
-
|
| 169 |
-
|
| 170 |
-
|
| 171 |
-
|
| 172 |
-
|
| 173 |
-
|
| 174 |
-
|
| 175 |
-
|
| 176 |
-
|
| 177 |
-
|
| 178 |
-
|
| 179 |
-
def
|
| 180 |
-
|
| 181 |
-
|
| 182 |
-
|
| 183 |
-
|
| 184 |
-
|
| 185 |
-
"
|
| 186 |
-
"
|
| 187 |
-
|
| 188 |
-
|
| 189 |
-
|
| 190 |
-
|
| 191 |
-
|
| 192 |
-
|
| 193 |
-
|
| 194 |
-
|
| 195 |
-
|
| 196 |
-
with self._connect() as conn:
|
| 197 |
-
def
|
| 198 |
-
return
|
| 199 |
-
|
| 200 |
-
return {
|
| 201 |
-
"
|
| 202 |
-
|
| 203 |
-
|
| 204 |
-
),
|
| 205 |
-
"
|
| 206 |
-
|
| 207 |
-
|
| 208 |
-
|
| 209 |
-
|
| 210 |
-
|
| 211 |
-
|
| 212 |
-
|
| 213 |
-
|
| 214 |
-
|
| 215 |
-
|
| 216 |
-
|
| 217 |
-
|
| 218 |
-
|
| 219 |
-
|
| 220 |
-
|
| 221 |
-
|
| 222 |
-
|
| 223 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""SQLite logging: queries, citations, clicks, votes and failures.
|
| 2 |
+
|
| 3 |
+
This is the instrumentation that makes a small deployment worth running. With
|
| 4 |
+
several people using it, the interesting question is not "did it work" but
|
| 5 |
+
"where is it failing, and on what kind of question" - which needs the failures
|
| 6 |
+
recorded as deliberately as the successes.
|
| 7 |
+
|
| 8 |
+
Four signals, in ascending order of how much they tell you:
|
| 9 |
+
|
| 10 |
+
clicks which citation someone opened. Weak, plentiful, unbiased.
|
| 11 |
+
votes an explicit thumbs up/down. Strong, scarce, self-selected.
|
| 12 |
+
low queries the retriever itself flagged as out-of-corpus. Free, and the
|
| 13 |
+
fastest route to "what do people ask that this cannot answer".
|
| 14 |
+
errors exceptions, with endpoint and query. Silent otherwise.
|
| 15 |
+
"""
|
| 16 |
+
|
| 17 |
+
from __future__ import annotations
|
| 18 |
+
|
| 19 |
+
import sqlite3
|
| 20 |
+
import traceback
|
| 21 |
+
from datetime import datetime, timezone
|
| 22 |
+
from pathlib import Path
|
| 23 |
+
from typing import Any
|
| 24 |
+
|
| 25 |
+
SCHEMA = """
|
| 26 |
+
CREATE TABLE IF NOT EXISTS queries (
|
| 27 |
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
| 28 |
+
ts TEXT NOT NULL,
|
| 29 |
+
query TEXT NOT NULL,
|
| 30 |
+
answer TEXT,
|
| 31 |
+
latency_ms REAL,
|
| 32 |
+
backend TEXT
|
| 33 |
+
);
|
| 34 |
+
|
| 35 |
+
CREATE TABLE IF NOT EXISTS citations (
|
| 36 |
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
| 37 |
+
query_id INTEGER NOT NULL REFERENCES queries(id),
|
| 38 |
+
doc_id TEXT NOT NULL,
|
| 39 |
+
rank INTEGER NOT NULL,
|
| 40 |
+
score REAL,
|
| 41 |
+
title TEXT
|
| 42 |
+
);
|
| 43 |
+
|
| 44 |
+
CREATE TABLE IF NOT EXISTS citation_clicks (
|
| 45 |
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
| 46 |
+
ts TEXT NOT NULL,
|
| 47 |
+
query_id INTEGER NOT NULL REFERENCES queries(id),
|
| 48 |
+
doc_id TEXT NOT NULL
|
| 49 |
+
);
|
| 50 |
+
|
| 51 |
+
CREATE TABLE IF NOT EXISTS votes (
|
| 52 |
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
| 53 |
+
ts TEXT NOT NULL,
|
| 54 |
+
query_id INTEGER NOT NULL REFERENCES queries(id),
|
| 55 |
+
helpful INTEGER NOT NULL,
|
| 56 |
+
comment TEXT
|
| 57 |
+
);
|
| 58 |
+
|
| 59 |
+
CREATE TABLE IF NOT EXISTS errors (
|
| 60 |
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
| 61 |
+
ts TEXT NOT NULL,
|
| 62 |
+
endpoint TEXT NOT NULL,
|
| 63 |
+
query TEXT,
|
| 64 |
+
error_type TEXT NOT NULL,
|
| 65 |
+
message TEXT,
|
| 66 |
+
stack TEXT
|
| 67 |
+
);
|
| 68 |
+
|
| 69 |
+
CREATE INDEX IF NOT EXISTS idx_citations_query ON citations(query_id);
|
| 70 |
+
CREATE INDEX IF NOT EXISTS idx_clicks_query ON citation_clicks(query_id);
|
| 71 |
+
CREATE INDEX IF NOT EXISTS idx_votes_query ON votes(query_id);
|
| 72 |
+
CREATE INDEX IF NOT EXISTS idx_errors_ts ON errors(ts);
|
| 73 |
+
"""
|
| 74 |
+
|
| 75 |
+
# Columns added after the first release. SQLite has no "ADD COLUMN IF NOT
|
| 76 |
+
# EXISTS", so they are applied one at a time and duplicates ignored.
|
| 77 |
+
MIGRATIONS = [
|
| 78 |
+
("queries", "mode", "TEXT"),
|
| 79 |
+
("queries", "confident", "INTEGER"),
|
| 80 |
+
("queries", "top_score", "REAL"),
|
| 81 |
+
("queries", "mean_top_score", "REAL"),
|
| 82 |
+
("queries", "n_results", "INTEGER"),
|
| 83 |
+
("queries", "generated", "INTEGER"),
|
| 84 |
+
# v2: what the diagnosis said, and whether a query was a retry of one.
|
| 85 |
+
("queries", "source", "TEXT"),
|
| 86 |
+
("queries", "diagnosis_location", "TEXT"),
|
| 87 |
+
("queries", "suggested_query", "TEXT"),
|
| 88 |
+
("queries", "retry_of", "INTEGER"),
|
| 89 |
+
]
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
def _now() -> str:
|
| 93 |
+
return datetime.now(timezone.utc).isoformat(timespec="seconds")
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
class FeedbackLog:
|
| 97 |
+
"""Thin wrapper over a SQLite file. Safe to construct per request."""
|
| 98 |
+
|
| 99 |
+
def __init__(self, path: str = "results/feedback.db") -> None:
|
| 100 |
+
self.path = Path(path)
|
| 101 |
+
self.path.parent.mkdir(parents=True, exist_ok=True)
|
| 102 |
+
with self._connect() as conn:
|
| 103 |
+
conn.executescript(SCHEMA)
|
| 104 |
+
for table, column, coltype in MIGRATIONS:
|
| 105 |
+
try:
|
| 106 |
+
conn.execute(f"ALTER TABLE {table} ADD COLUMN {column} {coltype}")
|
| 107 |
+
except sqlite3.OperationalError:
|
| 108 |
+
pass # already present
|
| 109 |
+
|
| 110 |
+
def _connect(self) -> sqlite3.Connection:
|
| 111 |
+
conn = sqlite3.connect(self.path, timeout=10.0)
|
| 112 |
+
conn.row_factory = sqlite3.Row
|
| 113 |
+
return conn
|
| 114 |
+
|
| 115 |
+
# ------------------------------------------------------------- recording
|
| 116 |
+
|
| 117 |
+
def log_query(
|
| 118 |
+
self,
|
| 119 |
+
query: str,
|
| 120 |
+
answer: str | None,
|
| 121 |
+
latency_ms: float,
|
| 122 |
+
backend: str,
|
| 123 |
+
citations: list[dict[str, Any]],
|
| 124 |
+
mode: str = "ask",
|
| 125 |
+
confident: bool | None = None,
|
| 126 |
+
top_score: float | None = None,
|
| 127 |
+
mean_top_score: float | None = None,
|
| 128 |
+
generated: bool | None = None,
|
| 129 |
+
source: str | None = None,
|
| 130 |
+
diagnosis: dict[str, Any] | None = None,
|
| 131 |
+
retry_of: int | None = None,
|
| 132 |
+
) -> int:
|
| 133 |
+
"""Record a query, its answer, the passages shown, and any v2 diagnosis."""
|
| 134 |
+
with self._connect() as conn:
|
| 135 |
+
if retry_of is not None and conn.execute(
|
| 136 |
+
"SELECT 1 FROM queries WHERE id = ?", (retry_of,)).fetchone() is None:
|
| 137 |
+
retry_of = None # a stale or forged id must not corrupt the funnel
|
| 138 |
+
cursor = conn.execute(
|
| 139 |
+
"INSERT INTO queries (ts, query, answer, latency_ms, backend, mode, "
|
| 140 |
+
"confident, top_score, mean_top_score, n_results, generated, source, "
|
| 141 |
+
"diagnosis_location, suggested_query, retry_of) "
|
| 142 |
+
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
| 143 |
+
(
|
| 144 |
+
_now(), query, answer, latency_ms, backend, mode,
|
| 145 |
+
None if confident is None else int(confident),
|
| 146 |
+
top_score, mean_top_score, len(citations),
|
| 147 |
+
None if generated is None else int(generated),
|
| 148 |
+
source,
|
| 149 |
+
(diagnosis or {}).get("location"),
|
| 150 |
+
(diagnosis or {}).get("suggested_query"),
|
| 151 |
+
retry_of,
|
| 152 |
+
),
|
| 153 |
+
)
|
| 154 |
+
query_id = int(cursor.lastrowid)
|
| 155 |
+
conn.executemany(
|
| 156 |
+
"INSERT INTO citations (query_id, doc_id, rank, score, title) VALUES (?, ?, ?, ?, ?)",
|
| 157 |
+
[
|
| 158 |
+
(query_id, c.get("doc_id", ""), rank, c.get("score"), c.get("title"))
|
| 159 |
+
for rank, c in enumerate(citations, start=1)
|
| 160 |
+
],
|
| 161 |
+
)
|
| 162 |
+
return query_id
|
| 163 |
+
|
| 164 |
+
def log_click(self, query_id: int, doc_id: str) -> None:
|
| 165 |
+
with self._connect() as conn:
|
| 166 |
+
conn.execute(
|
| 167 |
+
"INSERT INTO citation_clicks (ts, query_id, doc_id) VALUES (?, ?, ?)",
|
| 168 |
+
(_now(), query_id, doc_id),
|
| 169 |
+
)
|
| 170 |
+
|
| 171 |
+
def log_vote(self, query_id: int, helpful: bool, comment: str | None = None) -> None:
|
| 172 |
+
"""Explicit thumbs up/down. The scarcest and most valuable signal."""
|
| 173 |
+
with self._connect() as conn:
|
| 174 |
+
conn.execute(
|
| 175 |
+
"INSERT INTO votes (ts, query_id, helpful, comment) VALUES (?, ?, ?, ?)",
|
| 176 |
+
(_now(), query_id, int(helpful), comment),
|
| 177 |
+
)
|
| 178 |
+
|
| 179 |
+
def log_error(
|
| 180 |
+
self, endpoint: str, error: BaseException, query: str | None = None
|
| 181 |
+
) -> None:
|
| 182 |
+
"""Record an exception. Without this, failures are invisible."""
|
| 183 |
+
with self._connect() as conn:
|
| 184 |
+
conn.execute(
|
| 185 |
+
"INSERT INTO errors (ts, endpoint, query, error_type, message, stack) "
|
| 186 |
+
"VALUES (?, ?, ?, ?, ?, ?)",
|
| 187 |
+
(
|
| 188 |
+
_now(), endpoint, query, type(error).__name__, str(error)[:2000],
|
| 189 |
+
"".join(traceback.format_exception(type(error), error, error.__traceback__))[-4000:],
|
| 190 |
+
),
|
| 191 |
+
)
|
| 192 |
+
|
| 193 |
+
# ------------------------------------------------------------- reporting
|
| 194 |
+
|
| 195 |
+
def stats(self) -> dict[str, int]:
|
| 196 |
+
with self._connect() as conn:
|
| 197 |
+
def count(sql: str) -> int:
|
| 198 |
+
return int(conn.execute(sql).fetchone()[0])
|
| 199 |
+
|
| 200 |
+
return {
|
| 201 |
+
"queries": count("SELECT COUNT(*) FROM queries"),
|
| 202 |
+
"citations": count("SELECT COUNT(*) FROM citations"),
|
| 203 |
+
"clicks": count("SELECT COUNT(*) FROM citation_clicks"),
|
| 204 |
+
"votes_up": count("SELECT COUNT(*) FROM votes WHERE helpful=1"),
|
| 205 |
+
"votes_down": count("SELECT COUNT(*) FROM votes WHERE helpful=0"),
|
| 206 |
+
"errors": count("SELECT COUNT(*) FROM errors"),
|
| 207 |
+
"low_confidence": count("SELECT COUNT(*) FROM queries WHERE confident=0"),
|
| 208 |
+
}
|
| 209 |
+
|
| 210 |
+
def v2_metrics(self) -> dict[str, Any]:
|
| 211 |
+
"""The live v2 funnel: diagnosis shown -> retry clicked -> result used.
|
| 212 |
+
|
| 213 |
+
Offline evaluation can only ask whether the target paper ranked; this
|
| 214 |
+
asks whether real users found the suggestion worth following, and
|
| 215 |
+
whether following it led to a paper they opened. The baseline is
|
| 216 |
+
gate-fired queries nobody retried: did those users open anything?
|
| 217 |
+
"""
|
| 218 |
+
with self._connect() as conn:
|
| 219 |
+
def one(sql: str) -> float:
|
| 220 |
+
return float(conn.execute(sql).fetchone()[0] or 0)
|
| 221 |
+
|
| 222 |
+
shown = one("SELECT COUNT(*) FROM queries WHERE diagnosis_location IS NOT NULL")
|
| 223 |
+
with_query = one("SELECT COUNT(*) FROM queries WHERE suggested_query IS NOT NULL")
|
| 224 |
+
retries = one("SELECT COUNT(*) FROM queries WHERE retry_of IS NOT NULL")
|
| 225 |
+
retry_clicked = one(
|
| 226 |
+
"SELECT COUNT(DISTINCT q.id) FROM queries q JOIN citation_clicks c "
|
| 227 |
+
"ON c.query_id = q.id WHERE q.retry_of IS NOT NULL")
|
| 228 |
+
fired_not_retried = one(
|
| 229 |
+
"SELECT COUNT(*) FROM queries q WHERE q.confident = 0 AND q.retry_of IS NULL "
|
| 230 |
+
"AND NOT EXISTS (SELECT 1 FROM queries r WHERE r.retry_of = q.id)")
|
| 231 |
+
fired_not_retried_clicked = one(
|
| 232 |
+
"SELECT COUNT(DISTINCT q.id) FROM queries q JOIN citation_clicks c "
|
| 233 |
+
"ON c.query_id = q.id WHERE q.confident = 0 AND q.retry_of IS NULL "
|
| 234 |
+
"AND NOT EXISTS (SELECT 1 FROM queries r WHERE r.retry_of = q.id)")
|
| 235 |
+
by_location = {
|
| 236 |
+
(r[0] or "none").split(":")[0]: r[1] for r in conn.execute(
|
| 237 |
+
"SELECT diagnosis_location, COUNT(*) FROM queries "
|
| 238 |
+
"WHERE diagnosis_location IS NOT NULL GROUP BY 1")
|
| 239 |
+
}
|
| 240 |
+
by_source = {r[0] or "corpus": r[1] for r in conn.execute(
|
| 241 |
+
"SELECT source, COUNT(*) FROM queries GROUP BY 1")}
|
| 242 |
+
|
| 243 |
+
def rate(a: float, b: float) -> float | None:
|
| 244 |
+
return round(a / b, 4) if b else None
|
| 245 |
+
|
| 246 |
+
return {
|
| 247 |
+
"diagnoses_shown": int(shown),
|
| 248 |
+
"diagnoses_with_query": int(with_query),
|
| 249 |
+
"retries": int(retries),
|
| 250 |
+
"retry_rate": rate(retries, with_query),
|
| 251 |
+
"retry_click_through": rate(retry_clicked, retries),
|
| 252 |
+
"baseline_click_through": rate(fired_not_retried_clicked, fired_not_retried),
|
| 253 |
+
"baseline_queries": int(fired_not_retried),
|
| 254 |
+
"diagnosis_locations": by_location,
|
| 255 |
+
"answer_sources": by_source,
|
| 256 |
+
}
|
| 257 |
+
|
| 258 |
+
def v2_labels(self, limit: int = 500) -> list[dict[str, Any]]:
|
| 259 |
+
"""Retried-then-clicked pairs: free supervision for a failure detector.
|
| 260 |
+
|
| 261 |
+
Each row says "the original search did not satisfy this user, the
|
| 262 |
+
suggested query did, and here is the paper they opened" - exactly the
|
| 263 |
+
signal the offline gate could not see.
|
| 264 |
+
"""
|
| 265 |
+
with self._connect() as conn:
|
| 266 |
+
return [dict(r) for r in conn.execute(
|
| 267 |
+
"SELECT o.id AS original_id, o.query AS original_query, "
|
| 268 |
+
"o.mean_top_score AS original_mean_score, o.diagnosis_location, "
|
| 269 |
+
"r.id AS retry_id, r.query AS retry_query, c.doc_id AS clicked_doc, "
|
| 270 |
+
"r.ts AS ts FROM queries r JOIN queries o ON o.id = r.retry_of "
|
| 271 |
+
"JOIN citation_clicks c ON c.query_id = r.id "
|
| 272 |
+
"ORDER BY r.id DESC LIMIT ?", (limit,)).fetchall()]
|
| 273 |
+
|
| 274 |
+
def recent_failures(self, limit: int = 50) -> dict[str, list[dict[str, Any]]]:
|
| 275 |
+
"""Everything worth reading when asking 'how is this failing?'."""
|
| 276 |
+
with self._connect() as conn:
|
| 277 |
+
def rows(sql: str, *args: Any) -> list[dict[str, Any]]:
|
| 278 |
+
return [dict(r) for r in conn.execute(sql, args).fetchall()]
|
| 279 |
+
|
| 280 |
+
return {
|
| 281 |
+
"errors": rows(
|
| 282 |
+
"SELECT ts, endpoint, query, error_type, message FROM errors "
|
| 283 |
+
"ORDER BY id DESC LIMIT ?", limit
|
| 284 |
+
),
|
| 285 |
+
"downvoted": rows(
|
| 286 |
+
"SELECT v.ts, q.query, q.mode, v.comment FROM votes v "
|
| 287 |
+
"JOIN queries q ON q.id = v.query_id WHERE v.helpful = 0 "
|
| 288 |
+
"ORDER BY v.id DESC LIMIT ?", limit
|
| 289 |
+
),
|
| 290 |
+
"out_of_corpus": rows(
|
| 291 |
+
"SELECT ts, query, mode, mean_top_score FROM queries "
|
| 292 |
+
"WHERE confident = 0 ORDER BY id DESC LIMIT ?", limit
|
| 293 |
+
),
|
| 294 |
+
"slowest": rows(
|
| 295 |
+
"SELECT ts, query, mode, latency_ms FROM queries "
|
| 296 |
+
"WHERE latency_ms IS NOT NULL ORDER BY latency_ms DESC LIMIT ?", limit
|
| 297 |
+
),
|
| 298 |
+
"never_clicked": rows(
|
| 299 |
+
"SELECT q.ts, q.query, q.mode FROM queries q "
|
| 300 |
+
"LEFT JOIN citation_clicks c ON c.query_id = q.id "
|
| 301 |
+
"WHERE c.id IS NULL AND q.confident = 1 ORDER BY q.id DESC LIMIT ?", limit
|
| 302 |
+
),
|
| 303 |
+
}
|
src/api/ml_app.py
CHANGED
|
@@ -31,6 +31,8 @@ from src.api.ml_pipeline import MLPipeline
|
|
| 31 |
from src.api.security import allowed_origins, api_token, guard, rate_limit_per_minute
|
| 32 |
from src.api.uploads import UploadError, UploadStore
|
| 33 |
from src.config import load_config
|
|
|
|
|
|
|
| 34 |
|
| 35 |
CONFIG_PATH = os.environ.get("RESEARCHOS_ML_CONFIG", "configs/ml.yaml")
|
| 36 |
|
|
@@ -80,11 +82,15 @@ class AskRequest(BaseModel):
|
|
| 80 |
query: str = Field(min_length=3, max_length=1000)
|
| 81 |
top_k: int = Field(default=6, ge=1, le=20)
|
| 82 |
generate: bool = True
|
|
|
|
|
|
|
|
|
|
| 83 |
|
| 84 |
|
| 85 |
class PapersRequest(BaseModel):
|
| 86 |
topic: str = Field(min_length=3, max_length=1000)
|
| 87 |
top_k: int = Field(default=10, ge=1, le=30)
|
|
|
|
| 88 |
|
| 89 |
|
| 90 |
class ExplainRequest(BaseModel):
|
|
@@ -93,6 +99,7 @@ class ExplainRequest(BaseModel):
|
|
| 93 |
# When set, the question is answered against this uploaded PDF instead of
|
| 94 |
# the corpus.
|
| 95 |
document_id: str | None = Field(default=None, max_length=64)
|
|
|
|
| 96 |
|
| 97 |
|
| 98 |
class ClickRequest(BaseModel):
|
|
@@ -134,6 +141,12 @@ def health() -> dict[str, Any]:
|
|
| 134 |
"generation_enabled": is_configured(),
|
| 135 |
"auth_required": api_token() is not None,
|
| 136 |
"rate_limit_per_minute": rate_limit_per_minute(),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 137 |
}
|
| 138 |
|
| 139 |
|
|
@@ -164,6 +177,9 @@ def papers(request: PapersRequest) -> dict[str, Any]:
|
|
| 164 |
top_score=result["top_score"],
|
| 165 |
mean_top_score=result["mean_top_score"],
|
| 166 |
generated=False,
|
|
|
|
|
|
|
|
|
|
| 167 |
)
|
| 168 |
return {"query_id": query_id, **result}
|
| 169 |
|
|
@@ -185,6 +201,9 @@ def ask(request: AskRequest) -> dict[str, Any]:
|
|
| 185 |
top_score=result["top_score"],
|
| 186 |
mean_top_score=result["mean_top_score"],
|
| 187 |
generated=bool(result["answer"]) and result["generation_configured"],
|
|
|
|
|
|
|
|
|
|
| 188 |
)
|
| 189 |
return {"query_id": query_id, **result}
|
| 190 |
|
|
@@ -237,6 +256,9 @@ def explain(request: ExplainRequest) -> dict[str, Any]:
|
|
| 237 |
mode="explain",
|
| 238 |
confident=result["resolved"],
|
| 239 |
generated=bool(result.get("explanation")) and result["generation_configured"],
|
|
|
|
|
|
|
|
|
|
| 240 |
)
|
| 241 |
return {"query_id": query_id, **result}
|
| 242 |
|
|
@@ -283,3 +305,16 @@ def failures(limit: int = 30) -> dict[str, Any]:
|
|
| 283 |
@app.get("/stats")
|
| 284 |
def stats() -> dict[str, int]:
|
| 285 |
return state["log"].stats()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 31 |
from src.api.security import allowed_origins, api_token, guard, rate_limit_per_minute
|
| 32 |
from src.api.uploads import UploadError, UploadStore
|
| 33 |
from src.config import load_config
|
| 34 |
+
from src.data.arxiv_source import breaker_open as arxiv_breaker_open
|
| 35 |
+
from src.data.openalex_source import status as openalex_status
|
| 36 |
|
| 37 |
CONFIG_PATH = os.environ.get("RESEARCHOS_ML_CONFIG", "configs/ml.yaml")
|
| 38 |
|
|
|
|
| 82 |
query: str = Field(min_length=3, max_length=1000)
|
| 83 |
top_k: int = Field(default=6, ge=1, le=20)
|
| 84 |
generate: bool = True
|
| 85 |
+
# Set when this query came from a v2 diagnosis's "Try: ..." button: the id
|
| 86 |
+
# of the query whose diagnosis suggested it. Drives the live v2 funnel.
|
| 87 |
+
retry_of: int | None = Field(default=None, ge=1)
|
| 88 |
|
| 89 |
|
| 90 |
class PapersRequest(BaseModel):
|
| 91 |
topic: str = Field(min_length=3, max_length=1000)
|
| 92 |
top_k: int = Field(default=10, ge=1, le=30)
|
| 93 |
+
retry_of: int | None = Field(default=None, ge=1)
|
| 94 |
|
| 95 |
|
| 96 |
class ExplainRequest(BaseModel):
|
|
|
|
| 99 |
# When set, the question is answered against this uploaded PDF instead of
|
| 100 |
# the corpus.
|
| 101 |
document_id: str | None = Field(default=None, max_length=64)
|
| 102 |
+
retry_of: int | None = Field(default=None, ge=1)
|
| 103 |
|
| 104 |
|
| 105 |
class ClickRequest(BaseModel):
|
|
|
|
| 141 |
"generation_enabled": is_configured(),
|
| 142 |
"auth_required": api_token() is not None,
|
| 143 |
"rate_limit_per_minute": rate_limit_per_minute(),
|
| 144 |
+
# Live sources are optional enrichments; this says which are usable
|
| 145 |
+
# right now and how much of OpenAlex's daily budget is left.
|
| 146 |
+
"live_sources": {
|
| 147 |
+
"arxiv": {"available": not arxiv_breaker_open()},
|
| 148 |
+
"openalex": openalex_status(),
|
| 149 |
+
},
|
| 150 |
}
|
| 151 |
|
| 152 |
|
|
|
|
| 177 |
top_score=result["top_score"],
|
| 178 |
mean_top_score=result["mean_top_score"],
|
| 179 |
generated=False,
|
| 180 |
+
source=result.get("source"),
|
| 181 |
+
diagnosis=result.get("diagnosis"),
|
| 182 |
+
retry_of=request.retry_of,
|
| 183 |
)
|
| 184 |
return {"query_id": query_id, **result}
|
| 185 |
|
|
|
|
| 201 |
top_score=result["top_score"],
|
| 202 |
mean_top_score=result["mean_top_score"],
|
| 203 |
generated=bool(result["answer"]) and result["generation_configured"],
|
| 204 |
+
source=result.get("source"),
|
| 205 |
+
diagnosis=result.get("diagnosis"),
|
| 206 |
+
retry_of=request.retry_of,
|
| 207 |
)
|
| 208 |
return {"query_id": query_id, **result}
|
| 209 |
|
|
|
|
| 256 |
mode="explain",
|
| 257 |
confident=result["resolved"],
|
| 258 |
generated=bool(result.get("explanation")) and result["generation_configured"],
|
| 259 |
+
source=result.get("source"),
|
| 260 |
+
diagnosis=result.get("diagnosis"),
|
| 261 |
+
retry_of=request.retry_of,
|
| 262 |
)
|
| 263 |
return {"query_id": query_id, **result}
|
| 264 |
|
|
|
|
| 305 |
@app.get("/stats")
|
| 306 |
def stats() -> dict[str, int]:
|
| 307 |
return state["log"].stats()
|
| 308 |
+
|
| 309 |
+
|
| 310 |
+
@app.get("/v2/metrics")
|
| 311 |
+
def v2_metrics() -> dict[str, Any]:
|
| 312 |
+
"""Live v2 funnel: diagnoses shown, retries taken, retries that led to a click."""
|
| 313 |
+
return state["log"].v2_metrics()
|
| 314 |
+
|
| 315 |
+
|
| 316 |
+
@app.get("/v2/labels")
|
| 317 |
+
def v2_labels(limit: int = 200) -> dict[str, Any]:
|
| 318 |
+
"""Retried-then-clicked pairs, exported as training data for a failure detector."""
|
| 319 |
+
rows = state["log"].v2_labels(limit=min(max(limit, 1), 1000))
|
| 320 |
+
return {"count": len(rows), "labels": rows}
|
src/api/ml_pipeline.py
CHANGED
|
@@ -38,6 +38,7 @@ from __future__ import annotations
|
|
| 38 |
import re
|
| 39 |
import threading
|
| 40 |
import time
|
|
|
|
| 41 |
from typing import Any
|
| 42 |
|
| 43 |
from src.api.diagnose import diagnose
|
|
@@ -46,8 +47,9 @@ from src.config import device
|
|
| 46 |
from src.data.arxiv_source import (
|
| 47 |
ArxivUnavailable, Paper, fetch_metadata_bounded, read_papers,
|
| 48 |
)
|
|
|
|
| 49 |
from src.data.chunking import chunk_paper, chunks_to_corpus, read_chunks
|
| 50 |
-
from src.data.citation import looks_like_a_question, resolve
|
| 51 |
from src.retrieval.bm25 import BM25Retriever
|
| 52 |
from src.retrieval.dense import DenseRetriever
|
| 53 |
from src.retrieval.diversify import cap_per_paper, distinct_papers
|
|
@@ -55,6 +57,18 @@ from src.retrieval.fusion import reciprocal_rank_fusion
|
|
| 55 |
|
| 56 |
PSEUDO_QID = "q"
|
| 57 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 58 |
_QUERY_STOPWORDS = {
|
| 59 |
"the", "and", "for", "with", "from", "what", "how", "why", "does", "explain",
|
| 60 |
"when", "are", "was", "that", "this", "there", "which", "about", "into",
|
|
@@ -249,7 +263,7 @@ class MLPipeline:
|
|
| 249 |
source = "corpus"
|
| 250 |
ac = self.cfg["answer"]
|
| 251 |
if not confident and self.cfg["citation"].get("allow_remote", True):
|
| 252 |
-
live = self.
|
| 253 |
if live:
|
| 254 |
live = self._rescore_live(topic, live)
|
| 255 |
floor = ac.get("live_min_score", 0.30)
|
|
@@ -261,11 +275,14 @@ class MLPipeline:
|
|
| 261 |
"paper_id": p["paper_id"],
|
| 262 |
"doc_id": p["doc_id"],
|
| 263 |
"title": p["title"],
|
| 264 |
-
"authors": [],
|
| 265 |
"abstract": p["text"],
|
| 266 |
"published": p["published"],
|
| 267 |
"primary_category": "",
|
| 268 |
"arxiv_url": p["arxiv_url"],
|
|
|
|
|
|
|
|
|
|
| 269 |
"score": round(p["score"], 4),
|
| 270 |
"matched_passages": 1,
|
| 271 |
"matched_sections": ["abstract"],
|
|
@@ -278,7 +295,10 @@ class MLPipeline:
|
|
| 278 |
if p["paper_id"] not in seen
|
| 279 |
]
|
| 280 |
if fetched:
|
| 281 |
-
papers
|
|
|
|
|
|
|
|
|
|
| 282 |
source = "arxiv_live"
|
| 283 |
confident = True
|
| 284 |
mean_top = sum(p["score"] for p in fetched) / len(fetched)
|
|
@@ -387,29 +407,69 @@ class MLPipeline:
|
|
| 387 |
},
|
| 388 |
}
|
| 389 |
|
| 390 |
-
# ------------------------------------------------------- live
|
| 391 |
|
| 392 |
-
def
|
| 393 |
-
"""Search arXiv
|
| 394 |
|
| 395 |
The corpus is a sample, not coverage: ~1,400 papers per category across
|
| 396 |
nine months, against the 50-150/day arXiv actually publishes. Most
|
| 397 |
papers a user asks about were never ingested, and telling them "not in
|
| 398 |
this corpus" is honest but useless when the paper plainly exists.
|
| 399 |
|
| 400 |
-
|
| 401 |
-
|
| 402 |
-
|
| 403 |
-
|
| 404 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 405 |
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 406 |
terms = [w for w in re.findall(r"[A-Za-z][A-Za-z0-9-]{2,}", question)
|
| 407 |
if w.lower() not in _QUERY_STOPWORDS][:12]
|
| 408 |
if not terms:
|
| 409 |
return []
|
| 410 |
-
|
| 411 |
-
budget = float(self.cfg["citation"].get("live_budget_s", 5.0))
|
| 412 |
-
deadline = time.monotonic() + budget
|
| 413 |
queries = [
|
| 414 |
"all:" + " AND ".join(f'"{t}"' for t in terms[:4]), # precise first
|
| 415 |
"all:" + " ".join(terms[:8]), # then looser
|
|
@@ -426,7 +486,6 @@ class MLPipeline:
|
|
| 426 |
return [] # a failed enrichment is not a failed request
|
| 427 |
if found:
|
| 428 |
break
|
| 429 |
-
|
| 430 |
return [
|
| 431 |
{
|
| 432 |
"chunk_id": f"live:{p.paper_id}",
|
|
@@ -438,10 +497,44 @@ class MLPipeline:
|
|
| 438 |
"section": "abstract",
|
| 439 |
"published": p.published,
|
| 440 |
"text": p.abstract,
|
|
|
|
|
|
|
|
|
|
|
|
|
| 441 |
}
|
| 442 |
for p in found
|
| 443 |
]
|
| 444 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 445 |
# ------------------------------------------------------------------- ask
|
| 446 |
|
| 447 |
def answer(self, question: str, generate: bool = True) -> dict[str, Any]:
|
|
@@ -478,7 +571,7 @@ class MLPipeline:
|
|
| 478 |
live_ms = 0.0
|
| 479 |
if remote_ok and (not confident or mean_top < enrich_below):
|
| 480 |
live_start = time.perf_counter()
|
| 481 |
-
live = self.
|
| 482 |
live_ms = (time.perf_counter() - live_start) * 1000.0
|
| 483 |
if live:
|
| 484 |
# arXiv always returns *something*, so live results must clear
|
|
@@ -588,7 +681,7 @@ class MLPipeline:
|
|
| 588 |
"chunk_id": c.chunk_id,
|
| 589 |
"doc_id": c.chunk_id,
|
| 590 |
"paper_id": c.paper_id,
|
| 591 |
-
"arxiv_url":
|
| 592 |
"score": 1.0,
|
| 593 |
"title": c.title,
|
| 594 |
"section": c.section,
|
|
@@ -700,7 +793,9 @@ class MLPipeline:
|
|
| 700 |
"authors": paper.authors[:8],
|
| 701 |
"published": paper.published,
|
| 702 |
"primary_category": paper.primary_category,
|
| 703 |
-
"arxiv_url":
|
|
|
|
|
|
|
| 704 |
"has_full_text": bool(paper.full_text),
|
| 705 |
},
|
| 706 |
"explanation": explanation,
|
|
|
|
| 38 |
import re
|
| 39 |
import threading
|
| 40 |
import time
|
| 41 |
+
from concurrent.futures import ThreadPoolExecutor
|
| 42 |
from typing import Any
|
| 43 |
|
| 44 |
from src.api.diagnose import diagnose
|
|
|
|
| 47 |
from src.data.arxiv_source import (
|
| 48 |
ArxivUnavailable, Paper, fetch_metadata_bounded, read_papers,
|
| 49 |
)
|
| 50 |
+
from src.data import openalex_source as openalex
|
| 51 |
from src.data.chunking import chunk_paper, chunks_to_corpus, read_chunks
|
| 52 |
+
from src.data.citation import looks_like_a_question, paper_url, resolve
|
| 53 |
from src.retrieval.bm25 import BM25Retriever
|
| 54 |
from src.retrieval.dense import DenseRetriever
|
| 55 |
from src.retrieval.diversify import cap_per_paper, distinct_papers
|
|
|
|
| 57 |
|
| 58 |
PSEUDO_QID = "q"
|
| 59 |
|
| 60 |
+
# Live sources run side by side under one deadline; their own clients bound
|
| 61 |
+
# each call, so this pool only has to hold the two waits of one request.
|
| 62 |
+
_LIVE_POOL = ThreadPoolExecutor(max_workers=8, thread_name_prefix="live-search")
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def _dedup_key(item: dict[str, Any]) -> str:
|
| 66 |
+
"""arXiv id when known, else a normalised title: one paper, one entry."""
|
| 67 |
+
pid = item.get("paper_id") or ""
|
| 68 |
+
if pid and not pid.startswith("openalex:"):
|
| 69 |
+
return pid
|
| 70 |
+
return re.sub(r"[^a-z0-9]", "", (item.get("title") or "").lower())[:120]
|
| 71 |
+
|
| 72 |
_QUERY_STOPWORDS = {
|
| 73 |
"the", "and", "for", "with", "from", "what", "how", "why", "does", "explain",
|
| 74 |
"when", "are", "was", "that", "this", "there", "which", "about", "into",
|
|
|
|
| 263 |
source = "corpus"
|
| 264 |
ac = self.cfg["answer"]
|
| 265 |
if not confident and self.cfg["citation"].get("allow_remote", True):
|
| 266 |
+
live = self.search_live(topic, top_k=top_k)
|
| 267 |
if live:
|
| 268 |
live = self._rescore_live(topic, live)
|
| 269 |
floor = ac.get("live_min_score", 0.30)
|
|
|
|
| 275 |
"paper_id": p["paper_id"],
|
| 276 |
"doc_id": p["doc_id"],
|
| 277 |
"title": p["title"],
|
| 278 |
+
"authors": p.get("authors", []),
|
| 279 |
"abstract": p["text"],
|
| 280 |
"published": p["published"],
|
| 281 |
"primary_category": "",
|
| 282 |
"arxiv_url": p["arxiv_url"],
|
| 283 |
+
"venue": p.get("venue", ""),
|
| 284 |
+
"cited_by_count": p.get("cited_by_count"),
|
| 285 |
+
"live_source": p.get("live_source", "arxiv"),
|
| 286 |
"score": round(p["score"], 4),
|
| 287 |
"matched_passages": 1,
|
| 288 |
"matched_sections": ["abstract"],
|
|
|
|
| 295 |
if p["paper_id"] not in seen
|
| 296 |
]
|
| 297 |
if fetched:
|
| 298 |
+
# Corpus papers only fill remaining slots if they clear
|
| 299 |
+
# the same bar - otherwise a 0.08 match sat under a
|
| 300 |
+
# result labelled confident.
|
| 301 |
+
papers = (fetched + [p for p in papers if p["score"] >= floor])[:top_k]
|
| 302 |
source = "arxiv_live"
|
| 303 |
confident = True
|
| 304 |
mean_top = sum(p["score"] for p in fetched) / len(fetched)
|
|
|
|
| 407 |
},
|
| 408 |
}
|
| 409 |
|
| 410 |
+
# ------------------------------------------------------------ live search
|
| 411 |
|
| 412 |
+
def search_live(self, question: str, top_k: int = 6) -> list[dict[str, Any]]:
|
| 413 |
+
"""Search arXiv and OpenAlex live, for questions the corpus cannot answer.
|
| 414 |
|
| 415 |
The corpus is a sample, not coverage: ~1,400 papers per category across
|
| 416 |
nine months, against the 50-150/day arXiv actually publishes. Most
|
| 417 |
papers a user asks about were never ingested, and telling them "not in
|
| 418 |
this corpus" is honest but useless when the paper plainly exists.
|
| 419 |
|
| 420 |
+
Two sources, because each misses what the other has. arXiv is the
|
| 421 |
+
freshest for preprints; OpenAlex adds conference and journal papers
|
| 422 |
+
that never had one. Both run in parallel under ONE wall-clock budget
|
| 423 |
+
(`citation.live_budget_s`), so adding a source never adds latency:
|
| 424 |
+
whatever has answered when the budget expires is used, and a source
|
| 425 |
+
that is down or paused costs nothing. The same paper found by both
|
| 426 |
+
collapses to one entry, keyed on its arXiv id or normalised title.
|
| 427 |
+
|
| 428 |
+
Abstracts only - no PDF fetch. Results are rescored by the corpus
|
| 429 |
+
reranker afterwards, so one confidence bar governs every source.
|
| 430 |
"""
|
| 431 |
+
sources = self.cfg["citation"].get("live_sources", ["arxiv", "openalex"])
|
| 432 |
+
budget = float(self.cfg["citation"].get("live_budget_s", 5.0))
|
| 433 |
+
deadline = time.monotonic() + budget
|
| 434 |
+
jobs = {}
|
| 435 |
+
if "arxiv" in sources:
|
| 436 |
+
jobs["arxiv"] = _LIVE_POOL.submit(self._live_arxiv, question, top_k, deadline)
|
| 437 |
+
if "openalex" in sources:
|
| 438 |
+
jobs["openalex"] = _LIVE_POOL.submit(self._live_openalex, question, top_k, deadline)
|
| 439 |
+
|
| 440 |
+
found: dict[str, list[dict[str, Any]]] = {}
|
| 441 |
+
for name, job in jobs.items():
|
| 442 |
+
try:
|
| 443 |
+
found[name] = job.result(timeout=max(deadline - time.monotonic(), 0.05))
|
| 444 |
+
except Exception: # noqa: BLE001 - a failed source is an empty source
|
| 445 |
+
found[name] = []
|
| 446 |
+
|
| 447 |
+
merged: list[dict[str, Any]] = []
|
| 448 |
+
seen: dict[str, dict[str, Any]] = {}
|
| 449 |
+
# arXiv first: on a duplicate, its record wins (fresher version), and
|
| 450 |
+
# OpenAlex contributes the venue and citation count it lacks.
|
| 451 |
+
for name in ("arxiv", "openalex"):
|
| 452 |
+
for item in found.get(name, []):
|
| 453 |
+
key = _dedup_key(item)
|
| 454 |
+
if key in seen:
|
| 455 |
+
kept = seen[key]
|
| 456 |
+
kept["venue"] = kept.get("venue") or item.get("venue", "")
|
| 457 |
+
kept["cited_by_count"] = max(kept.get("cited_by_count") or 0,
|
| 458 |
+
item.get("cited_by_count") or 0)
|
| 459 |
+
kept["live_source"] = "arxiv+openalex"
|
| 460 |
+
continue
|
| 461 |
+
seen[key] = item
|
| 462 |
+
merged.append(item)
|
| 463 |
+
return merged
|
| 464 |
+
|
| 465 |
+
# Kept for callers that predate OpenAlex.
|
| 466 |
+
search_arxiv_live = search_live
|
| 467 |
+
|
| 468 |
+
def _live_arxiv(self, question: str, top_k: int, deadline: float) -> list[dict[str, Any]]:
|
| 469 |
terms = [w for w in re.findall(r"[A-Za-z][A-Za-z0-9-]{2,}", question)
|
| 470 |
if w.lower() not in _QUERY_STOPWORDS][:12]
|
| 471 |
if not terms:
|
| 472 |
return []
|
|
|
|
|
|
|
|
|
|
| 473 |
queries = [
|
| 474 |
"all:" + " AND ".join(f'"{t}"' for t in terms[:4]), # precise first
|
| 475 |
"all:" + " ".join(terms[:8]), # then looser
|
|
|
|
| 486 |
return [] # a failed enrichment is not a failed request
|
| 487 |
if found:
|
| 488 |
break
|
|
|
|
| 489 |
return [
|
| 490 |
{
|
| 491 |
"chunk_id": f"live:{p.paper_id}",
|
|
|
|
| 497 |
"section": "abstract",
|
| 498 |
"published": p.published,
|
| 499 |
"text": p.abstract,
|
| 500 |
+
"authors": p.authors[:8],
|
| 501 |
+
"venue": "arXiv",
|
| 502 |
+
"cited_by_count": None,
|
| 503 |
+
"live_source": "arxiv",
|
| 504 |
}
|
| 505 |
for p in found
|
| 506 |
]
|
| 507 |
|
| 508 |
+
def _live_openalex(self, question: str, top_k: int, deadline: float) -> list[dict[str, Any]]:
|
| 509 |
+
remaining = deadline - time.monotonic()
|
| 510 |
+
if remaining <= 0.25:
|
| 511 |
+
return []
|
| 512 |
+
try:
|
| 513 |
+
works = openalex.search_works(question, max_results=top_k, budget_s=remaining)
|
| 514 |
+
except openalex.OpenAlexUnavailable:
|
| 515 |
+
return []
|
| 516 |
+
return [
|
| 517 |
+
{
|
| 518 |
+
"chunk_id": f"live:{w['paper_id']}",
|
| 519 |
+
"doc_id": f"live:{w['paper_id']}",
|
| 520 |
+
"paper_id": w["paper_id"],
|
| 521 |
+
# Named arxiv_url for compatibility with every client; for a
|
| 522 |
+
# paper with no arXiv copy it is the DOI or landing page.
|
| 523 |
+
"arxiv_url": w["url"],
|
| 524 |
+
"score": 1.0,
|
| 525 |
+
"title": w["title"],
|
| 526 |
+
"section": "abstract",
|
| 527 |
+
"published": w["published"],
|
| 528 |
+
"text": w["abstract"],
|
| 529 |
+
"authors": w["authors"][:8],
|
| 530 |
+
"venue": w["venue"],
|
| 531 |
+
"cited_by_count": w["cited_by_count"],
|
| 532 |
+
"doi": w["doi"],
|
| 533 |
+
"live_source": "openalex",
|
| 534 |
+
}
|
| 535 |
+
for w in works
|
| 536 |
+
]
|
| 537 |
+
|
| 538 |
# ------------------------------------------------------------------- ask
|
| 539 |
|
| 540 |
def answer(self, question: str, generate: bool = True) -> dict[str, Any]:
|
|
|
|
| 571 |
live_ms = 0.0
|
| 572 |
if remote_ok and (not confident or mean_top < enrich_below):
|
| 573 |
live_start = time.perf_counter()
|
| 574 |
+
live = self.search_live(question)
|
| 575 |
live_ms = (time.perf_counter() - live_start) * 1000.0
|
| 576 |
if live:
|
| 577 |
# arXiv always returns *something*, so live results must clear
|
|
|
|
| 681 |
"chunk_id": c.chunk_id,
|
| 682 |
"doc_id": c.chunk_id,
|
| 683 |
"paper_id": c.paper_id,
|
| 684 |
+
"arxiv_url": paper_url(paper),
|
| 685 |
"score": 1.0,
|
| 686 |
"title": c.title,
|
| 687 |
"section": c.section,
|
|
|
|
| 793 |
"authors": paper.authors[:8],
|
| 794 |
"published": paper.published,
|
| 795 |
"primary_category": paper.primary_category,
|
| 796 |
+
"arxiv_url": paper_url(paper),
|
| 797 |
+
"venue": (paper.metadata or {}).get("venue", ""),
|
| 798 |
+
"doi": (paper.metadata or {}).get("doi", ""),
|
| 799 |
"has_full_text": bool(paper.full_text),
|
| 800 |
},
|
| 801 |
"explanation": explanation,
|
src/data/citation.py
CHANGED
|
@@ -6,13 +6,14 @@ The use case: you are reading a paper, you hit "...following [23] (Assran et al.
|
|
| 6 |
Accepted inputs, in the order they are tried:
|
| 7 |
|
| 8 |
arXiv id 2301.08243, arXiv:2301.08243v2, https://arxiv.org/abs/2301.08243
|
|
|
|
| 9 |
bracketed ref [23] M. Assran et al. "Self-Supervised Learning from Images..."
|
| 10 |
bare title Self-Supervised Learning from Images with a Joint-Embedding...
|
| 11 |
|
| 12 |
-
Resolution is local-first: the ingested corpus is checked before
|
| 13 |
-
|
| 14 |
-
not found locally is fetched on demand
|
| 15 |
-
|
| 16 |
"""
|
| 17 |
|
| 18 |
from __future__ import annotations
|
|
@@ -21,6 +22,7 @@ import re
|
|
| 21 |
from dataclasses import dataclass
|
| 22 |
from typing import Any
|
| 23 |
|
|
|
|
| 24 |
from src.data.arxiv_source import (
|
| 25 |
ArxivUnavailable, Paper, download_pdf, extract_text, fetch_metadata_bounded,
|
| 26 |
)
|
|
@@ -57,12 +59,52 @@ def _safe_fetch(query: str, max_results: int) -> list[Paper]:
|
|
| 57 |
return []
|
| 58 |
|
| 59 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 60 |
@dataclass
|
| 61 |
class Resolution:
|
| 62 |
"""What a citation string resolved to, and how."""
|
| 63 |
|
| 64 |
paper: Paper | None
|
| 65 |
-
method: str # arxiv_id | local_title | arxiv_search | unresolved
|
| 66 |
query_used: str
|
| 67 |
from_corpus: bool
|
| 68 |
candidates: list[dict[str, str]]
|
|
@@ -77,13 +119,14 @@ QUESTION_STARTS = (
|
|
| 77 |
# What a citation actually looks like. Anything matching one of these is worth
|
| 78 |
# the cost of resolution; nothing else is.
|
| 79 |
_CITATION_SHAPES = (
|
| 80 |
-
re.compile(r"
|
| 81 |
re.compile(r"^\s*[\[\(]\d{1,3}[\]\)]"), # [23] or (23)
|
| 82 |
re.compile(r"^\s*\d{1,3}\.\s+[A-Z]"), # 23. Author
|
| 83 |
re.compile(r"\((?:19|20)\d{2}\)"), # (2023)
|
| 84 |
-
re.compile(r",\s*(?:19|20)\d{2}
|
| 85 |
-
re.compile(r"
|
| 86 |
-
re.compile(r"
|
|
|
|
| 87 |
re.compile(r"[\"“‘']([^\"”’']{15,})[\"”’']"), # a quoted title
|
| 88 |
)
|
| 89 |
|
|
@@ -244,8 +287,13 @@ def resolve(
|
|
| 244 |
if not raw:
|
| 245 |
return Resolution(None, "unresolved", "", False, [])
|
| 246 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 247 |
# --- 1. explicit arXiv id ---
|
| 248 |
-
arxiv_id = extract_arxiv_id(raw)
|
| 249 |
if arxiv_id:
|
| 250 |
local = papers_by_id.get(arxiv_id)
|
| 251 |
if local:
|
|
@@ -258,6 +306,17 @@ def resolve(
|
|
| 258 |
_attach_text(paper, pdf_dir, text_dir, max_cache_gb)
|
| 259 |
return Resolution(paper, "arxiv_id", arxiv_id, False, [])
|
| 260 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 261 |
# --- 2. title match against the local corpus ---
|
| 262 |
title_query = clean_citation(raw)
|
| 263 |
local_match, candidates = find_in_corpus(title_query, papers_by_id)
|
|
@@ -281,6 +340,23 @@ def resolve(
|
|
| 281 |
for p in found
|
| 282 |
] + candidates
|
| 283 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 284 |
return Resolution(None, "unresolved", title_query, False, candidates[:5])
|
| 285 |
|
| 286 |
|
|
|
|
| 6 |
Accepted inputs, in the order they are tried:
|
| 7 |
|
| 8 |
arXiv id 2301.08243, arXiv:2301.08243v2, https://arxiv.org/abs/2301.08243
|
| 9 |
+
DOI 10.1109/CVPR.2016.90, https://doi.org/10.1109/CVPR.2016.90
|
| 10 |
bracketed ref [23] M. Assran et al. "Self-Supervised Learning from Images..."
|
| 11 |
bare title Self-Supervised Learning from Images with a Joint-Embedding...
|
| 12 |
|
| 13 |
+
Resolution is local-first: the ingested corpus is checked before any network
|
| 14 |
+
call, because most of what you are reading about is already indexed. Anything
|
| 15 |
+
not found locally is fetched on demand - arXiv first, then OpenAlex, which
|
| 16 |
+
covers the conference and journal papers that never had an arXiv preprint.
|
| 17 |
"""
|
| 18 |
|
| 19 |
from __future__ import annotations
|
|
|
|
| 22 |
from dataclasses import dataclass
|
| 23 |
from typing import Any
|
| 24 |
|
| 25 |
+
from src.data import openalex_source as openalex
|
| 26 |
from src.data.arxiv_source import (
|
| 27 |
ArxivUnavailable, Paper, download_pdf, extract_text, fetch_metadata_bounded,
|
| 28 |
)
|
|
|
|
| 59 |
return []
|
| 60 |
|
| 61 |
|
| 62 |
+
def _paper_from_openalex(work: dict[str, Any]) -> Paper:
|
| 63 |
+
"""An OpenAlex work in the Paper shape the rest of the pipeline expects."""
|
| 64 |
+
arxiv_id = work.get("arxiv_id")
|
| 65 |
+
return Paper(
|
| 66 |
+
paper_id=work["paper_id"],
|
| 67 |
+
version=1,
|
| 68 |
+
title=work["title"],
|
| 69 |
+
abstract=work["abstract"],
|
| 70 |
+
authors=work["authors"],
|
| 71 |
+
categories=[],
|
| 72 |
+
primary_category=work.get("venue", ""),
|
| 73 |
+
published=work.get("published", ""),
|
| 74 |
+
updated="",
|
| 75 |
+
# Only arXiv PDFs are ever fetched on the request path: a known host
|
| 76 |
+
# with stable, polite access. Other open-access PDFs are linked, not
|
| 77 |
+
# downloaded.
|
| 78 |
+
pdf_url=f"https://arxiv.org/pdf/{arxiv_id}" if arxiv_id else "",
|
| 79 |
+
metadata={
|
| 80 |
+
"source": "openalex",
|
| 81 |
+
"url": work.get("url", ""),
|
| 82 |
+
"venue": work.get("venue", ""),
|
| 83 |
+
"doi": work.get("doi") or "",
|
| 84 |
+
"cited_by_count": str(work.get("cited_by_count", 0)),
|
| 85 |
+
},
|
| 86 |
+
)
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
def _safe_openalex(fn, *args) -> Any:
|
| 90 |
+
try:
|
| 91 |
+
return fn(*args)
|
| 92 |
+
except openalex.OpenAlexUnavailable as exc:
|
| 93 |
+
print(f"[openalex] lookup skipped ({exc}); degrading")
|
| 94 |
+
return None
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
def paper_url(paper: Paper) -> str:
|
| 98 |
+
"""Where a reader should go for this paper, whichever source it came from."""
|
| 99 |
+
return (paper.metadata or {}).get("url") or f"https://arxiv.org/abs/{paper.paper_id}"
|
| 100 |
+
|
| 101 |
+
|
| 102 |
@dataclass
|
| 103 |
class Resolution:
|
| 104 |
"""What a citation string resolved to, and how."""
|
| 105 |
|
| 106 |
paper: Paper | None
|
| 107 |
+
method: str # arxiv_id | doi | local_title | arxiv_search | openalex_title | unresolved
|
| 108 |
query_used: str
|
| 109 |
from_corpus: bool
|
| 110 |
candidates: list[dict[str, str]]
|
|
|
|
| 119 |
# What a citation actually looks like. Anything matching one of these is worth
|
| 120 |
# the cost of resolution; nothing else is.
|
| 121 |
_CITATION_SHAPES = (
|
| 122 |
+
re.compile(r"\bet\s+al\b", re.IGNORECASE), # Assran et al.
|
| 123 |
re.compile(r"^\s*[\[\(]\d{1,3}[\]\)]"), # [23] or (23)
|
| 124 |
re.compile(r"^\s*\d{1,3}\.\s+[A-Z]"), # 23. Author
|
| 125 |
re.compile(r"\((?:19|20)\d{2}\)"), # (2023)
|
| 126 |
+
re.compile(r",\s*(?:19|20)\d{2}\b"), # , 2023
|
| 127 |
+
re.compile(r"\b(?:CVPR|NeurIPS|ICML|ICLR|ACL|EMNLP|AAAI|SIGGRAPH)\b"),
|
| 128 |
+
re.compile(r"\barxiv\b", re.IGNORECASE),
|
| 129 |
+
re.compile(r"\b10\.\d{4,9}/\S+"), # a DOI
|
| 130 |
re.compile(r"[\"“‘']([^\"”’']{15,})[\"”’']"), # a quoted title
|
| 131 |
)
|
| 132 |
|
|
|
|
| 287 |
if not raw:
|
| 288 |
return Resolution(None, "unresolved", "", False, [])
|
| 289 |
|
| 290 |
+
# A DOI's digits can look like an arXiv id ("10.1145/3292500.3330701"
|
| 291 |
+
# contains "2500.33307"), so a non-arXiv DOI takes precedence over it.
|
| 292 |
+
doi = openalex.extract_doi(raw)
|
| 293 |
+
arxiv_doi = bool(doi and doi.startswith("10.48550/arxiv."))
|
| 294 |
+
|
| 295 |
# --- 1. explicit arXiv id ---
|
| 296 |
+
arxiv_id = extract_arxiv_id(raw) if (not doi or arxiv_doi) else None
|
| 297 |
if arxiv_id:
|
| 298 |
local = papers_by_id.get(arxiv_id)
|
| 299 |
if local:
|
|
|
|
| 306 |
_attach_text(paper, pdf_dir, text_dir, max_cache_gb)
|
| 307 |
return Resolution(paper, "arxiv_id", arxiv_id, False, [])
|
| 308 |
|
| 309 |
+
# --- 1b. DOI: conference and journal papers arXiv does not know ---
|
| 310 |
+
if doi and not arxiv_doi and allow_remote:
|
| 311 |
+
work = _safe_openalex(openalex.get_by_doi, doi)
|
| 312 |
+
if work and work.get("title"):
|
| 313 |
+
if work.get("arxiv_id") and work["arxiv_id"] in papers_by_id:
|
| 314 |
+
return Resolution(Paper(**papers_by_id[work["arxiv_id"]]), "doi", doi, True, [])
|
| 315 |
+
paper = _paper_from_openalex(work)
|
| 316 |
+
if fetch_full_text and paper.pdf_url:
|
| 317 |
+
_attach_text(paper, pdf_dir, text_dir, max_cache_gb)
|
| 318 |
+
return Resolution(paper, "doi", doi, False, [])
|
| 319 |
+
|
| 320 |
# --- 2. title match against the local corpus ---
|
| 321 |
title_query = clean_citation(raw)
|
| 322 |
local_match, candidates = find_in_corpus(title_query, papers_by_id)
|
|
|
|
| 340 |
for p in found
|
| 341 |
] + candidates
|
| 342 |
|
| 343 |
+
# --- 4. ask OpenAlex: the paper may simply never have been on arXiv ---
|
| 344 |
+
if allow_remote and title_query:
|
| 345 |
+
works = _safe_openalex(openalex.search_title, title_query, 8) or []
|
| 346 |
+
if works:
|
| 347 |
+
best = max(works, key=lambda w: _token_overlap(title_query, w["title"]))
|
| 348 |
+
overlap = _token_overlap(title_query, best["title"])
|
| 349 |
+
if overlap >= 0.6 and best.get("abstract"):
|
| 350 |
+
paper = _paper_from_openalex(best)
|
| 351 |
+
if fetch_full_text and paper.pdf_url:
|
| 352 |
+
_attach_text(paper, pdf_dir, text_dir, max_cache_gb)
|
| 353 |
+
return Resolution(paper, "openalex_title", title_query, False, candidates)
|
| 354 |
+
candidates = candidates + [
|
| 355 |
+
{"paper_id": w["paper_id"], "title": w["title"],
|
| 356 |
+
"score": f"{_token_overlap(title_query, w['title']):.2f}"}
|
| 357 |
+
for w in works[:3]
|
| 358 |
+
]
|
| 359 |
+
|
| 360 |
return Resolution(None, "unresolved", title_query, False, candidates[:5])
|
| 361 |
|
| 362 |
|
src/data/openalex_source.py
ADDED
|
@@ -0,0 +1,307 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""OpenAlex: a second live source, for papers arXiv does not have.
|
| 2 |
+
|
| 3 |
+
arXiv covers preprints. A large share of what researchers cite never went
|
| 4 |
+
there: conference papers with no preprint, journal articles, DOI-only
|
| 5 |
+
references. OpenAlex indexes ~250M scholarly works under CC0, so it fills
|
| 6 |
+
both gaps this project had: live search beyond arXiv, and citation lookup
|
| 7 |
+
by DOI or by a title arXiv cannot find.
|
| 8 |
+
|
| 9 |
+
It is wrapped in the same rails as the arXiv client, because it sits on the
|
| 10 |
+
request path and is always an enhancement, never the only way to answer:
|
| 11 |
+
|
| 12 |
+
* a hard wall-clock budget per call - a slow API costs the user nothing
|
| 13 |
+
beyond that budget
|
| 14 |
+
* a circuit breaker - after a failure, stop asking for a while instead of
|
| 15 |
+
paying the timeout on every request
|
| 16 |
+
* an LRU cache - the same paper gets looked up repeatedly
|
| 17 |
+
* budget awareness - OpenAlex meters usage in dollars per day (keyless:
|
| 18 |
+
$0.10/day shared by everyone behind the same IP, about 100 searches; a
|
| 19 |
+
free key: $1/day). Every response reports what is left, so the client
|
| 20 |
+
stops *before* the budget is gone, leaving headroom, instead of
|
| 21 |
+
discovering the limit as a stream of 429s.
|
| 22 |
+
|
| 23 |
+
Set OPENALEX_API_KEY to use a free key (https://openalex.org/settings/api).
|
| 24 |
+
Without one everything works, within the keyless budget.
|
| 25 |
+
"""
|
| 26 |
+
|
| 27 |
+
from __future__ import annotations
|
| 28 |
+
|
| 29 |
+
import os
|
| 30 |
+
import re
|
| 31 |
+
import threading
|
| 32 |
+
import time
|
| 33 |
+
from collections import OrderedDict
|
| 34 |
+
from concurrent.futures import ThreadPoolExecutor
|
| 35 |
+
from concurrent.futures import TimeoutError as FuturesTimeout
|
| 36 |
+
from typing import Any
|
| 37 |
+
|
| 38 |
+
import httpx
|
| 39 |
+
|
| 40 |
+
BASE_URL = "https://api.openalex.org"
|
| 41 |
+
|
| 42 |
+
# OpenAlex's own topic taxonomy: field 17 is Computer Science. Without this,
|
| 43 |
+
# an off-topic question ("sourdough fermentation") is answered from food
|
| 44 |
+
# science, and the app's honest "this covers AI/ML research only" decline
|
| 45 |
+
# turns into a confident answer from the wrong literature.
|
| 46 |
+
#
|
| 47 |
+
# Restricted to papers, in English: unfiltered, a PageRank query ranked a
|
| 48 |
+
# Spanish thesis and a university course page above the PageRank paper itself.
|
| 49 |
+
# OpenAlex types conference papers as "article", so venues are kept.
|
| 50 |
+
DEFAULT_FILTER = ("primary_topic.field.id:fields/17,has_abstract:true,"
|
| 51 |
+
"type:article|preprint|review,language:en")
|
| 52 |
+
|
| 53 |
+
SELECT = ",".join((
|
| 54 |
+
"id", "doi", "title", "publication_year", "publication_date", "type",
|
| 55 |
+
"abstract_inverted_index", "authorships", "primary_location", "locations",
|
| 56 |
+
"cited_by_count", "open_access",
|
| 57 |
+
))
|
| 58 |
+
|
| 59 |
+
REQUEST_BUDGET_S = 5.0
|
| 60 |
+
BREAKER_COOLDOWN_S = 120.0
|
| 61 |
+
# Stop with this much of the daily budget left. Keyless usage is shared with
|
| 62 |
+
# everyone on the host's IP, so running it to zero would also starve them.
|
| 63 |
+
BUDGET_RESERVE_USD = 0.01
|
| 64 |
+
MAX_BUDGET_PAUSE_S = 3600.0
|
| 65 |
+
|
| 66 |
+
_POOL = ThreadPoolExecutor(max_workers=4, thread_name_prefix="openalex")
|
| 67 |
+
_LOCK = threading.Lock()
|
| 68 |
+
_STATE: dict[str, Any] = {
|
| 69 |
+
"open_until": 0.0,
|
| 70 |
+
"reason": "",
|
| 71 |
+
"remaining_usd": None,
|
| 72 |
+
"limit_usd": None,
|
| 73 |
+
"calls": 0,
|
| 74 |
+
"failures": 0,
|
| 75 |
+
}
|
| 76 |
+
_CACHE: OrderedDict[str, Any] = OrderedDict()
|
| 77 |
+
_CACHE_MAX = 512
|
| 78 |
+
|
| 79 |
+
_ARXIV_IN_URL = re.compile(r"arxiv\.org/(?:abs|pdf)/(?P<id>\d{4}\.\d{4,5})", re.IGNORECASE)
|
| 80 |
+
_ARXIV_DOI = re.compile(r"10\.48550/arxiv\.(?P<id>\d{4}\.\d{4,5})", re.IGNORECASE)
|
| 81 |
+
DOI_RE = re.compile(r"\b(?P<doi>10\.\d{4,9}/[^\s\"'<>]+)", re.IGNORECASE)
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
class OpenAlexUnavailable(RuntimeError):
|
| 85 |
+
"""OpenAlex did not answer in time, refused, or is paused."""
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
# ---------------------------------------------------------------- state ----
|
| 89 |
+
def breaker_open() -> bool:
|
| 90 |
+
with _LOCK:
|
| 91 |
+
return time.monotonic() < _STATE["open_until"]
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
def _pause(seconds: float, reason: str) -> None:
|
| 95 |
+
with _LOCK:
|
| 96 |
+
_STATE["open_until"] = time.monotonic() + seconds
|
| 97 |
+
_STATE["reason"] = reason
|
| 98 |
+
print(f"[openalex] {reason} - pausing live lookups for {seconds:.0f}s", flush=True)
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
def status() -> dict[str, Any]:
|
| 102 |
+
"""For /health: is the source usable right now, and how much budget is left."""
|
| 103 |
+
with _LOCK:
|
| 104 |
+
paused_for = max(0.0, _STATE["open_until"] - time.monotonic())
|
| 105 |
+
return {
|
| 106 |
+
"available": paused_for == 0.0,
|
| 107 |
+
"paused_for_s": round(paused_for, 1),
|
| 108 |
+
"reason": _STATE["reason"] if paused_for else "",
|
| 109 |
+
"keyed": bool(os.environ.get("OPENALEX_API_KEY")),
|
| 110 |
+
"remaining_usd": _STATE["remaining_usd"],
|
| 111 |
+
"limit_usd": _STATE["limit_usd"],
|
| 112 |
+
"calls": _STATE["calls"],
|
| 113 |
+
"failures": _STATE["failures"],
|
| 114 |
+
}
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
def _record_budget(headers: httpx.Headers) -> None:
|
| 118 |
+
"""Read the metering headers and pause ahead of exhaustion."""
|
| 119 |
+
try:
|
| 120 |
+
remaining = float(headers.get("X-RateLimit-Remaining-USD", "nan"))
|
| 121 |
+
limit = float(headers.get("X-RateLimit-Limit-USD", "nan"))
|
| 122 |
+
reset_s = float(headers.get("X-RateLimit-Reset", "3600"))
|
| 123 |
+
except ValueError:
|
| 124 |
+
return
|
| 125 |
+
if remaining != remaining: # NaN: header absent
|
| 126 |
+
return
|
| 127 |
+
with _LOCK:
|
| 128 |
+
_STATE["remaining_usd"] = round(remaining, 4)
|
| 129 |
+
_STATE["limit_usd"] = None if limit != limit else round(limit, 4)
|
| 130 |
+
if remaining < BUDGET_RESERVE_USD:
|
| 131 |
+
_pause(min(reset_s, MAX_BUDGET_PAUSE_S),
|
| 132 |
+
f"daily budget nearly spent (${remaining:.3f} left)")
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
# -------------------------------------------------------------- parsing ----
|
| 136 |
+
def reconstruct_abstract(inverted: dict[str, list[int]] | None) -> str:
|
| 137 |
+
"""OpenAlex ships abstracts as {word: [positions]} for licensing reasons."""
|
| 138 |
+
if not inverted:
|
| 139 |
+
return ""
|
| 140 |
+
slots: dict[int, str] = {}
|
| 141 |
+
for word, positions in inverted.items():
|
| 142 |
+
for pos in positions:
|
| 143 |
+
slots[pos] = word
|
| 144 |
+
return " ".join(slots[i] for i in sorted(slots))
|
| 145 |
+
|
| 146 |
+
|
| 147 |
+
def _arxiv_id(work: dict[str, Any]) -> str | None:
|
| 148 |
+
"""The arXiv id of a work, if any copy of it lives on arXiv."""
|
| 149 |
+
doi = work.get("doi") or ""
|
| 150 |
+
m = _ARXIV_DOI.search(doi)
|
| 151 |
+
if m:
|
| 152 |
+
return m.group("id")
|
| 153 |
+
for loc in work.get("locations") or []:
|
| 154 |
+
for key in ("landing_page_url", "pdf_url"):
|
| 155 |
+
m = _ARXIV_IN_URL.search(loc.get(key) or "")
|
| 156 |
+
if m:
|
| 157 |
+
return m.group("id")
|
| 158 |
+
return None
|
| 159 |
+
|
| 160 |
+
|
| 161 |
+
def normalise_doi(doi: str | None) -> str | None:
|
| 162 |
+
if not doi:
|
| 163 |
+
return None
|
| 164 |
+
doi = doi.strip().rstrip(".,;)")
|
| 165 |
+
doi = re.sub(r"^(?:https?://(?:dx\.)?doi\.org/|doi:)", "", doi, flags=re.IGNORECASE)
|
| 166 |
+
return doi.lower() or None
|
| 167 |
+
|
| 168 |
+
|
| 169 |
+
def parse_work(work: dict[str, Any]) -> dict[str, Any]:
|
| 170 |
+
"""One OpenAlex work -> the fields the pipeline uses."""
|
| 171 |
+
openalex_id = (work.get("id") or "").rsplit("/", 1)[-1]
|
| 172 |
+
arxiv_id = _arxiv_id(work)
|
| 173 |
+
doi = normalise_doi(work.get("doi"))
|
| 174 |
+
primary = work.get("primary_location") or {}
|
| 175 |
+
# Some venues arrive as "Name/Name/Proceedings - Name"; the first is enough.
|
| 176 |
+
venue = ((primary.get("source") or {}).get("display_name") or "").split("/")[0].strip()
|
| 177 |
+
if arxiv_id:
|
| 178 |
+
url = f"https://arxiv.org/abs/{arxiv_id}"
|
| 179 |
+
elif doi:
|
| 180 |
+
url = f"https://doi.org/{doi}"
|
| 181 |
+
else:
|
| 182 |
+
url = primary.get("landing_page_url") or work.get("id") or ""
|
| 183 |
+
authors = [
|
| 184 |
+
(a.get("author") or {}).get("display_name", "")
|
| 185 |
+
for a in (work.get("authorships") or [])[:20]
|
| 186 |
+
]
|
| 187 |
+
return {
|
| 188 |
+
"openalex_id": openalex_id,
|
| 189 |
+
# arXiv id when there is one, so the same paper found through arXiv,
|
| 190 |
+
# OpenAlex and the corpus collapses to a single entry.
|
| 191 |
+
"paper_id": arxiv_id or f"openalex:{openalex_id}",
|
| 192 |
+
"arxiv_id": arxiv_id,
|
| 193 |
+
"doi": doi,
|
| 194 |
+
"title": re.sub(r"\s+", " ", work.get("title") or "").strip(),
|
| 195 |
+
"abstract": re.sub(r"\s+", " ", reconstruct_abstract(work.get("abstract_inverted_index"))).strip(),
|
| 196 |
+
"authors": [a for a in authors if a],
|
| 197 |
+
"published": work.get("publication_date") or str(work.get("publication_year") or ""),
|
| 198 |
+
"year": work.get("publication_year"),
|
| 199 |
+
"venue": venue,
|
| 200 |
+
"type": work.get("type") or "",
|
| 201 |
+
"cited_by_count": int(work.get("cited_by_count") or 0),
|
| 202 |
+
"url": url,
|
| 203 |
+
"pdf_url": (work.get("open_access") or {}).get("oa_url") or "",
|
| 204 |
+
}
|
| 205 |
+
|
| 206 |
+
|
| 207 |
+
# ------------------------------------------------------------------ http ----
|
| 208 |
+
def _get(path: str, params: dict[str, Any]) -> dict[str, Any]:
|
| 209 |
+
headers = {"User-Agent": "ResearchOS (https://github.com/Kagzyyy/ResearchOS)"}
|
| 210 |
+
key = os.environ.get("OPENALEX_API_KEY")
|
| 211 |
+
if key:
|
| 212 |
+
headers["Authorization"] = f"Bearer {key}"
|
| 213 |
+
with _LOCK:
|
| 214 |
+
_STATE["calls"] += 1
|
| 215 |
+
r = httpx.get(f"{BASE_URL}{path}", params=params, headers=headers, timeout=10.0)
|
| 216 |
+
_record_budget(r.headers)
|
| 217 |
+
if r.status_code == 404:
|
| 218 |
+
return {}
|
| 219 |
+
if r.status_code == 429:
|
| 220 |
+
retry = float(r.headers.get("Retry-After", "60") or 60)
|
| 221 |
+
_pause(min(max(retry, 30.0), MAX_BUDGET_PAUSE_S), "rate limited (429)")
|
| 222 |
+
raise OpenAlexUnavailable("rate limited")
|
| 223 |
+
r.raise_for_status()
|
| 224 |
+
return r.json()
|
| 225 |
+
|
| 226 |
+
|
| 227 |
+
def _bounded(key: str, fn, budget_s: float):
|
| 228 |
+
"""Run fn() inside the budget, through the cache and the breaker."""
|
| 229 |
+
with _LOCK:
|
| 230 |
+
if key in _CACHE:
|
| 231 |
+
_CACHE.move_to_end(key)
|
| 232 |
+
return _CACHE[key]
|
| 233 |
+
if breaker_open():
|
| 234 |
+
raise OpenAlexUnavailable("paused after a recent failure")
|
| 235 |
+
future = _POOL.submit(fn)
|
| 236 |
+
try:
|
| 237 |
+
result = future.result(timeout=max(budget_s, 0.1))
|
| 238 |
+
except FuturesTimeout:
|
| 239 |
+
with _LOCK:
|
| 240 |
+
_STATE["failures"] += 1
|
| 241 |
+
_pause(BREAKER_COOLDOWN_S, f"no answer within {budget_s:.1f}s")
|
| 242 |
+
raise OpenAlexUnavailable(f"timed out after {budget_s:.1f}s") from None
|
| 243 |
+
except OpenAlexUnavailable:
|
| 244 |
+
with _LOCK:
|
| 245 |
+
_STATE["failures"] += 1
|
| 246 |
+
raise
|
| 247 |
+
except Exception as exc: # noqa: BLE001
|
| 248 |
+
with _LOCK:
|
| 249 |
+
_STATE["failures"] += 1
|
| 250 |
+
_pause(BREAKER_COOLDOWN_S, f"{type(exc).__name__}: {str(exc)[:80]}")
|
| 251 |
+
raise OpenAlexUnavailable(str(exc)) from exc
|
| 252 |
+
with _LOCK:
|
| 253 |
+
_CACHE[key] = result
|
| 254 |
+
if len(_CACHE) > _CACHE_MAX:
|
| 255 |
+
_CACHE.popitem(last=False)
|
| 256 |
+
return result
|
| 257 |
+
|
| 258 |
+
|
| 259 |
+
# ------------------------------------------------------------------- api ----
|
| 260 |
+
def search_works(query: str, max_results: int = 8, budget_s: float = REQUEST_BUDGET_S,
|
| 261 |
+
filter_: str = DEFAULT_FILTER) -> list[dict[str, Any]]:
|
| 262 |
+
"""Relevance-ranked search over titles, abstracts and full text."""
|
| 263 |
+
query = (query or "").strip()
|
| 264 |
+
if not query:
|
| 265 |
+
return []
|
| 266 |
+
params = {"search": query[:300], "per_page": max(1, min(max_results, 25)),
|
| 267 |
+
"select": SELECT, "filter": filter_}
|
| 268 |
+
|
| 269 |
+
def run() -> list[dict[str, Any]]:
|
| 270 |
+
works = _get("/works", params).get("results") or []
|
| 271 |
+
return [w for w in (parse_work(x) for x in works) if w["title"] and w["abstract"]]
|
| 272 |
+
|
| 273 |
+
return _bounded(f"search|{filter_}|{max_results}|{query.lower()}", run, budget_s)
|
| 274 |
+
|
| 275 |
+
|
| 276 |
+
def search_title(title: str, max_results: int = 8,
|
| 277 |
+
budget_s: float = REQUEST_BUDGET_S) -> list[dict[str, Any]]:
|
| 278 |
+
"""Title-only search, for citation resolution. No field filter: a cited
|
| 279 |
+
paper can be from any field, and the title match itself is the check."""
|
| 280 |
+
title = re.sub(r"[,:;()\[\]\"]", " ", title or "").strip()
|
| 281 |
+
if not title:
|
| 282 |
+
return []
|
| 283 |
+
params = {"filter": f"title.search:{title[:250]}",
|
| 284 |
+
"per_page": max(1, min(max_results, 25)), "select": SELECT}
|
| 285 |
+
|
| 286 |
+
def run() -> list[dict[str, Any]]:
|
| 287 |
+
works = _get("/works", params).get("results") or []
|
| 288 |
+
return [parse_work(x) for x in works if x.get("title")]
|
| 289 |
+
|
| 290 |
+
return _bounded(f"title|{max_results}|{title.lower()}", run, budget_s)
|
| 291 |
+
|
| 292 |
+
|
| 293 |
+
def get_by_doi(doi: str, budget_s: float = REQUEST_BUDGET_S) -> dict[str, Any] | None:
|
| 294 |
+
doi = normalise_doi(doi)
|
| 295 |
+
if not doi:
|
| 296 |
+
return None
|
| 297 |
+
|
| 298 |
+
def run() -> dict[str, Any] | None:
|
| 299 |
+
work = _get(f"/works/doi:{doi}", {"select": SELECT})
|
| 300 |
+
return parse_work(work) if work.get("id") else None
|
| 301 |
+
|
| 302 |
+
return _bounded(f"doi|{doi}", run, budget_s)
|
| 303 |
+
|
| 304 |
+
|
| 305 |
+
def extract_doi(text: str) -> str | None:
|
| 306 |
+
m = DOI_RE.search(text or "")
|
| 307 |
+
return normalise_doi(m.group("doi")) if m else None
|