Kagzy commited on
Commit
a3c7ac6
·
verified ·
1 Parent(s): 5eec49d

OpenAlex live source, DOI citations, live v2 funnel

Browse files
app.py CHANGED
@@ -152,7 +152,8 @@ def _render_papers(result: dict[str, Any]) -> str:
152
  lines: list[str] = [diag] if diag else []
153
  if result.get("source") == "arxiv_live":
154
  lines.append(
155
- "> Not in the indexed corpus, so these came from a live arXiv search just now.\n"
 
156
  )
157
  elif not result.get("confident"):
158
  lines.append("> Weak matches — treat these as leads rather than answers.\n")
@@ -166,7 +167,10 @@ def _render_papers(result: dict[str, Any]) -> str:
166
  meta = [m for m in (
167
  _authors(paper.get("authors") or []),
168
  (paper.get("published") or "")[:10],
169
- paper.get("primary_category") or "",
 
 
 
170
  f"score {paper.get('score')}",
171
  ) if m]
172
  lines.append(" · ".join(meta))
 
152
  lines: list[str] = [diag] if diag else []
153
  if result.get("source") == "arxiv_live":
154
  lines.append(
155
+ "> Not in the indexed corpus, so these came from a live search of arXiv and "
156
+ "OpenAlex just now, rescored by the same reranker.\n"
157
  )
158
  elif not result.get("confident"):
159
  lines.append("> Weak matches — treat these as leads rather than answers.\n")
 
167
  meta = [m for m in (
168
  _authors(paper.get("authors") or []),
169
  (paper.get("published") or "")[:10],
170
+ (paper.get("venue") if paper.get("venue") not in (None, "", "arXiv")
171
+ else paper.get("primary_category")) or "",
172
+ f"{paper['cited_by_count']:,} citations" if paper.get("cited_by_count") else "",
173
+ f"live · {paper['live_source']}" if paper.get("live_source") else "",
174
  f"score {paper.get('score')}",
175
  ) if m]
176
  lines.append(" · ".join(meta))
configs/ml.cpu.yaml CHANGED
@@ -115,6 +115,10 @@ citation:
115
  # shared IPs (Hugging Face Spaces) and a throttled call can hang ~50s before
116
  # failing; past this budget the answer goes out from the corpus instead.
117
  live_budget_s: 5.0
 
 
 
 
118
  fetch_full_text: true # download the PDF for on-demand papers
119
  # Cap the on-demand PDF cache. Citation lookup downloads one PDF per
120
  # unresolved paper, which grows without bound on a shared deployment.
 
115
  # shared IPs (Hugging Face Spaces) and a throttled call can hang ~50s before
116
  # failing; past this budget the answer goes out from the corpus instead.
117
  live_budget_s: 5.0
118
+ # Queried in parallel under the one budget above. OpenAlex adds the
119
+ # conference/journal papers with no arXiv preprint, plus DOI lookup; set
120
+ # OPENALEX_API_KEY for a 10x daily budget (keyless works, ~100 searches/day).
121
+ live_sources: [arxiv, openalex]
122
  fetch_full_text: true # download the PDF for on-demand papers
123
  # Cap the on-demand PDF cache. Citation lookup downloads one PDF per
124
  # unresolved paper, which grows without bound on a shared deployment.
configs/ml.yaml CHANGED
@@ -105,6 +105,10 @@ citation:
105
  # shared IPs (Hugging Face Spaces) and a throttled call can hang ~50s before
106
  # failing; past this budget the answer goes out from the corpus instead.
107
  live_budget_s: 5.0
 
 
 
 
108
  fetch_full_text: true # download the PDF for on-demand papers
109
  # Cap the on-demand PDF cache. Citation lookup downloads one PDF per
110
  # unresolved paper, which grows without bound on a shared deployment.
 
105
  # shared IPs (Hugging Face Spaces) and a throttled call can hang ~50s before
106
  # failing; past this budget the answer goes out from the corpus instead.
107
  live_budget_s: 5.0
108
+ # Queried in parallel under the one budget above. OpenAlex adds the
109
+ # conference/journal papers with no arXiv preprint, plus DOI lookup; set
110
+ # OPENALEX_API_KEY for a 10x daily budget (keyless works, ~100 searches/day).
111
+ live_sources: [arxiv, openalex]
112
  fetch_full_text: true # download the PDF for on-demand papers
113
  # Cap the on-demand PDF cache. Citation lookup downloads one PDF per
114
  # unresolved paper, which grows without bound on a shared deployment.
src/api/db.py CHANGED
@@ -1,223 +1,303 @@
1
- """SQLite logging: queries, citations, clicks, votes and failures.
2
-
3
- This is the instrumentation that makes a small deployment worth running. With
4
- several people using it, the interesting question is not "did it work" but
5
- "where is it failing, and on what kind of question" - which needs the failures
6
- recorded as deliberately as the successes.
7
-
8
- Four signals, in ascending order of how much they tell you:
9
-
10
- clicks which citation someone opened. Weak, plentiful, unbiased.
11
- votes an explicit thumbs up/down. Strong, scarce, self-selected.
12
- low queries the retriever itself flagged as out-of-corpus. Free, and the
13
- fastest route to "what do people ask that this cannot answer".
14
- errors exceptions, with endpoint and query. Silent otherwise.
15
- """
16
-
17
- from __future__ import annotations
18
-
19
- import sqlite3
20
- import traceback
21
- from datetime import datetime, timezone
22
- from pathlib import Path
23
- from typing import Any
24
-
25
- SCHEMA = """
26
- CREATE TABLE IF NOT EXISTS queries (
27
- id INTEGER PRIMARY KEY AUTOINCREMENT,
28
- ts TEXT NOT NULL,
29
- query TEXT NOT NULL,
30
- answer TEXT,
31
- latency_ms REAL,
32
- backend TEXT
33
- );
34
-
35
- CREATE TABLE IF NOT EXISTS citations (
36
- id INTEGER PRIMARY KEY AUTOINCREMENT,
37
- query_id INTEGER NOT NULL REFERENCES queries(id),
38
- doc_id TEXT NOT NULL,
39
- rank INTEGER NOT NULL,
40
- score REAL,
41
- title TEXT
42
- );
43
-
44
- CREATE TABLE IF NOT EXISTS citation_clicks (
45
- id INTEGER PRIMARY KEY AUTOINCREMENT,
46
- ts TEXT NOT NULL,
47
- query_id INTEGER NOT NULL REFERENCES queries(id),
48
- doc_id TEXT NOT NULL
49
- );
50
-
51
- CREATE TABLE IF NOT EXISTS votes (
52
- id INTEGER PRIMARY KEY AUTOINCREMENT,
53
- ts TEXT NOT NULL,
54
- query_id INTEGER NOT NULL REFERENCES queries(id),
55
- helpful INTEGER NOT NULL,
56
- comment TEXT
57
- );
58
-
59
- CREATE TABLE IF NOT EXISTS errors (
60
- id INTEGER PRIMARY KEY AUTOINCREMENT,
61
- ts TEXT NOT NULL,
62
- endpoint TEXT NOT NULL,
63
- query TEXT,
64
- error_type TEXT NOT NULL,
65
- message TEXT,
66
- stack TEXT
67
- );
68
-
69
- CREATE INDEX IF NOT EXISTS idx_citations_query ON citations(query_id);
70
- CREATE INDEX IF NOT EXISTS idx_clicks_query ON citation_clicks(query_id);
71
- CREATE INDEX IF NOT EXISTS idx_votes_query ON votes(query_id);
72
- CREATE INDEX IF NOT EXISTS idx_errors_ts ON errors(ts);
73
- """
74
-
75
- # Columns added after the first release. SQLite has no "ADD COLUMN IF NOT
76
- # EXISTS", so they are applied one at a time and duplicates ignored.
77
- MIGRATIONS = [
78
- ("queries", "mode", "TEXT"),
79
- ("queries", "confident", "INTEGER"),
80
- ("queries", "top_score", "REAL"),
81
- ("queries", "mean_top_score", "REAL"),
82
- ("queries", "n_results", "INTEGER"),
83
- ("queries", "generated", "INTEGER"),
84
- ]
85
-
86
-
87
- def _now() -> str:
88
- return datetime.now(timezone.utc).isoformat(timespec="seconds")
89
-
90
-
91
- class FeedbackLog:
92
- """Thin wrapper over a SQLite file. Safe to construct per request."""
93
-
94
- def __init__(self, path: str = "results/feedback.db") -> None:
95
- self.path = Path(path)
96
- self.path.parent.mkdir(parents=True, exist_ok=True)
97
- with self._connect() as conn:
98
- conn.executescript(SCHEMA)
99
- for table, column, coltype in MIGRATIONS:
100
- try:
101
- conn.execute(f"ALTER TABLE {table} ADD COLUMN {column} {coltype}")
102
- except sqlite3.OperationalError:
103
- pass # already present
104
-
105
- def _connect(self) -> sqlite3.Connection:
106
- conn = sqlite3.connect(self.path, timeout=10.0)
107
- conn.row_factory = sqlite3.Row
108
- return conn
109
-
110
- # ------------------------------------------------------------- recording
111
-
112
- def log_query(
113
- self,
114
- query: str,
115
- answer: str | None,
116
- latency_ms: float,
117
- backend: str,
118
- citations: list[dict[str, Any]],
119
- mode: str = "ask",
120
- confident: bool | None = None,
121
- top_score: float | None = None,
122
- mean_top_score: float | None = None,
123
- generated: bool | None = None,
124
- ) -> int:
125
- """Record a query, its answer, and the passages shown."""
126
- with self._connect() as conn:
127
- cursor = conn.execute(
128
- "INSERT INTO queries (ts, query, answer, latency_ms, backend, mode, "
129
- "confident, top_score, mean_top_score, n_results, generated) "
130
- "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
131
- (
132
- _now(), query, answer, latency_ms, backend, mode,
133
- None if confident is None else int(confident),
134
- top_score, mean_top_score, len(citations),
135
- None if generated is None else int(generated),
136
- ),
137
- )
138
- query_id = int(cursor.lastrowid)
139
- conn.executemany(
140
- "INSERT INTO citations (query_id, doc_id, rank, score, title) VALUES (?, ?, ?, ?, ?)",
141
- [
142
- (query_id, c.get("doc_id", ""), rank, c.get("score"), c.get("title"))
143
- for rank, c in enumerate(citations, start=1)
144
- ],
145
- )
146
- return query_id
147
-
148
- def log_click(self, query_id: int, doc_id: str) -> None:
149
- with self._connect() as conn:
150
- conn.execute(
151
- "INSERT INTO citation_clicks (ts, query_id, doc_id) VALUES (?, ?, ?)",
152
- (_now(), query_id, doc_id),
153
- )
154
-
155
- def log_vote(self, query_id: int, helpful: bool, comment: str | None = None) -> None:
156
- """Explicit thumbs up/down. The scarcest and most valuable signal."""
157
- with self._connect() as conn:
158
- conn.execute(
159
- "INSERT INTO votes (ts, query_id, helpful, comment) VALUES (?, ?, ?, ?)",
160
- (_now(), query_id, int(helpful), comment),
161
- )
162
-
163
- def log_error(
164
- self, endpoint: str, error: BaseException, query: str | None = None
165
- ) -> None:
166
- """Record an exception. Without this, failures are invisible."""
167
- with self._connect() as conn:
168
- conn.execute(
169
- "INSERT INTO errors (ts, endpoint, query, error_type, message, stack) "
170
- "VALUES (?, ?, ?, ?, ?, ?)",
171
- (
172
- _now(), endpoint, query, type(error).__name__, str(error)[:2000],
173
- "".join(traceback.format_exception(type(error), error, error.__traceback__))[-4000:],
174
- ),
175
- )
176
-
177
- # ------------------------------------------------------------- reporting
178
-
179
- def stats(self) -> dict[str, int]:
180
- with self._connect() as conn:
181
- def count(sql: str) -> int:
182
- return int(conn.execute(sql).fetchone()[0])
183
-
184
- return {
185
- "queries": count("SELECT COUNT(*) FROM queries"),
186
- "citations": count("SELECT COUNT(*) FROM citations"),
187
- "clicks": count("SELECT COUNT(*) FROM citation_clicks"),
188
- "votes_up": count("SELECT COUNT(*) FROM votes WHERE helpful=1"),
189
- "votes_down": count("SELECT COUNT(*) FROM votes WHERE helpful=0"),
190
- "errors": count("SELECT COUNT(*) FROM errors"),
191
- "low_confidence": count("SELECT COUNT(*) FROM queries WHERE confident=0"),
192
- }
193
-
194
- def recent_failures(self, limit: int = 50) -> dict[str, list[dict[str, Any]]]:
195
- """Everything worth reading when asking 'how is this failing?'."""
196
- with self._connect() as conn:
197
- def rows(sql: str, *args: Any) -> list[dict[str, Any]]:
198
- return [dict(r) for r in conn.execute(sql, args).fetchall()]
199
-
200
- return {
201
- "errors": rows(
202
- "SELECT ts, endpoint, query, error_type, message FROM errors "
203
- "ORDER BY id DESC LIMIT ?", limit
204
- ),
205
- "downvoted": rows(
206
- "SELECT v.ts, q.query, q.mode, v.comment FROM votes v "
207
- "JOIN queries q ON q.id = v.query_id WHERE v.helpful = 0 "
208
- "ORDER BY v.id DESC LIMIT ?", limit
209
- ),
210
- "out_of_corpus": rows(
211
- "SELECT ts, query, mode, mean_top_score FROM queries "
212
- "WHERE confident = 0 ORDER BY id DESC LIMIT ?", limit
213
- ),
214
- "slowest": rows(
215
- "SELECT ts, query, mode, latency_ms FROM queries "
216
- "WHERE latency_ms IS NOT NULL ORDER BY latency_ms DESC LIMIT ?", limit
217
- ),
218
- "never_clicked": rows(
219
- "SELECT q.ts, q.query, q.mode FROM queries q "
220
- "LEFT JOIN citation_clicks c ON c.query_id = q.id "
221
- "WHERE c.id IS NULL AND q.confident = 1 ORDER BY q.id DESC LIMIT ?", limit
222
- ),
223
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """SQLite logging: queries, citations, clicks, votes and failures.
2
+
3
+ This is the instrumentation that makes a small deployment worth running. With
4
+ several people using it, the interesting question is not "did it work" but
5
+ "where is it failing, and on what kind of question" - which needs the failures
6
+ recorded as deliberately as the successes.
7
+
8
+ Four signals, in ascending order of how much they tell you:
9
+
10
+ clicks which citation someone opened. Weak, plentiful, unbiased.
11
+ votes an explicit thumbs up/down. Strong, scarce, self-selected.
12
+ low queries the retriever itself flagged as out-of-corpus. Free, and the
13
+ fastest route to "what do people ask that this cannot answer".
14
+ errors exceptions, with endpoint and query. Silent otherwise.
15
+ """
16
+
17
+ from __future__ import annotations
18
+
19
+ import sqlite3
20
+ import traceback
21
+ from datetime import datetime, timezone
22
+ from pathlib import Path
23
+ from typing import Any
24
+
25
+ SCHEMA = """
26
+ CREATE TABLE IF NOT EXISTS queries (
27
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
28
+ ts TEXT NOT NULL,
29
+ query TEXT NOT NULL,
30
+ answer TEXT,
31
+ latency_ms REAL,
32
+ backend TEXT
33
+ );
34
+
35
+ CREATE TABLE IF NOT EXISTS citations (
36
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
37
+ query_id INTEGER NOT NULL REFERENCES queries(id),
38
+ doc_id TEXT NOT NULL,
39
+ rank INTEGER NOT NULL,
40
+ score REAL,
41
+ title TEXT
42
+ );
43
+
44
+ CREATE TABLE IF NOT EXISTS citation_clicks (
45
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
46
+ ts TEXT NOT NULL,
47
+ query_id INTEGER NOT NULL REFERENCES queries(id),
48
+ doc_id TEXT NOT NULL
49
+ );
50
+
51
+ CREATE TABLE IF NOT EXISTS votes (
52
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
53
+ ts TEXT NOT NULL,
54
+ query_id INTEGER NOT NULL REFERENCES queries(id),
55
+ helpful INTEGER NOT NULL,
56
+ comment TEXT
57
+ );
58
+
59
+ CREATE TABLE IF NOT EXISTS errors (
60
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
61
+ ts TEXT NOT NULL,
62
+ endpoint TEXT NOT NULL,
63
+ query TEXT,
64
+ error_type TEXT NOT NULL,
65
+ message TEXT,
66
+ stack TEXT
67
+ );
68
+
69
+ CREATE INDEX IF NOT EXISTS idx_citations_query ON citations(query_id);
70
+ CREATE INDEX IF NOT EXISTS idx_clicks_query ON citation_clicks(query_id);
71
+ CREATE INDEX IF NOT EXISTS idx_votes_query ON votes(query_id);
72
+ CREATE INDEX IF NOT EXISTS idx_errors_ts ON errors(ts);
73
+ """
74
+
75
+ # Columns added after the first release. SQLite has no "ADD COLUMN IF NOT
76
+ # EXISTS", so they are applied one at a time and duplicates ignored.
77
+ MIGRATIONS = [
78
+ ("queries", "mode", "TEXT"),
79
+ ("queries", "confident", "INTEGER"),
80
+ ("queries", "top_score", "REAL"),
81
+ ("queries", "mean_top_score", "REAL"),
82
+ ("queries", "n_results", "INTEGER"),
83
+ ("queries", "generated", "INTEGER"),
84
+ # v2: what the diagnosis said, and whether a query was a retry of one.
85
+ ("queries", "source", "TEXT"),
86
+ ("queries", "diagnosis_location", "TEXT"),
87
+ ("queries", "suggested_query", "TEXT"),
88
+ ("queries", "retry_of", "INTEGER"),
89
+ ]
90
+
91
+
92
+ def _now() -> str:
93
+ return datetime.now(timezone.utc).isoformat(timespec="seconds")
94
+
95
+
96
+ class FeedbackLog:
97
+ """Thin wrapper over a SQLite file. Safe to construct per request."""
98
+
99
+ def __init__(self, path: str = "results/feedback.db") -> None:
100
+ self.path = Path(path)
101
+ self.path.parent.mkdir(parents=True, exist_ok=True)
102
+ with self._connect() as conn:
103
+ conn.executescript(SCHEMA)
104
+ for table, column, coltype in MIGRATIONS:
105
+ try:
106
+ conn.execute(f"ALTER TABLE {table} ADD COLUMN {column} {coltype}")
107
+ except sqlite3.OperationalError:
108
+ pass # already present
109
+
110
+ def _connect(self) -> sqlite3.Connection:
111
+ conn = sqlite3.connect(self.path, timeout=10.0)
112
+ conn.row_factory = sqlite3.Row
113
+ return conn
114
+
115
+ # ------------------------------------------------------------- recording
116
+
117
+ def log_query(
118
+ self,
119
+ query: str,
120
+ answer: str | None,
121
+ latency_ms: float,
122
+ backend: str,
123
+ citations: list[dict[str, Any]],
124
+ mode: str = "ask",
125
+ confident: bool | None = None,
126
+ top_score: float | None = None,
127
+ mean_top_score: float | None = None,
128
+ generated: bool | None = None,
129
+ source: str | None = None,
130
+ diagnosis: dict[str, Any] | None = None,
131
+ retry_of: int | None = None,
132
+ ) -> int:
133
+ """Record a query, its answer, the passages shown, and any v2 diagnosis."""
134
+ with self._connect() as conn:
135
+ if retry_of is not None and conn.execute(
136
+ "SELECT 1 FROM queries WHERE id = ?", (retry_of,)).fetchone() is None:
137
+ retry_of = None # a stale or forged id must not corrupt the funnel
138
+ cursor = conn.execute(
139
+ "INSERT INTO queries (ts, query, answer, latency_ms, backend, mode, "
140
+ "confident, top_score, mean_top_score, n_results, generated, source, "
141
+ "diagnosis_location, suggested_query, retry_of) "
142
+ "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
143
+ (
144
+ _now(), query, answer, latency_ms, backend, mode,
145
+ None if confident is None else int(confident),
146
+ top_score, mean_top_score, len(citations),
147
+ None if generated is None else int(generated),
148
+ source,
149
+ (diagnosis or {}).get("location"),
150
+ (diagnosis or {}).get("suggested_query"),
151
+ retry_of,
152
+ ),
153
+ )
154
+ query_id = int(cursor.lastrowid)
155
+ conn.executemany(
156
+ "INSERT INTO citations (query_id, doc_id, rank, score, title) VALUES (?, ?, ?, ?, ?)",
157
+ [
158
+ (query_id, c.get("doc_id", ""), rank, c.get("score"), c.get("title"))
159
+ for rank, c in enumerate(citations, start=1)
160
+ ],
161
+ )
162
+ return query_id
163
+
164
+ def log_click(self, query_id: int, doc_id: str) -> None:
165
+ with self._connect() as conn:
166
+ conn.execute(
167
+ "INSERT INTO citation_clicks (ts, query_id, doc_id) VALUES (?, ?, ?)",
168
+ (_now(), query_id, doc_id),
169
+ )
170
+
171
+ def log_vote(self, query_id: int, helpful: bool, comment: str | None = None) -> None:
172
+ """Explicit thumbs up/down. The scarcest and most valuable signal."""
173
+ with self._connect() as conn:
174
+ conn.execute(
175
+ "INSERT INTO votes (ts, query_id, helpful, comment) VALUES (?, ?, ?, ?)",
176
+ (_now(), query_id, int(helpful), comment),
177
+ )
178
+
179
+ def log_error(
180
+ self, endpoint: str, error: BaseException, query: str | None = None
181
+ ) -> None:
182
+ """Record an exception. Without this, failures are invisible."""
183
+ with self._connect() as conn:
184
+ conn.execute(
185
+ "INSERT INTO errors (ts, endpoint, query, error_type, message, stack) "
186
+ "VALUES (?, ?, ?, ?, ?, ?)",
187
+ (
188
+ _now(), endpoint, query, type(error).__name__, str(error)[:2000],
189
+ "".join(traceback.format_exception(type(error), error, error.__traceback__))[-4000:],
190
+ ),
191
+ )
192
+
193
+ # ------------------------------------------------------------- reporting
194
+
195
+ def stats(self) -> dict[str, int]:
196
+ with self._connect() as conn:
197
+ def count(sql: str) -> int:
198
+ return int(conn.execute(sql).fetchone()[0])
199
+
200
+ return {
201
+ "queries": count("SELECT COUNT(*) FROM queries"),
202
+ "citations": count("SELECT COUNT(*) FROM citations"),
203
+ "clicks": count("SELECT COUNT(*) FROM citation_clicks"),
204
+ "votes_up": count("SELECT COUNT(*) FROM votes WHERE helpful=1"),
205
+ "votes_down": count("SELECT COUNT(*) FROM votes WHERE helpful=0"),
206
+ "errors": count("SELECT COUNT(*) FROM errors"),
207
+ "low_confidence": count("SELECT COUNT(*) FROM queries WHERE confident=0"),
208
+ }
209
+
210
+ def v2_metrics(self) -> dict[str, Any]:
211
+ """The live v2 funnel: diagnosis shown -> retry clicked -> result used.
212
+
213
+ Offline evaluation can only ask whether the target paper ranked; this
214
+ asks whether real users found the suggestion worth following, and
215
+ whether following it led to a paper they opened. The baseline is
216
+ gate-fired queries nobody retried: did those users open anything?
217
+ """
218
+ with self._connect() as conn:
219
+ def one(sql: str) -> float:
220
+ return float(conn.execute(sql).fetchone()[0] or 0)
221
+
222
+ shown = one("SELECT COUNT(*) FROM queries WHERE diagnosis_location IS NOT NULL")
223
+ with_query = one("SELECT COUNT(*) FROM queries WHERE suggested_query IS NOT NULL")
224
+ retries = one("SELECT COUNT(*) FROM queries WHERE retry_of IS NOT NULL")
225
+ retry_clicked = one(
226
+ "SELECT COUNT(DISTINCT q.id) FROM queries q JOIN citation_clicks c "
227
+ "ON c.query_id = q.id WHERE q.retry_of IS NOT NULL")
228
+ fired_not_retried = one(
229
+ "SELECT COUNT(*) FROM queries q WHERE q.confident = 0 AND q.retry_of IS NULL "
230
+ "AND NOT EXISTS (SELECT 1 FROM queries r WHERE r.retry_of = q.id)")
231
+ fired_not_retried_clicked = one(
232
+ "SELECT COUNT(DISTINCT q.id) FROM queries q JOIN citation_clicks c "
233
+ "ON c.query_id = q.id WHERE q.confident = 0 AND q.retry_of IS NULL "
234
+ "AND NOT EXISTS (SELECT 1 FROM queries r WHERE r.retry_of = q.id)")
235
+ by_location = {
236
+ (r[0] or "none").split(":")[0]: r[1] for r in conn.execute(
237
+ "SELECT diagnosis_location, COUNT(*) FROM queries "
238
+ "WHERE diagnosis_location IS NOT NULL GROUP BY 1")
239
+ }
240
+ by_source = {r[0] or "corpus": r[1] for r in conn.execute(
241
+ "SELECT source, COUNT(*) FROM queries GROUP BY 1")}
242
+
243
+ def rate(a: float, b: float) -> float | None:
244
+ return round(a / b, 4) if b else None
245
+
246
+ return {
247
+ "diagnoses_shown": int(shown),
248
+ "diagnoses_with_query": int(with_query),
249
+ "retries": int(retries),
250
+ "retry_rate": rate(retries, with_query),
251
+ "retry_click_through": rate(retry_clicked, retries),
252
+ "baseline_click_through": rate(fired_not_retried_clicked, fired_not_retried),
253
+ "baseline_queries": int(fired_not_retried),
254
+ "diagnosis_locations": by_location,
255
+ "answer_sources": by_source,
256
+ }
257
+
258
+ def v2_labels(self, limit: int = 500) -> list[dict[str, Any]]:
259
+ """Retried-then-clicked pairs: free supervision for a failure detector.
260
+
261
+ Each row says "the original search did not satisfy this user, the
262
+ suggested query did, and here is the paper they opened" - exactly the
263
+ signal the offline gate could not see.
264
+ """
265
+ with self._connect() as conn:
266
+ return [dict(r) for r in conn.execute(
267
+ "SELECT o.id AS original_id, o.query AS original_query, "
268
+ "o.mean_top_score AS original_mean_score, o.diagnosis_location, "
269
+ "r.id AS retry_id, r.query AS retry_query, c.doc_id AS clicked_doc, "
270
+ "r.ts AS ts FROM queries r JOIN queries o ON o.id = r.retry_of "
271
+ "JOIN citation_clicks c ON c.query_id = r.id "
272
+ "ORDER BY r.id DESC LIMIT ?", (limit,)).fetchall()]
273
+
274
+ def recent_failures(self, limit: int = 50) -> dict[str, list[dict[str, Any]]]:
275
+ """Everything worth reading when asking 'how is this failing?'."""
276
+ with self._connect() as conn:
277
+ def rows(sql: str, *args: Any) -> list[dict[str, Any]]:
278
+ return [dict(r) for r in conn.execute(sql, args).fetchall()]
279
+
280
+ return {
281
+ "errors": rows(
282
+ "SELECT ts, endpoint, query, error_type, message FROM errors "
283
+ "ORDER BY id DESC LIMIT ?", limit
284
+ ),
285
+ "downvoted": rows(
286
+ "SELECT v.ts, q.query, q.mode, v.comment FROM votes v "
287
+ "JOIN queries q ON q.id = v.query_id WHERE v.helpful = 0 "
288
+ "ORDER BY v.id DESC LIMIT ?", limit
289
+ ),
290
+ "out_of_corpus": rows(
291
+ "SELECT ts, query, mode, mean_top_score FROM queries "
292
+ "WHERE confident = 0 ORDER BY id DESC LIMIT ?", limit
293
+ ),
294
+ "slowest": rows(
295
+ "SELECT ts, query, mode, latency_ms FROM queries "
296
+ "WHERE latency_ms IS NOT NULL ORDER BY latency_ms DESC LIMIT ?", limit
297
+ ),
298
+ "never_clicked": rows(
299
+ "SELECT q.ts, q.query, q.mode FROM queries q "
300
+ "LEFT JOIN citation_clicks c ON c.query_id = q.id "
301
+ "WHERE c.id IS NULL AND q.confident = 1 ORDER BY q.id DESC LIMIT ?", limit
302
+ ),
303
+ }
src/api/ml_app.py CHANGED
@@ -31,6 +31,8 @@ from src.api.ml_pipeline import MLPipeline
31
  from src.api.security import allowed_origins, api_token, guard, rate_limit_per_minute
32
  from src.api.uploads import UploadError, UploadStore
33
  from src.config import load_config
 
 
34
 
35
  CONFIG_PATH = os.environ.get("RESEARCHOS_ML_CONFIG", "configs/ml.yaml")
36
 
@@ -80,11 +82,15 @@ class AskRequest(BaseModel):
80
  query: str = Field(min_length=3, max_length=1000)
81
  top_k: int = Field(default=6, ge=1, le=20)
82
  generate: bool = True
 
 
 
83
 
84
 
85
  class PapersRequest(BaseModel):
86
  topic: str = Field(min_length=3, max_length=1000)
87
  top_k: int = Field(default=10, ge=1, le=30)
 
88
 
89
 
90
  class ExplainRequest(BaseModel):
@@ -93,6 +99,7 @@ class ExplainRequest(BaseModel):
93
  # When set, the question is answered against this uploaded PDF instead of
94
  # the corpus.
95
  document_id: str | None = Field(default=None, max_length=64)
 
96
 
97
 
98
  class ClickRequest(BaseModel):
@@ -134,6 +141,12 @@ def health() -> dict[str, Any]:
134
  "generation_enabled": is_configured(),
135
  "auth_required": api_token() is not None,
136
  "rate_limit_per_minute": rate_limit_per_minute(),
 
 
 
 
 
 
137
  }
138
 
139
 
@@ -164,6 +177,9 @@ def papers(request: PapersRequest) -> dict[str, Any]:
164
  top_score=result["top_score"],
165
  mean_top_score=result["mean_top_score"],
166
  generated=False,
 
 
 
167
  )
168
  return {"query_id": query_id, **result}
169
 
@@ -185,6 +201,9 @@ def ask(request: AskRequest) -> dict[str, Any]:
185
  top_score=result["top_score"],
186
  mean_top_score=result["mean_top_score"],
187
  generated=bool(result["answer"]) and result["generation_configured"],
 
 
 
188
  )
189
  return {"query_id": query_id, **result}
190
 
@@ -237,6 +256,9 @@ def explain(request: ExplainRequest) -> dict[str, Any]:
237
  mode="explain",
238
  confident=result["resolved"],
239
  generated=bool(result.get("explanation")) and result["generation_configured"],
 
 
 
240
  )
241
  return {"query_id": query_id, **result}
242
 
@@ -283,3 +305,16 @@ def failures(limit: int = 30) -> dict[str, Any]:
283
  @app.get("/stats")
284
  def stats() -> dict[str, int]:
285
  return state["log"].stats()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
31
  from src.api.security import allowed_origins, api_token, guard, rate_limit_per_minute
32
  from src.api.uploads import UploadError, UploadStore
33
  from src.config import load_config
34
+ from src.data.arxiv_source import breaker_open as arxiv_breaker_open
35
+ from src.data.openalex_source import status as openalex_status
36
 
37
  CONFIG_PATH = os.environ.get("RESEARCHOS_ML_CONFIG", "configs/ml.yaml")
38
 
 
82
  query: str = Field(min_length=3, max_length=1000)
83
  top_k: int = Field(default=6, ge=1, le=20)
84
  generate: bool = True
85
+ # Set when this query came from a v2 diagnosis's "Try: ..." button: the id
86
+ # of the query whose diagnosis suggested it. Drives the live v2 funnel.
87
+ retry_of: int | None = Field(default=None, ge=1)
88
 
89
 
90
  class PapersRequest(BaseModel):
91
  topic: str = Field(min_length=3, max_length=1000)
92
  top_k: int = Field(default=10, ge=1, le=30)
93
+ retry_of: int | None = Field(default=None, ge=1)
94
 
95
 
96
  class ExplainRequest(BaseModel):
 
99
  # When set, the question is answered against this uploaded PDF instead of
100
  # the corpus.
101
  document_id: str | None = Field(default=None, max_length=64)
102
+ retry_of: int | None = Field(default=None, ge=1)
103
 
104
 
105
  class ClickRequest(BaseModel):
 
141
  "generation_enabled": is_configured(),
142
  "auth_required": api_token() is not None,
143
  "rate_limit_per_minute": rate_limit_per_minute(),
144
+ # Live sources are optional enrichments; this says which are usable
145
+ # right now and how much of OpenAlex's daily budget is left.
146
+ "live_sources": {
147
+ "arxiv": {"available": not arxiv_breaker_open()},
148
+ "openalex": openalex_status(),
149
+ },
150
  }
151
 
152
 
 
177
  top_score=result["top_score"],
178
  mean_top_score=result["mean_top_score"],
179
  generated=False,
180
+ source=result.get("source"),
181
+ diagnosis=result.get("diagnosis"),
182
+ retry_of=request.retry_of,
183
  )
184
  return {"query_id": query_id, **result}
185
 
 
201
  top_score=result["top_score"],
202
  mean_top_score=result["mean_top_score"],
203
  generated=bool(result["answer"]) and result["generation_configured"],
204
+ source=result.get("source"),
205
+ diagnosis=result.get("diagnosis"),
206
+ retry_of=request.retry_of,
207
  )
208
  return {"query_id": query_id, **result}
209
 
 
256
  mode="explain",
257
  confident=result["resolved"],
258
  generated=bool(result.get("explanation")) and result["generation_configured"],
259
+ source=result.get("source"),
260
+ diagnosis=result.get("diagnosis"),
261
+ retry_of=request.retry_of,
262
  )
263
  return {"query_id": query_id, **result}
264
 
 
305
  @app.get("/stats")
306
  def stats() -> dict[str, int]:
307
  return state["log"].stats()
308
+
309
+
310
+ @app.get("/v2/metrics")
311
+ def v2_metrics() -> dict[str, Any]:
312
+ """Live v2 funnel: diagnoses shown, retries taken, retries that led to a click."""
313
+ return state["log"].v2_metrics()
314
+
315
+
316
+ @app.get("/v2/labels")
317
+ def v2_labels(limit: int = 200) -> dict[str, Any]:
318
+ """Retried-then-clicked pairs, exported as training data for a failure detector."""
319
+ rows = state["log"].v2_labels(limit=min(max(limit, 1), 1000))
320
+ return {"count": len(rows), "labels": rows}
src/api/ml_pipeline.py CHANGED
@@ -38,6 +38,7 @@ from __future__ import annotations
38
  import re
39
  import threading
40
  import time
 
41
  from typing import Any
42
 
43
  from src.api.diagnose import diagnose
@@ -46,8 +47,9 @@ from src.config import device
46
  from src.data.arxiv_source import (
47
  ArxivUnavailable, Paper, fetch_metadata_bounded, read_papers,
48
  )
 
49
  from src.data.chunking import chunk_paper, chunks_to_corpus, read_chunks
50
- from src.data.citation import looks_like_a_question, resolve
51
  from src.retrieval.bm25 import BM25Retriever
52
  from src.retrieval.dense import DenseRetriever
53
  from src.retrieval.diversify import cap_per_paper, distinct_papers
@@ -55,6 +57,18 @@ from src.retrieval.fusion import reciprocal_rank_fusion
55
 
56
  PSEUDO_QID = "q"
57
 
 
 
 
 
 
 
 
 
 
 
 
 
58
  _QUERY_STOPWORDS = {
59
  "the", "and", "for", "with", "from", "what", "how", "why", "does", "explain",
60
  "when", "are", "was", "that", "this", "there", "which", "about", "into",
@@ -249,7 +263,7 @@ class MLPipeline:
249
  source = "corpus"
250
  ac = self.cfg["answer"]
251
  if not confident and self.cfg["citation"].get("allow_remote", True):
252
- live = self.search_arxiv_live(topic, top_k=top_k)
253
  if live:
254
  live = self._rescore_live(topic, live)
255
  floor = ac.get("live_min_score", 0.30)
@@ -261,11 +275,14 @@ class MLPipeline:
261
  "paper_id": p["paper_id"],
262
  "doc_id": p["doc_id"],
263
  "title": p["title"],
264
- "authors": [],
265
  "abstract": p["text"],
266
  "published": p["published"],
267
  "primary_category": "",
268
  "arxiv_url": p["arxiv_url"],
 
 
 
269
  "score": round(p["score"], 4),
270
  "matched_passages": 1,
271
  "matched_sections": ["abstract"],
@@ -278,7 +295,10 @@ class MLPipeline:
278
  if p["paper_id"] not in seen
279
  ]
280
  if fetched:
281
- papers = (fetched + papers)[:top_k]
 
 
 
282
  source = "arxiv_live"
283
  confident = True
284
  mean_top = sum(p["score"] for p in fetched) / len(fetched)
@@ -387,29 +407,69 @@ class MLPipeline:
387
  },
388
  }
389
 
390
- # ------------------------------------------------------- live arxiv search
391
 
392
- def search_arxiv_live(self, question: str, top_k: int = 6) -> list[dict[str, Any]]:
393
- """Search arXiv directly, for questions the local corpus cannot answer.
394
 
395
  The corpus is a sample, not coverage: ~1,400 papers per category across
396
  nine months, against the 50-150/day arXiv actually publishes. Most
397
  papers a user asks about were never ingested, and telling them "not in
398
  this corpus" is honest but useless when the paper plainly exists.
399
 
400
- Abstracts only - no PDF fetch. Bounded by `citation.live_budget_s`
401
- (default 5s) of wall-clock across both queries: this is an optional
402
- enrichment, so when arXiv is slow the answer goes out from the corpus
403
- rather than waiting. Uses the breaker shared with citation lookup, so
404
- once arXiv throttles one of them, neither keeps asking.
 
 
 
 
 
405
  """
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
406
  terms = [w for w in re.findall(r"[A-Za-z][A-Za-z0-9-]{2,}", question)
407
  if w.lower() not in _QUERY_STOPWORDS][:12]
408
  if not terms:
409
  return []
410
-
411
- budget = float(self.cfg["citation"].get("live_budget_s", 5.0))
412
- deadline = time.monotonic() + budget
413
  queries = [
414
  "all:" + " AND ".join(f'"{t}"' for t in terms[:4]), # precise first
415
  "all:" + " ".join(terms[:8]), # then looser
@@ -426,7 +486,6 @@ class MLPipeline:
426
  return [] # a failed enrichment is not a failed request
427
  if found:
428
  break
429
-
430
  return [
431
  {
432
  "chunk_id": f"live:{p.paper_id}",
@@ -438,10 +497,44 @@ class MLPipeline:
438
  "section": "abstract",
439
  "published": p.published,
440
  "text": p.abstract,
 
 
 
 
441
  }
442
  for p in found
443
  ]
444
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
445
  # ------------------------------------------------------------------- ask
446
 
447
  def answer(self, question: str, generate: bool = True) -> dict[str, Any]:
@@ -478,7 +571,7 @@ class MLPipeline:
478
  live_ms = 0.0
479
  if remote_ok and (not confident or mean_top < enrich_below):
480
  live_start = time.perf_counter()
481
- live = self.search_arxiv_live(question)
482
  live_ms = (time.perf_counter() - live_start) * 1000.0
483
  if live:
484
  # arXiv always returns *something*, so live results must clear
@@ -588,7 +681,7 @@ class MLPipeline:
588
  "chunk_id": c.chunk_id,
589
  "doc_id": c.chunk_id,
590
  "paper_id": c.paper_id,
591
- "arxiv_url": f"https://arxiv.org/abs/{c.paper_id}",
592
  "score": 1.0,
593
  "title": c.title,
594
  "section": c.section,
@@ -700,7 +793,9 @@ class MLPipeline:
700
  "authors": paper.authors[:8],
701
  "published": paper.published,
702
  "primary_category": paper.primary_category,
703
- "arxiv_url": f"https://arxiv.org/abs/{paper.paper_id}",
 
 
704
  "has_full_text": bool(paper.full_text),
705
  },
706
  "explanation": explanation,
 
38
  import re
39
  import threading
40
  import time
41
+ from concurrent.futures import ThreadPoolExecutor
42
  from typing import Any
43
 
44
  from src.api.diagnose import diagnose
 
47
  from src.data.arxiv_source import (
48
  ArxivUnavailable, Paper, fetch_metadata_bounded, read_papers,
49
  )
50
+ from src.data import openalex_source as openalex
51
  from src.data.chunking import chunk_paper, chunks_to_corpus, read_chunks
52
+ from src.data.citation import looks_like_a_question, paper_url, resolve
53
  from src.retrieval.bm25 import BM25Retriever
54
  from src.retrieval.dense import DenseRetriever
55
  from src.retrieval.diversify import cap_per_paper, distinct_papers
 
57
 
58
  PSEUDO_QID = "q"
59
 
60
+ # Live sources run side by side under one deadline; their own clients bound
61
+ # each call, so this pool only has to hold the two waits of one request.
62
+ _LIVE_POOL = ThreadPoolExecutor(max_workers=8, thread_name_prefix="live-search")
63
+
64
+
65
+ def _dedup_key(item: dict[str, Any]) -> str:
66
+ """arXiv id when known, else a normalised title: one paper, one entry."""
67
+ pid = item.get("paper_id") or ""
68
+ if pid and not pid.startswith("openalex:"):
69
+ return pid
70
+ return re.sub(r"[^a-z0-9]", "", (item.get("title") or "").lower())[:120]
71
+
72
  _QUERY_STOPWORDS = {
73
  "the", "and", "for", "with", "from", "what", "how", "why", "does", "explain",
74
  "when", "are", "was", "that", "this", "there", "which", "about", "into",
 
263
  source = "corpus"
264
  ac = self.cfg["answer"]
265
  if not confident and self.cfg["citation"].get("allow_remote", True):
266
+ live = self.search_live(topic, top_k=top_k)
267
  if live:
268
  live = self._rescore_live(topic, live)
269
  floor = ac.get("live_min_score", 0.30)
 
275
  "paper_id": p["paper_id"],
276
  "doc_id": p["doc_id"],
277
  "title": p["title"],
278
+ "authors": p.get("authors", []),
279
  "abstract": p["text"],
280
  "published": p["published"],
281
  "primary_category": "",
282
  "arxiv_url": p["arxiv_url"],
283
+ "venue": p.get("venue", ""),
284
+ "cited_by_count": p.get("cited_by_count"),
285
+ "live_source": p.get("live_source", "arxiv"),
286
  "score": round(p["score"], 4),
287
  "matched_passages": 1,
288
  "matched_sections": ["abstract"],
 
295
  if p["paper_id"] not in seen
296
  ]
297
  if fetched:
298
+ # Corpus papers only fill remaining slots if they clear
299
+ # the same bar - otherwise a 0.08 match sat under a
300
+ # result labelled confident.
301
+ papers = (fetched + [p for p in papers if p["score"] >= floor])[:top_k]
302
  source = "arxiv_live"
303
  confident = True
304
  mean_top = sum(p["score"] for p in fetched) / len(fetched)
 
407
  },
408
  }
409
 
410
+ # ------------------------------------------------------------ live search
411
 
412
+ def search_live(self, question: str, top_k: int = 6) -> list[dict[str, Any]]:
413
+ """Search arXiv and OpenAlex live, for questions the corpus cannot answer.
414
 
415
  The corpus is a sample, not coverage: ~1,400 papers per category across
416
  nine months, against the 50-150/day arXiv actually publishes. Most
417
  papers a user asks about were never ingested, and telling them "not in
418
  this corpus" is honest but useless when the paper plainly exists.
419
 
420
+ Two sources, because each misses what the other has. arXiv is the
421
+ freshest for preprints; OpenAlex adds conference and journal papers
422
+ that never had one. Both run in parallel under ONE wall-clock budget
423
+ (`citation.live_budget_s`), so adding a source never adds latency:
424
+ whatever has answered when the budget expires is used, and a source
425
+ that is down or paused costs nothing. The same paper found by both
426
+ collapses to one entry, keyed on its arXiv id or normalised title.
427
+
428
+ Abstracts only - no PDF fetch. Results are rescored by the corpus
429
+ reranker afterwards, so one confidence bar governs every source.
430
  """
431
+ sources = self.cfg["citation"].get("live_sources", ["arxiv", "openalex"])
432
+ budget = float(self.cfg["citation"].get("live_budget_s", 5.0))
433
+ deadline = time.monotonic() + budget
434
+ jobs = {}
435
+ if "arxiv" in sources:
436
+ jobs["arxiv"] = _LIVE_POOL.submit(self._live_arxiv, question, top_k, deadline)
437
+ if "openalex" in sources:
438
+ jobs["openalex"] = _LIVE_POOL.submit(self._live_openalex, question, top_k, deadline)
439
+
440
+ found: dict[str, list[dict[str, Any]]] = {}
441
+ for name, job in jobs.items():
442
+ try:
443
+ found[name] = job.result(timeout=max(deadline - time.monotonic(), 0.05))
444
+ except Exception: # noqa: BLE001 - a failed source is an empty source
445
+ found[name] = []
446
+
447
+ merged: list[dict[str, Any]] = []
448
+ seen: dict[str, dict[str, Any]] = {}
449
+ # arXiv first: on a duplicate, its record wins (fresher version), and
450
+ # OpenAlex contributes the venue and citation count it lacks.
451
+ for name in ("arxiv", "openalex"):
452
+ for item in found.get(name, []):
453
+ key = _dedup_key(item)
454
+ if key in seen:
455
+ kept = seen[key]
456
+ kept["venue"] = kept.get("venue") or item.get("venue", "")
457
+ kept["cited_by_count"] = max(kept.get("cited_by_count") or 0,
458
+ item.get("cited_by_count") or 0)
459
+ kept["live_source"] = "arxiv+openalex"
460
+ continue
461
+ seen[key] = item
462
+ merged.append(item)
463
+ return merged
464
+
465
+ # Kept for callers that predate OpenAlex.
466
+ search_arxiv_live = search_live
467
+
468
+ def _live_arxiv(self, question: str, top_k: int, deadline: float) -> list[dict[str, Any]]:
469
  terms = [w for w in re.findall(r"[A-Za-z][A-Za-z0-9-]{2,}", question)
470
  if w.lower() not in _QUERY_STOPWORDS][:12]
471
  if not terms:
472
  return []
 
 
 
473
  queries = [
474
  "all:" + " AND ".join(f'"{t}"' for t in terms[:4]), # precise first
475
  "all:" + " ".join(terms[:8]), # then looser
 
486
  return [] # a failed enrichment is not a failed request
487
  if found:
488
  break
 
489
  return [
490
  {
491
  "chunk_id": f"live:{p.paper_id}",
 
497
  "section": "abstract",
498
  "published": p.published,
499
  "text": p.abstract,
500
+ "authors": p.authors[:8],
501
+ "venue": "arXiv",
502
+ "cited_by_count": None,
503
+ "live_source": "arxiv",
504
  }
505
  for p in found
506
  ]
507
 
508
+ def _live_openalex(self, question: str, top_k: int, deadline: float) -> list[dict[str, Any]]:
509
+ remaining = deadline - time.monotonic()
510
+ if remaining <= 0.25:
511
+ return []
512
+ try:
513
+ works = openalex.search_works(question, max_results=top_k, budget_s=remaining)
514
+ except openalex.OpenAlexUnavailable:
515
+ return []
516
+ return [
517
+ {
518
+ "chunk_id": f"live:{w['paper_id']}",
519
+ "doc_id": f"live:{w['paper_id']}",
520
+ "paper_id": w["paper_id"],
521
+ # Named arxiv_url for compatibility with every client; for a
522
+ # paper with no arXiv copy it is the DOI or landing page.
523
+ "arxiv_url": w["url"],
524
+ "score": 1.0,
525
+ "title": w["title"],
526
+ "section": "abstract",
527
+ "published": w["published"],
528
+ "text": w["abstract"],
529
+ "authors": w["authors"][:8],
530
+ "venue": w["venue"],
531
+ "cited_by_count": w["cited_by_count"],
532
+ "doi": w["doi"],
533
+ "live_source": "openalex",
534
+ }
535
+ for w in works
536
+ ]
537
+
538
  # ------------------------------------------------------------------- ask
539
 
540
  def answer(self, question: str, generate: bool = True) -> dict[str, Any]:
 
571
  live_ms = 0.0
572
  if remote_ok and (not confident or mean_top < enrich_below):
573
  live_start = time.perf_counter()
574
+ live = self.search_live(question)
575
  live_ms = (time.perf_counter() - live_start) * 1000.0
576
  if live:
577
  # arXiv always returns *something*, so live results must clear
 
681
  "chunk_id": c.chunk_id,
682
  "doc_id": c.chunk_id,
683
  "paper_id": c.paper_id,
684
+ "arxiv_url": paper_url(paper),
685
  "score": 1.0,
686
  "title": c.title,
687
  "section": c.section,
 
793
  "authors": paper.authors[:8],
794
  "published": paper.published,
795
  "primary_category": paper.primary_category,
796
+ "arxiv_url": paper_url(paper),
797
+ "venue": (paper.metadata or {}).get("venue", ""),
798
+ "doi": (paper.metadata or {}).get("doi", ""),
799
  "has_full_text": bool(paper.full_text),
800
  },
801
  "explanation": explanation,
src/data/citation.py CHANGED
@@ -6,13 +6,14 @@ The use case: you are reading a paper, you hit "...following [23] (Assran et al.
6
  Accepted inputs, in the order they are tried:
7
 
8
  arXiv id 2301.08243, arXiv:2301.08243v2, https://arxiv.org/abs/2301.08243
 
9
  bracketed ref [23] M. Assran et al. "Self-Supervised Learning from Images..."
10
  bare title Self-Supervised Learning from Images with a Joint-Embedding...
11
 
12
- Resolution is local-first: the ingested corpus is checked before arXiv is
13
- called, because most of what you are reading about is already indexed. Anything
14
- not found locally is fetched on demand, so the tool is not limited to the
15
- corpus snapshot.
16
  """
17
 
18
  from __future__ import annotations
@@ -21,6 +22,7 @@ import re
21
  from dataclasses import dataclass
22
  from typing import Any
23
 
 
24
  from src.data.arxiv_source import (
25
  ArxivUnavailable, Paper, download_pdf, extract_text, fetch_metadata_bounded,
26
  )
@@ -57,12 +59,52 @@ def _safe_fetch(query: str, max_results: int) -> list[Paper]:
57
  return []
58
 
59
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
60
  @dataclass
61
  class Resolution:
62
  """What a citation string resolved to, and how."""
63
 
64
  paper: Paper | None
65
- method: str # arxiv_id | local_title | arxiv_search | unresolved
66
  query_used: str
67
  from_corpus: bool
68
  candidates: list[dict[str, str]]
@@ -77,13 +119,14 @@ QUESTION_STARTS = (
77
  # What a citation actually looks like. Anything matching one of these is worth
78
  # the cost of resolution; nothing else is.
79
  _CITATION_SHAPES = (
80
- re.compile(r"et\s+al", re.IGNORECASE), # Assran et al.
81
  re.compile(r"^\s*[\[\(]\d{1,3}[\]\)]"), # [23] or (23)
82
  re.compile(r"^\s*\d{1,3}\.\s+[A-Z]"), # 23. Author
83
  re.compile(r"\((?:19|20)\d{2}\)"), # (2023)
84
- re.compile(r",\s*(?:19|20)\d{2}"), # , 2023
85
- re.compile(r"(?:CVPR|NeurIPS|ICML|ICLR|ACL|EMNLP|AAAI|SIGGRAPH)"),
86
- re.compile(r"arxiv", re.IGNORECASE),
 
87
  re.compile(r"[\"“‘']([^\"”’']{15,})[\"”’']"), # a quoted title
88
  )
89
 
@@ -244,8 +287,13 @@ def resolve(
244
  if not raw:
245
  return Resolution(None, "unresolved", "", False, [])
246
 
 
 
 
 
 
247
  # --- 1. explicit arXiv id ---
248
- arxiv_id = extract_arxiv_id(raw)
249
  if arxiv_id:
250
  local = papers_by_id.get(arxiv_id)
251
  if local:
@@ -258,6 +306,17 @@ def resolve(
258
  _attach_text(paper, pdf_dir, text_dir, max_cache_gb)
259
  return Resolution(paper, "arxiv_id", arxiv_id, False, [])
260
 
 
 
 
 
 
 
 
 
 
 
 
261
  # --- 2. title match against the local corpus ---
262
  title_query = clean_citation(raw)
263
  local_match, candidates = find_in_corpus(title_query, papers_by_id)
@@ -281,6 +340,23 @@ def resolve(
281
  for p in found
282
  ] + candidates
283
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
284
  return Resolution(None, "unresolved", title_query, False, candidates[:5])
285
 
286
 
 
6
  Accepted inputs, in the order they are tried:
7
 
8
  arXiv id 2301.08243, arXiv:2301.08243v2, https://arxiv.org/abs/2301.08243
9
+ DOI 10.1109/CVPR.2016.90, https://doi.org/10.1109/CVPR.2016.90
10
  bracketed ref [23] M. Assran et al. "Self-Supervised Learning from Images..."
11
  bare title Self-Supervised Learning from Images with a Joint-Embedding...
12
 
13
+ Resolution is local-first: the ingested corpus is checked before any network
14
+ call, because most of what you are reading about is already indexed. Anything
15
+ not found locally is fetched on demand - arXiv first, then OpenAlex, which
16
+ covers the conference and journal papers that never had an arXiv preprint.
17
  """
18
 
19
  from __future__ import annotations
 
22
  from dataclasses import dataclass
23
  from typing import Any
24
 
25
+ from src.data import openalex_source as openalex
26
  from src.data.arxiv_source import (
27
  ArxivUnavailable, Paper, download_pdf, extract_text, fetch_metadata_bounded,
28
  )
 
59
  return []
60
 
61
 
62
+ def _paper_from_openalex(work: dict[str, Any]) -> Paper:
63
+ """An OpenAlex work in the Paper shape the rest of the pipeline expects."""
64
+ arxiv_id = work.get("arxiv_id")
65
+ return Paper(
66
+ paper_id=work["paper_id"],
67
+ version=1,
68
+ title=work["title"],
69
+ abstract=work["abstract"],
70
+ authors=work["authors"],
71
+ categories=[],
72
+ primary_category=work.get("venue", ""),
73
+ published=work.get("published", ""),
74
+ updated="",
75
+ # Only arXiv PDFs are ever fetched on the request path: a known host
76
+ # with stable, polite access. Other open-access PDFs are linked, not
77
+ # downloaded.
78
+ pdf_url=f"https://arxiv.org/pdf/{arxiv_id}" if arxiv_id else "",
79
+ metadata={
80
+ "source": "openalex",
81
+ "url": work.get("url", ""),
82
+ "venue": work.get("venue", ""),
83
+ "doi": work.get("doi") or "",
84
+ "cited_by_count": str(work.get("cited_by_count", 0)),
85
+ },
86
+ )
87
+
88
+
89
+ def _safe_openalex(fn, *args) -> Any:
90
+ try:
91
+ return fn(*args)
92
+ except openalex.OpenAlexUnavailable as exc:
93
+ print(f"[openalex] lookup skipped ({exc}); degrading")
94
+ return None
95
+
96
+
97
+ def paper_url(paper: Paper) -> str:
98
+ """Where a reader should go for this paper, whichever source it came from."""
99
+ return (paper.metadata or {}).get("url") or f"https://arxiv.org/abs/{paper.paper_id}"
100
+
101
+
102
  @dataclass
103
  class Resolution:
104
  """What a citation string resolved to, and how."""
105
 
106
  paper: Paper | None
107
+ method: str # arxiv_id | doi | local_title | arxiv_search | openalex_title | unresolved
108
  query_used: str
109
  from_corpus: bool
110
  candidates: list[dict[str, str]]
 
119
  # What a citation actually looks like. Anything matching one of these is worth
120
  # the cost of resolution; nothing else is.
121
  _CITATION_SHAPES = (
122
+ re.compile(r"\bet\s+al\b", re.IGNORECASE), # Assran et al.
123
  re.compile(r"^\s*[\[\(]\d{1,3}[\]\)]"), # [23] or (23)
124
  re.compile(r"^\s*\d{1,3}\.\s+[A-Z]"), # 23. Author
125
  re.compile(r"\((?:19|20)\d{2}\)"), # (2023)
126
+ re.compile(r",\s*(?:19|20)\d{2}\b"), # , 2023
127
+ re.compile(r"\b(?:CVPR|NeurIPS|ICML|ICLR|ACL|EMNLP|AAAI|SIGGRAPH)\b"),
128
+ re.compile(r"\barxiv\b", re.IGNORECASE),
129
+ re.compile(r"\b10\.\d{4,9}/\S+"), # a DOI
130
  re.compile(r"[\"“‘']([^\"”’']{15,})[\"”’']"), # a quoted title
131
  )
132
 
 
287
  if not raw:
288
  return Resolution(None, "unresolved", "", False, [])
289
 
290
+ # A DOI's digits can look like an arXiv id ("10.1145/3292500.3330701"
291
+ # contains "2500.33307"), so a non-arXiv DOI takes precedence over it.
292
+ doi = openalex.extract_doi(raw)
293
+ arxiv_doi = bool(doi and doi.startswith("10.48550/arxiv."))
294
+
295
  # --- 1. explicit arXiv id ---
296
+ arxiv_id = extract_arxiv_id(raw) if (not doi or arxiv_doi) else None
297
  if arxiv_id:
298
  local = papers_by_id.get(arxiv_id)
299
  if local:
 
306
  _attach_text(paper, pdf_dir, text_dir, max_cache_gb)
307
  return Resolution(paper, "arxiv_id", arxiv_id, False, [])
308
 
309
+ # --- 1b. DOI: conference and journal papers arXiv does not know ---
310
+ if doi and not arxiv_doi and allow_remote:
311
+ work = _safe_openalex(openalex.get_by_doi, doi)
312
+ if work and work.get("title"):
313
+ if work.get("arxiv_id") and work["arxiv_id"] in papers_by_id:
314
+ return Resolution(Paper(**papers_by_id[work["arxiv_id"]]), "doi", doi, True, [])
315
+ paper = _paper_from_openalex(work)
316
+ if fetch_full_text and paper.pdf_url:
317
+ _attach_text(paper, pdf_dir, text_dir, max_cache_gb)
318
+ return Resolution(paper, "doi", doi, False, [])
319
+
320
  # --- 2. title match against the local corpus ---
321
  title_query = clean_citation(raw)
322
  local_match, candidates = find_in_corpus(title_query, papers_by_id)
 
340
  for p in found
341
  ] + candidates
342
 
343
+ # --- 4. ask OpenAlex: the paper may simply never have been on arXiv ---
344
+ if allow_remote and title_query:
345
+ works = _safe_openalex(openalex.search_title, title_query, 8) or []
346
+ if works:
347
+ best = max(works, key=lambda w: _token_overlap(title_query, w["title"]))
348
+ overlap = _token_overlap(title_query, best["title"])
349
+ if overlap >= 0.6 and best.get("abstract"):
350
+ paper = _paper_from_openalex(best)
351
+ if fetch_full_text and paper.pdf_url:
352
+ _attach_text(paper, pdf_dir, text_dir, max_cache_gb)
353
+ return Resolution(paper, "openalex_title", title_query, False, candidates)
354
+ candidates = candidates + [
355
+ {"paper_id": w["paper_id"], "title": w["title"],
356
+ "score": f"{_token_overlap(title_query, w['title']):.2f}"}
357
+ for w in works[:3]
358
+ ]
359
+
360
  return Resolution(None, "unresolved", title_query, False, candidates[:5])
361
 
362
 
src/data/openalex_source.py ADDED
@@ -0,0 +1,307 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """OpenAlex: a second live source, for papers arXiv does not have.
2
+
3
+ arXiv covers preprints. A large share of what researchers cite never went
4
+ there: conference papers with no preprint, journal articles, DOI-only
5
+ references. OpenAlex indexes ~250M scholarly works under CC0, so it fills
6
+ both gaps this project had: live search beyond arXiv, and citation lookup
7
+ by DOI or by a title arXiv cannot find.
8
+
9
+ It is wrapped in the same rails as the arXiv client, because it sits on the
10
+ request path and is always an enhancement, never the only way to answer:
11
+
12
+ * a hard wall-clock budget per call - a slow API costs the user nothing
13
+ beyond that budget
14
+ * a circuit breaker - after a failure, stop asking for a while instead of
15
+ paying the timeout on every request
16
+ * an LRU cache - the same paper gets looked up repeatedly
17
+ * budget awareness - OpenAlex meters usage in dollars per day (keyless:
18
+ $0.10/day shared by everyone behind the same IP, about 100 searches; a
19
+ free key: $1/day). Every response reports what is left, so the client
20
+ stops *before* the budget is gone, leaving headroom, instead of
21
+ discovering the limit as a stream of 429s.
22
+
23
+ Set OPENALEX_API_KEY to use a free key (https://openalex.org/settings/api).
24
+ Without one everything works, within the keyless budget.
25
+ """
26
+
27
+ from __future__ import annotations
28
+
29
+ import os
30
+ import re
31
+ import threading
32
+ import time
33
+ from collections import OrderedDict
34
+ from concurrent.futures import ThreadPoolExecutor
35
+ from concurrent.futures import TimeoutError as FuturesTimeout
36
+ from typing import Any
37
+
38
+ import httpx
39
+
40
+ BASE_URL = "https://api.openalex.org"
41
+
42
+ # OpenAlex's own topic taxonomy: field 17 is Computer Science. Without this,
43
+ # an off-topic question ("sourdough fermentation") is answered from food
44
+ # science, and the app's honest "this covers AI/ML research only" decline
45
+ # turns into a confident answer from the wrong literature.
46
+ #
47
+ # Restricted to papers, in English: unfiltered, a PageRank query ranked a
48
+ # Spanish thesis and a university course page above the PageRank paper itself.
49
+ # OpenAlex types conference papers as "article", so venues are kept.
50
+ DEFAULT_FILTER = ("primary_topic.field.id:fields/17,has_abstract:true,"
51
+ "type:article|preprint|review,language:en")
52
+
53
+ SELECT = ",".join((
54
+ "id", "doi", "title", "publication_year", "publication_date", "type",
55
+ "abstract_inverted_index", "authorships", "primary_location", "locations",
56
+ "cited_by_count", "open_access",
57
+ ))
58
+
59
+ REQUEST_BUDGET_S = 5.0
60
+ BREAKER_COOLDOWN_S = 120.0
61
+ # Stop with this much of the daily budget left. Keyless usage is shared with
62
+ # everyone on the host's IP, so running it to zero would also starve them.
63
+ BUDGET_RESERVE_USD = 0.01
64
+ MAX_BUDGET_PAUSE_S = 3600.0
65
+
66
+ _POOL = ThreadPoolExecutor(max_workers=4, thread_name_prefix="openalex")
67
+ _LOCK = threading.Lock()
68
+ _STATE: dict[str, Any] = {
69
+ "open_until": 0.0,
70
+ "reason": "",
71
+ "remaining_usd": None,
72
+ "limit_usd": None,
73
+ "calls": 0,
74
+ "failures": 0,
75
+ }
76
+ _CACHE: OrderedDict[str, Any] = OrderedDict()
77
+ _CACHE_MAX = 512
78
+
79
+ _ARXIV_IN_URL = re.compile(r"arxiv\.org/(?:abs|pdf)/(?P<id>\d{4}\.\d{4,5})", re.IGNORECASE)
80
+ _ARXIV_DOI = re.compile(r"10\.48550/arxiv\.(?P<id>\d{4}\.\d{4,5})", re.IGNORECASE)
81
+ DOI_RE = re.compile(r"\b(?P<doi>10\.\d{4,9}/[^\s\"'<>]+)", re.IGNORECASE)
82
+
83
+
84
+ class OpenAlexUnavailable(RuntimeError):
85
+ """OpenAlex did not answer in time, refused, or is paused."""
86
+
87
+
88
+ # ---------------------------------------------------------------- state ----
89
+ def breaker_open() -> bool:
90
+ with _LOCK:
91
+ return time.monotonic() < _STATE["open_until"]
92
+
93
+
94
+ def _pause(seconds: float, reason: str) -> None:
95
+ with _LOCK:
96
+ _STATE["open_until"] = time.monotonic() + seconds
97
+ _STATE["reason"] = reason
98
+ print(f"[openalex] {reason} - pausing live lookups for {seconds:.0f}s", flush=True)
99
+
100
+
101
+ def status() -> dict[str, Any]:
102
+ """For /health: is the source usable right now, and how much budget is left."""
103
+ with _LOCK:
104
+ paused_for = max(0.0, _STATE["open_until"] - time.monotonic())
105
+ return {
106
+ "available": paused_for == 0.0,
107
+ "paused_for_s": round(paused_for, 1),
108
+ "reason": _STATE["reason"] if paused_for else "",
109
+ "keyed": bool(os.environ.get("OPENALEX_API_KEY")),
110
+ "remaining_usd": _STATE["remaining_usd"],
111
+ "limit_usd": _STATE["limit_usd"],
112
+ "calls": _STATE["calls"],
113
+ "failures": _STATE["failures"],
114
+ }
115
+
116
+
117
+ def _record_budget(headers: httpx.Headers) -> None:
118
+ """Read the metering headers and pause ahead of exhaustion."""
119
+ try:
120
+ remaining = float(headers.get("X-RateLimit-Remaining-USD", "nan"))
121
+ limit = float(headers.get("X-RateLimit-Limit-USD", "nan"))
122
+ reset_s = float(headers.get("X-RateLimit-Reset", "3600"))
123
+ except ValueError:
124
+ return
125
+ if remaining != remaining: # NaN: header absent
126
+ return
127
+ with _LOCK:
128
+ _STATE["remaining_usd"] = round(remaining, 4)
129
+ _STATE["limit_usd"] = None if limit != limit else round(limit, 4)
130
+ if remaining < BUDGET_RESERVE_USD:
131
+ _pause(min(reset_s, MAX_BUDGET_PAUSE_S),
132
+ f"daily budget nearly spent (${remaining:.3f} left)")
133
+
134
+
135
+ # -------------------------------------------------------------- parsing ----
136
+ def reconstruct_abstract(inverted: dict[str, list[int]] | None) -> str:
137
+ """OpenAlex ships abstracts as {word: [positions]} for licensing reasons."""
138
+ if not inverted:
139
+ return ""
140
+ slots: dict[int, str] = {}
141
+ for word, positions in inverted.items():
142
+ for pos in positions:
143
+ slots[pos] = word
144
+ return " ".join(slots[i] for i in sorted(slots))
145
+
146
+
147
+ def _arxiv_id(work: dict[str, Any]) -> str | None:
148
+ """The arXiv id of a work, if any copy of it lives on arXiv."""
149
+ doi = work.get("doi") or ""
150
+ m = _ARXIV_DOI.search(doi)
151
+ if m:
152
+ return m.group("id")
153
+ for loc in work.get("locations") or []:
154
+ for key in ("landing_page_url", "pdf_url"):
155
+ m = _ARXIV_IN_URL.search(loc.get(key) or "")
156
+ if m:
157
+ return m.group("id")
158
+ return None
159
+
160
+
161
+ def normalise_doi(doi: str | None) -> str | None:
162
+ if not doi:
163
+ return None
164
+ doi = doi.strip().rstrip(".,;)")
165
+ doi = re.sub(r"^(?:https?://(?:dx\.)?doi\.org/|doi:)", "", doi, flags=re.IGNORECASE)
166
+ return doi.lower() or None
167
+
168
+
169
+ def parse_work(work: dict[str, Any]) -> dict[str, Any]:
170
+ """One OpenAlex work -> the fields the pipeline uses."""
171
+ openalex_id = (work.get("id") or "").rsplit("/", 1)[-1]
172
+ arxiv_id = _arxiv_id(work)
173
+ doi = normalise_doi(work.get("doi"))
174
+ primary = work.get("primary_location") or {}
175
+ # Some venues arrive as "Name/Name/Proceedings - Name"; the first is enough.
176
+ venue = ((primary.get("source") or {}).get("display_name") or "").split("/")[0].strip()
177
+ if arxiv_id:
178
+ url = f"https://arxiv.org/abs/{arxiv_id}"
179
+ elif doi:
180
+ url = f"https://doi.org/{doi}"
181
+ else:
182
+ url = primary.get("landing_page_url") or work.get("id") or ""
183
+ authors = [
184
+ (a.get("author") or {}).get("display_name", "")
185
+ for a in (work.get("authorships") or [])[:20]
186
+ ]
187
+ return {
188
+ "openalex_id": openalex_id,
189
+ # arXiv id when there is one, so the same paper found through arXiv,
190
+ # OpenAlex and the corpus collapses to a single entry.
191
+ "paper_id": arxiv_id or f"openalex:{openalex_id}",
192
+ "arxiv_id": arxiv_id,
193
+ "doi": doi,
194
+ "title": re.sub(r"\s+", " ", work.get("title") or "").strip(),
195
+ "abstract": re.sub(r"\s+", " ", reconstruct_abstract(work.get("abstract_inverted_index"))).strip(),
196
+ "authors": [a for a in authors if a],
197
+ "published": work.get("publication_date") or str(work.get("publication_year") or ""),
198
+ "year": work.get("publication_year"),
199
+ "venue": venue,
200
+ "type": work.get("type") or "",
201
+ "cited_by_count": int(work.get("cited_by_count") or 0),
202
+ "url": url,
203
+ "pdf_url": (work.get("open_access") or {}).get("oa_url") or "",
204
+ }
205
+
206
+
207
+ # ------------------------------------------------------------------ http ----
208
+ def _get(path: str, params: dict[str, Any]) -> dict[str, Any]:
209
+ headers = {"User-Agent": "ResearchOS (https://github.com/Kagzyyy/ResearchOS)"}
210
+ key = os.environ.get("OPENALEX_API_KEY")
211
+ if key:
212
+ headers["Authorization"] = f"Bearer {key}"
213
+ with _LOCK:
214
+ _STATE["calls"] += 1
215
+ r = httpx.get(f"{BASE_URL}{path}", params=params, headers=headers, timeout=10.0)
216
+ _record_budget(r.headers)
217
+ if r.status_code == 404:
218
+ return {}
219
+ if r.status_code == 429:
220
+ retry = float(r.headers.get("Retry-After", "60") or 60)
221
+ _pause(min(max(retry, 30.0), MAX_BUDGET_PAUSE_S), "rate limited (429)")
222
+ raise OpenAlexUnavailable("rate limited")
223
+ r.raise_for_status()
224
+ return r.json()
225
+
226
+
227
+ def _bounded(key: str, fn, budget_s: float):
228
+ """Run fn() inside the budget, through the cache and the breaker."""
229
+ with _LOCK:
230
+ if key in _CACHE:
231
+ _CACHE.move_to_end(key)
232
+ return _CACHE[key]
233
+ if breaker_open():
234
+ raise OpenAlexUnavailable("paused after a recent failure")
235
+ future = _POOL.submit(fn)
236
+ try:
237
+ result = future.result(timeout=max(budget_s, 0.1))
238
+ except FuturesTimeout:
239
+ with _LOCK:
240
+ _STATE["failures"] += 1
241
+ _pause(BREAKER_COOLDOWN_S, f"no answer within {budget_s:.1f}s")
242
+ raise OpenAlexUnavailable(f"timed out after {budget_s:.1f}s") from None
243
+ except OpenAlexUnavailable:
244
+ with _LOCK:
245
+ _STATE["failures"] += 1
246
+ raise
247
+ except Exception as exc: # noqa: BLE001
248
+ with _LOCK:
249
+ _STATE["failures"] += 1
250
+ _pause(BREAKER_COOLDOWN_S, f"{type(exc).__name__}: {str(exc)[:80]}")
251
+ raise OpenAlexUnavailable(str(exc)) from exc
252
+ with _LOCK:
253
+ _CACHE[key] = result
254
+ if len(_CACHE) > _CACHE_MAX:
255
+ _CACHE.popitem(last=False)
256
+ return result
257
+
258
+
259
+ # ------------------------------------------------------------------- api ----
260
+ def search_works(query: str, max_results: int = 8, budget_s: float = REQUEST_BUDGET_S,
261
+ filter_: str = DEFAULT_FILTER) -> list[dict[str, Any]]:
262
+ """Relevance-ranked search over titles, abstracts and full text."""
263
+ query = (query or "").strip()
264
+ if not query:
265
+ return []
266
+ params = {"search": query[:300], "per_page": max(1, min(max_results, 25)),
267
+ "select": SELECT, "filter": filter_}
268
+
269
+ def run() -> list[dict[str, Any]]:
270
+ works = _get("/works", params).get("results") or []
271
+ return [w for w in (parse_work(x) for x in works) if w["title"] and w["abstract"]]
272
+
273
+ return _bounded(f"search|{filter_}|{max_results}|{query.lower()}", run, budget_s)
274
+
275
+
276
+ def search_title(title: str, max_results: int = 8,
277
+ budget_s: float = REQUEST_BUDGET_S) -> list[dict[str, Any]]:
278
+ """Title-only search, for citation resolution. No field filter: a cited
279
+ paper can be from any field, and the title match itself is the check."""
280
+ title = re.sub(r"[,:;()\[\]\"]", " ", title or "").strip()
281
+ if not title:
282
+ return []
283
+ params = {"filter": f"title.search:{title[:250]}",
284
+ "per_page": max(1, min(max_results, 25)), "select": SELECT}
285
+
286
+ def run() -> list[dict[str, Any]]:
287
+ works = _get("/works", params).get("results") or []
288
+ return [parse_work(x) for x in works if x.get("title")]
289
+
290
+ return _bounded(f"title|{max_results}|{title.lower()}", run, budget_s)
291
+
292
+
293
+ def get_by_doi(doi: str, budget_s: float = REQUEST_BUDGET_S) -> dict[str, Any] | None:
294
+ doi = normalise_doi(doi)
295
+ if not doi:
296
+ return None
297
+
298
+ def run() -> dict[str, Any] | None:
299
+ work = _get(f"/works/doi:{doi}", {"select": SELECT})
300
+ return parse_work(work) if work.get("id") else None
301
+
302
+ return _bounded(f"doi|{doi}", run, budget_s)
303
+
304
+
305
+ def extract_doi(text: str) -> str | None:
306
+ m = DOI_RE.search(text or "")
307
+ return normalise_doi(m.group("doi")) if m else None