Spaces:
Running on Zero
Running on Zero
File size: 13,426 Bytes
a3c7ac6 2299777 a3c7ac6 2299777 a3c7ac6 2299777 a3c7ac6 2299777 a3c7ac6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 | """OpenAlex: a second live source, for papers arXiv does not have.
arXiv covers preprints. A large share of what researchers cite never went
there: conference papers with no preprint, journal articles, DOI-only
references. OpenAlex indexes ~250M scholarly works under CC0, so it fills
both gaps this project had: live search beyond arXiv, and citation lookup
by DOI or by a title arXiv cannot find.
It is wrapped in the same rails as the arXiv client, because it sits on the
request path and is always an enhancement, never the only way to answer:
* a hard wall-clock budget per call - a slow API costs the user nothing
beyond that budget
* a circuit breaker - after a failure, stop asking for a while instead of
paying the timeout on every request
* an LRU cache - the same paper gets looked up repeatedly
* budget awareness - OpenAlex meters usage in dollars per day (keyless:
$0.10/day shared by everyone behind the same IP, about 100 searches; a
free key: $1/day). Every response reports what is left, so the client
stops *before* the budget is gone, leaving headroom, instead of
discovering the limit as a stream of 429s.
Set OPENALEX_API_KEY to use a free key (https://openalex.org/settings/api).
Without one everything works, within the keyless budget.
"""
from __future__ import annotations
import os
import re
import threading
import time
from collections import OrderedDict
from concurrent.futures import ThreadPoolExecutor
from concurrent.futures import TimeoutError as FuturesTimeout
from typing import Any
import httpx
BASE_URL = "https://api.openalex.org"
# OpenAlex's own topic taxonomy: field 17 is Computer Science. Without this,
# an off-topic question ("sourdough fermentation") is answered from food
# science, and the app's honest "this covers AI/ML research only" decline
# turns into a confident answer from the wrong literature.
#
# Restricted to papers, in English: unfiltered, a PageRank query ranked a
# Spanish thesis and a university course page above the PageRank paper itself.
# OpenAlex types conference papers as "article", so venues are kept.
DEFAULT_FILTER = ("primary_topic.field.id:fields/17,has_abstract:true,"
"type:article|preprint|review,language:en")
SELECT = ",".join((
"id", "doi", "title", "publication_year", "publication_date", "type",
"abstract_inverted_index", "authorships", "primary_location", "locations",
"cited_by_count", "open_access",
))
REQUEST_BUDGET_S = 5.0
BREAKER_COOLDOWN_S = 120.0
# Stop with this much of the daily budget left. Keyless usage is shared with
# everyone on the host's IP, so running it to zero would also starve them.
BUDGET_RESERVE_USD = 0.01
MAX_BUDGET_PAUSE_S = 3600.0
_POOL = ThreadPoolExecutor(max_workers=4, thread_name_prefix="openalex")
_LOCK = threading.Lock()
_STATE: dict[str, Any] = {
"open_until": 0.0,
"reason": "",
"remaining_usd": None,
"limit_usd": None,
"calls": 0,
"failures": 0,
}
_CACHE: OrderedDict[str, Any] = OrderedDict()
_CACHE_MAX = 512
_ARXIV_IN_URL = re.compile(r"arxiv\.org/(?:abs|pdf)/(?P<id>\d{4}\.\d{4,5})", re.IGNORECASE)
_ARXIV_DOI = re.compile(r"10\.48550/arxiv\.(?P<id>\d{4}\.\d{4,5})", re.IGNORECASE)
DOI_RE = re.compile(r"\b(?P<doi>10\.\d{4,9}/[^\s\"'<>]+)", re.IGNORECASE)
class OpenAlexUnavailable(RuntimeError):
"""OpenAlex did not answer in time, refused, or is paused."""
# Characters OpenAlex's search syntax gives meaning to. User text is prose, not
# a query language: "What is the bias-variance tradeoff?" contains a "?", which
# OpenAlex reads as a wildcard and rejects with a 400 - so every question-form
# query failed. Quotes, parentheses and operators are dropped the same way.
_SYNTAX = re.compile(r"[?*~^|<>{}\\\[\]()\"]")
def clean_query(text: str, drop: str = "") -> str:
text = _SYNTAX.sub(" ", text or "")
if drop:
text = re.sub(f"[{re.escape(drop)}]", " ", text)
return re.sub(r"\s+", " ", text).strip()
# ---------------------------------------------------------------- state ----
def breaker_open() -> bool:
with _LOCK:
return time.monotonic() < _STATE["open_until"]
def _pause(seconds: float, reason: str) -> None:
with _LOCK:
_STATE["open_until"] = time.monotonic() + seconds
_STATE["reason"] = reason
print(f"[openalex] {reason} - pausing live lookups for {seconds:.0f}s", flush=True)
def status() -> dict[str, Any]:
"""For /health: is the source usable right now, and how much budget is left."""
with _LOCK:
paused_for = max(0.0, _STATE["open_until"] - time.monotonic())
return {
"available": paused_for == 0.0,
"paused_for_s": round(paused_for, 1),
"reason": _STATE["reason"] if paused_for else "",
"keyed": bool(os.environ.get("OPENALEX_API_KEY")),
"remaining_usd": _STATE["remaining_usd"],
"limit_usd": _STATE["limit_usd"],
"calls": _STATE["calls"],
"failures": _STATE["failures"],
}
def _record_budget(headers: httpx.Headers) -> None:
"""Read the metering headers and pause ahead of exhaustion."""
try:
remaining = float(headers.get("X-RateLimit-Remaining-USD", "nan"))
limit = float(headers.get("X-RateLimit-Limit-USD", "nan"))
reset_s = float(headers.get("X-RateLimit-Reset", "3600"))
except ValueError:
return
if remaining != remaining: # NaN: header absent
return
with _LOCK:
_STATE["remaining_usd"] = round(remaining, 4)
_STATE["limit_usd"] = None if limit != limit else round(limit, 4)
if remaining < BUDGET_RESERVE_USD:
_pause(min(reset_s, MAX_BUDGET_PAUSE_S),
f"daily budget nearly spent (${remaining:.3f} left)")
# -------------------------------------------------------------- parsing ----
def reconstruct_abstract(inverted: dict[str, list[int]] | None) -> str:
"""OpenAlex ships abstracts as {word: [positions]} for licensing reasons."""
if not inverted:
return ""
slots: dict[int, str] = {}
for word, positions in inverted.items():
for pos in positions:
slots[pos] = word
return " ".join(slots[i] for i in sorted(slots))
def _arxiv_id(work: dict[str, Any]) -> str | None:
"""The arXiv id of a work, if any copy of it lives on arXiv."""
doi = work.get("doi") or ""
m = _ARXIV_DOI.search(doi)
if m:
return m.group("id")
for loc in work.get("locations") or []:
for key in ("landing_page_url", "pdf_url"):
m = _ARXIV_IN_URL.search(loc.get(key) or "")
if m:
return m.group("id")
return None
def normalise_doi(doi: str | None) -> str | None:
if not doi:
return None
doi = doi.strip().rstrip(".,;)")
doi = re.sub(r"^(?:https?://(?:dx\.)?doi\.org/|doi:)", "", doi, flags=re.IGNORECASE)
return doi.lower() or None
def parse_work(work: dict[str, Any]) -> dict[str, Any]:
"""One OpenAlex work -> the fields the pipeline uses."""
openalex_id = (work.get("id") or "").rsplit("/", 1)[-1]
arxiv_id = _arxiv_id(work)
doi = normalise_doi(work.get("doi"))
primary = work.get("primary_location") or {}
# Some venues arrive as "Name/Name/Proceedings - Name"; the first is enough.
venue = ((primary.get("source") or {}).get("display_name") or "").split("/")[0].strip()
if arxiv_id:
url = f"https://arxiv.org/abs/{arxiv_id}"
elif doi:
url = f"https://doi.org/{doi}"
else:
url = primary.get("landing_page_url") or work.get("id") or ""
authors = [
(a.get("author") or {}).get("display_name", "")
for a in (work.get("authorships") or [])[:20]
]
return {
"openalex_id": openalex_id,
# arXiv id when there is one, so the same paper found through arXiv,
# OpenAlex and the corpus collapses to a single entry.
"paper_id": arxiv_id or f"openalex:{openalex_id}",
"arxiv_id": arxiv_id,
"doi": doi,
"title": re.sub(r"\s+", " ", work.get("title") or "").strip(),
"abstract": re.sub(r"\s+", " ", reconstruct_abstract(work.get("abstract_inverted_index"))).strip(),
"authors": [a for a in authors if a],
"published": work.get("publication_date") or str(work.get("publication_year") or ""),
"year": work.get("publication_year"),
"venue": venue,
"type": work.get("type") or "",
"cited_by_count": int(work.get("cited_by_count") or 0),
"url": url,
"pdf_url": (work.get("open_access") or {}).get("oa_url") or "",
}
# ------------------------------------------------------------------ http ----
def _get(path: str, params: dict[str, Any]) -> dict[str, Any]:
headers = {"User-Agent": "ResearchOS (https://github.com/Kagzyyy/ResearchOS)"}
key = os.environ.get("OPENALEX_API_KEY")
if key:
headers["Authorization"] = f"Bearer {key}"
with _LOCK:
_STATE["calls"] += 1
r = httpx.get(f"{BASE_URL}{path}", params=params, headers=headers, timeout=10.0)
_record_budget(r.headers)
if r.status_code == 404:
return {}
if r.status_code == 429:
retry = float(r.headers.get("Retry-After", "60") or 60)
_pause(min(max(retry, 30.0), MAX_BUDGET_PAUSE_S), "rate limited (429)")
raise OpenAlexUnavailable("rate limited")
if 400 <= r.status_code < 500 and r.status_code != 403:
# This query was rejected; OpenAlex is fine. Raised as "unavailable" so
# callers degrade, but WITHOUT pausing the source - one malformed
# query must not switch off live search for everyone for two minutes.
raise OpenAlexUnavailable(f"query rejected ({r.status_code})")
r.raise_for_status()
return r.json()
def _bounded(key: str, fn, budget_s: float):
"""Run fn() inside the budget, through the cache and the breaker."""
with _LOCK:
if key in _CACHE:
_CACHE.move_to_end(key)
return _CACHE[key]
if breaker_open():
raise OpenAlexUnavailable("paused after a recent failure")
future = _POOL.submit(fn)
try:
result = future.result(timeout=max(budget_s, 0.1))
except FuturesTimeout:
with _LOCK:
_STATE["failures"] += 1
_pause(BREAKER_COOLDOWN_S, f"no answer within {budget_s:.1f}s")
raise OpenAlexUnavailable(f"timed out after {budget_s:.1f}s") from None
except OpenAlexUnavailable:
with _LOCK:
_STATE["failures"] += 1
raise
except Exception as exc: # noqa: BLE001
with _LOCK:
_STATE["failures"] += 1
_pause(BREAKER_COOLDOWN_S, f"{type(exc).__name__}: {str(exc)[:80]}")
raise OpenAlexUnavailable(str(exc)) from exc
with _LOCK:
_CACHE[key] = result
if len(_CACHE) > _CACHE_MAX:
_CACHE.popitem(last=False)
return result
# ------------------------------------------------------------------- api ----
def search_works(query: str, max_results: int = 8, budget_s: float = REQUEST_BUDGET_S,
filter_: str = DEFAULT_FILTER) -> list[dict[str, Any]]:
"""Relevance-ranked search over titles, abstracts and full text."""
query = clean_query(query)
if not query:
return []
params = {"search": query[:300], "per_page": max(1, min(max_results, 25)),
"select": SELECT, "filter": filter_}
def run() -> list[dict[str, Any]]:
works = _get("/works", params).get("results") or []
return [w for w in (parse_work(x) for x in works) if w["title"] and w["abstract"]]
return _bounded(f"search|{filter_}|{max_results}|{query.lower()}", run, budget_s)
def search_title(title: str, max_results: int = 8,
budget_s: float = REQUEST_BUDGET_S) -> list[dict[str, Any]]:
"""Title-only search, for citation resolution. No field filter: a cited
paper can be from any field, and the title match itself is the check."""
# In a filter value "," separates filters and ":" separates a field from
# its value, so both are removed along with the search-syntax characters.
title = clean_query(title, drop=",:;")
if not title:
return []
params = {"filter": f"title.search:{title[:250]}",
"per_page": max(1, min(max_results, 25)), "select": SELECT}
def run() -> list[dict[str, Any]]:
works = _get("/works", params).get("results") or []
return [parse_work(x) for x in works if x.get("title")]
return _bounded(f"title|{max_results}|{title.lower()}", run, budget_s)
def get_by_doi(doi: str, budget_s: float = REQUEST_BUDGET_S) -> dict[str, Any] | None:
doi = normalise_doi(doi)
if not doi:
return None
def run() -> dict[str, Any] | None:
work = _get(f"/works/doi:{doi}", {"select": SELECT})
return parse_work(work) if work.get("id") else None
return _bounded(f"doi|{doi}", run, budget_s)
def extract_doi(text: str) -> str | None:
m = DOI_RE.search(text or "")
return normalise_doi(m.group("doi")) if m else None
|