ResearchOS / src /data /openalex_source.py
Kagzy's picture
answer any AI question; adaptive rerank + query cache; corpus loader with fallback; OpenAlex query fixes
2299777 verified
Raw History Blame Contribute Delete
13.4 kB
"""OpenAlex: a second live source, for papers arXiv does not have.
arXiv covers preprints. A large share of what researchers cite never went
there: conference papers with no preprint, journal articles, DOI-only
references. OpenAlex indexes ~250M scholarly works under CC0, so it fills
both gaps this project had: live search beyond arXiv, and citation lookup
by DOI or by a title arXiv cannot find.
It is wrapped in the same rails as the arXiv client, because it sits on the
request path and is always an enhancement, never the only way to answer:
* a hard wall-clock budget per call - a slow API costs the user nothing
beyond that budget
* a circuit breaker - after a failure, stop asking for a while instead of
paying the timeout on every request
* an LRU cache - the same paper gets looked up repeatedly
* budget awareness - OpenAlex meters usage in dollars per day (keyless:
$0.10/day shared by everyone behind the same IP, about 100 searches; a
free key: $1/day). Every response reports what is left, so the client
stops *before* the budget is gone, leaving headroom, instead of
discovering the limit as a stream of 429s.
Set OPENALEX_API_KEY to use a free key (https://openalex.org/settings/api).
Without one everything works, within the keyless budget.
"""
from __future__ import annotations
import os
import re
import threading
import time
from collections import OrderedDict
from concurrent.futures import ThreadPoolExecutor
from concurrent.futures import TimeoutError as FuturesTimeout
from typing import Any
import httpx
BASE_URL = "https://api.openalex.org"
# OpenAlex's own topic taxonomy: field 17 is Computer Science. Without this,
# an off-topic question ("sourdough fermentation") is answered from food
# science, and the app's honest "this covers AI/ML research only" decline
# turns into a confident answer from the wrong literature.
#
# Restricted to papers, in English: unfiltered, a PageRank query ranked a
# Spanish thesis and a university course page above the PageRank paper itself.
# OpenAlex types conference papers as "article", so venues are kept.
DEFAULT_FILTER = ("primary_topic.field.id:fields/17,has_abstract:true,"
"type:article|preprint|review,language:en")
SELECT = ",".join((
"id", "doi", "title", "publication_year", "publication_date", "type",
"abstract_inverted_index", "authorships", "primary_location", "locations",
"cited_by_count", "open_access",
))
REQUEST_BUDGET_S = 5.0
BREAKER_COOLDOWN_S = 120.0
# Stop with this much of the daily budget left. Keyless usage is shared with
# everyone on the host's IP, so running it to zero would also starve them.
BUDGET_RESERVE_USD = 0.01
MAX_BUDGET_PAUSE_S = 3600.0
_POOL = ThreadPoolExecutor(max_workers=4, thread_name_prefix="openalex")
_LOCK = threading.Lock()
_STATE: dict[str, Any] = {
"open_until": 0.0,
"reason": "",
"remaining_usd": None,
"limit_usd": None,
"calls": 0,
"failures": 0,
}
_CACHE: OrderedDict[str, Any] = OrderedDict()
_CACHE_MAX = 512
_ARXIV_IN_URL = re.compile(r"arxiv\.org/(?:abs|pdf)/(?P<id>\d{4}\.\d{4,5})", re.IGNORECASE)
_ARXIV_DOI = re.compile(r"10\.48550/arxiv\.(?P<id>\d{4}\.\d{4,5})", re.IGNORECASE)
DOI_RE = re.compile(r"\b(?P<doi>10\.\d{4,9}/[^\s\"'<>]+)", re.IGNORECASE)
class OpenAlexUnavailable(RuntimeError):
"""OpenAlex did not answer in time, refused, or is paused."""
# Characters OpenAlex's search syntax gives meaning to. User text is prose, not
# a query language: "What is the bias-variance tradeoff?" contains a "?", which
# OpenAlex reads as a wildcard and rejects with a 400 - so every question-form
# query failed. Quotes, parentheses and operators are dropped the same way.
_SYNTAX = re.compile(r"[?*~^|<>{}\\\[\]()\"]")
def clean_query(text: str, drop: str = "") -> str:
text = _SYNTAX.sub(" ", text or "")
if drop:
text = re.sub(f"[{re.escape(drop)}]", " ", text)
return re.sub(r"\s+", " ", text).strip()
# ---------------------------------------------------------------- state ----
def breaker_open() -> bool:
with _LOCK:
return time.monotonic() < _STATE["open_until"]
def _pause(seconds: float, reason: str) -> None:
with _LOCK:
_STATE["open_until"] = time.monotonic() + seconds
_STATE["reason"] = reason
print(f"[openalex] {reason} - pausing live lookups for {seconds:.0f}s", flush=True)
def status() -> dict[str, Any]:
"""For /health: is the source usable right now, and how much budget is left."""
with _LOCK:
paused_for = max(0.0, _STATE["open_until"] - time.monotonic())
return {
"available": paused_for == 0.0,
"paused_for_s": round(paused_for, 1),
"reason": _STATE["reason"] if paused_for else "",
"keyed": bool(os.environ.get("OPENALEX_API_KEY")),
"remaining_usd": _STATE["remaining_usd"],
"limit_usd": _STATE["limit_usd"],
"calls": _STATE["calls"],
"failures": _STATE["failures"],
}
def _record_budget(headers: httpx.Headers) -> None:
"""Read the metering headers and pause ahead of exhaustion."""
try:
remaining = float(headers.get("X-RateLimit-Remaining-USD", "nan"))
limit = float(headers.get("X-RateLimit-Limit-USD", "nan"))
reset_s = float(headers.get("X-RateLimit-Reset", "3600"))
except ValueError:
return
if remaining != remaining: # NaN: header absent
return
with _LOCK:
_STATE["remaining_usd"] = round(remaining, 4)
_STATE["limit_usd"] = None if limit != limit else round(limit, 4)
if remaining < BUDGET_RESERVE_USD:
_pause(min(reset_s, MAX_BUDGET_PAUSE_S),
f"daily budget nearly spent (${remaining:.3f} left)")
# -------------------------------------------------------------- parsing ----
def reconstruct_abstract(inverted: dict[str, list[int]] | None) -> str:
"""OpenAlex ships abstracts as {word: [positions]} for licensing reasons."""
if not inverted:
return ""
slots: dict[int, str] = {}
for word, positions in inverted.items():
for pos in positions:
slots[pos] = word
return " ".join(slots[i] for i in sorted(slots))
def _arxiv_id(work: dict[str, Any]) -> str | None:
"""The arXiv id of a work, if any copy of it lives on arXiv."""
doi = work.get("doi") or ""
m = _ARXIV_DOI.search(doi)
if m:
return m.group("id")
for loc in work.get("locations") or []:
for key in ("landing_page_url", "pdf_url"):
m = _ARXIV_IN_URL.search(loc.get(key) or "")
if m:
return m.group("id")
return None
def normalise_doi(doi: str | None) -> str | None:
if not doi:
return None
doi = doi.strip().rstrip(".,;)")
doi = re.sub(r"^(?:https?://(?:dx\.)?doi\.org/|doi:)", "", doi, flags=re.IGNORECASE)
return doi.lower() or None
def parse_work(work: dict[str, Any]) -> dict[str, Any]:
"""One OpenAlex work -> the fields the pipeline uses."""
openalex_id = (work.get("id") or "").rsplit("/", 1)[-1]
arxiv_id = _arxiv_id(work)
doi = normalise_doi(work.get("doi"))
primary = work.get("primary_location") or {}
# Some venues arrive as "Name/Name/Proceedings - Name"; the first is enough.
venue = ((primary.get("source") or {}).get("display_name") or "").split("/")[0].strip()
if arxiv_id:
url = f"https://arxiv.org/abs/{arxiv_id}"
elif doi:
url = f"https://doi.org/{doi}"
else:
url = primary.get("landing_page_url") or work.get("id") or ""
authors = [
(a.get("author") or {}).get("display_name", "")
for a in (work.get("authorships") or [])[:20]
]
return {
"openalex_id": openalex_id,
# arXiv id when there is one, so the same paper found through arXiv,
# OpenAlex and the corpus collapses to a single entry.
"paper_id": arxiv_id or f"openalex:{openalex_id}",
"arxiv_id": arxiv_id,
"doi": doi,
"title": re.sub(r"\s+", " ", work.get("title") or "").strip(),
"abstract": re.sub(r"\s+", " ", reconstruct_abstract(work.get("abstract_inverted_index"))).strip(),
"authors": [a for a in authors if a],
"published": work.get("publication_date") or str(work.get("publication_year") or ""),
"year": work.get("publication_year"),
"venue": venue,
"type": work.get("type") or "",
"cited_by_count": int(work.get("cited_by_count") or 0),
"url": url,
"pdf_url": (work.get("open_access") or {}).get("oa_url") or "",
}
# ------------------------------------------------------------------ http ----
def _get(path: str, params: dict[str, Any]) -> dict[str, Any]:
headers = {"User-Agent": "ResearchOS (https://github.com/Kagzyyy/ResearchOS)"}
key = os.environ.get("OPENALEX_API_KEY")
if key:
headers["Authorization"] = f"Bearer {key}"
with _LOCK:
_STATE["calls"] += 1
r = httpx.get(f"{BASE_URL}{path}", params=params, headers=headers, timeout=10.0)
_record_budget(r.headers)
if r.status_code == 404:
return {}
if r.status_code == 429:
retry = float(r.headers.get("Retry-After", "60") or 60)
_pause(min(max(retry, 30.0), MAX_BUDGET_PAUSE_S), "rate limited (429)")
raise OpenAlexUnavailable("rate limited")
if 400 <= r.status_code < 500 and r.status_code != 403:
# This query was rejected; OpenAlex is fine. Raised as "unavailable" so
# callers degrade, but WITHOUT pausing the source - one malformed
# query must not switch off live search for everyone for two minutes.
raise OpenAlexUnavailable(f"query rejected ({r.status_code})")
r.raise_for_status()
return r.json()
def _bounded(key: str, fn, budget_s: float):
"""Run fn() inside the budget, through the cache and the breaker."""
with _LOCK:
if key in _CACHE:
_CACHE.move_to_end(key)
return _CACHE[key]
if breaker_open():
raise OpenAlexUnavailable("paused after a recent failure")
future = _POOL.submit(fn)
try:
result = future.result(timeout=max(budget_s, 0.1))
except FuturesTimeout:
with _LOCK:
_STATE["failures"] += 1
_pause(BREAKER_COOLDOWN_S, f"no answer within {budget_s:.1f}s")
raise OpenAlexUnavailable(f"timed out after {budget_s:.1f}s") from None
except OpenAlexUnavailable:
with _LOCK:
_STATE["failures"] += 1
raise
except Exception as exc: # noqa: BLE001
with _LOCK:
_STATE["failures"] += 1
_pause(BREAKER_COOLDOWN_S, f"{type(exc).__name__}: {str(exc)[:80]}")
raise OpenAlexUnavailable(str(exc)) from exc
with _LOCK:
_CACHE[key] = result
if len(_CACHE) > _CACHE_MAX:
_CACHE.popitem(last=False)
return result
# ------------------------------------------------------------------- api ----
def search_works(query: str, max_results: int = 8, budget_s: float = REQUEST_BUDGET_S,
filter_: str = DEFAULT_FILTER) -> list[dict[str, Any]]:
"""Relevance-ranked search over titles, abstracts and full text."""
query = clean_query(query)
if not query:
return []
params = {"search": query[:300], "per_page": max(1, min(max_results, 25)),
"select": SELECT, "filter": filter_}
def run() -> list[dict[str, Any]]:
works = _get("/works", params).get("results") or []
return [w for w in (parse_work(x) for x in works) if w["title"] and w["abstract"]]
return _bounded(f"search|{filter_}|{max_results}|{query.lower()}", run, budget_s)
def search_title(title: str, max_results: int = 8,
budget_s: float = REQUEST_BUDGET_S) -> list[dict[str, Any]]:
"""Title-only search, for citation resolution. No field filter: a cited
paper can be from any field, and the title match itself is the check."""
# In a filter value "," separates filters and ":" separates a field from
# its value, so both are removed along with the search-syntax characters.
title = clean_query(title, drop=",:;")
if not title:
return []
params = {"filter": f"title.search:{title[:250]}",
"per_page": max(1, min(max_results, 25)), "select": SELECT}
def run() -> list[dict[str, Any]]:
works = _get("/works", params).get("results") or []
return [parse_work(x) for x in works if x.get("title")]
return _bounded(f"title|{max_results}|{title.lower()}", run, budget_s)
def get_by_doi(doi: str, budget_s: float = REQUEST_BUDGET_S) -> dict[str, Any] | None:
doi = normalise_doi(doi)
if not doi:
return None
def run() -> dict[str, Any] | None:
work = _get(f"/works/doi:{doi}", {"select": SELECT})
return parse_work(work) if work.get("id") else None
return _bounded(f"doi|{doi}", run, budget_s)
def extract_doi(text: str) -> str | None:
m = DOI_RE.search(text or "")
return normalise_doi(m.group("doi")) if m else None