diff --git "a/app.py" "b/app.py" --- "a/app.py" +++ "b/app.py" @@ -1,3087 +1,3187 @@ -""" -ResearchBee - FastAPI backend v1.8.2 (reliability, policy and security repairs) - -Changes in v1.8.1: -- Subject browsing applies conservative closed-vocabulary spelling correction before the LLM. -- Meaningful unlisted subjects may be mapped to the nearest verified category with explicit disclosure. -- Subject responses include structured mapping provenance and alternative verified categories. -- The backend version aligns with the unified comparison/OA front end. - -Changes in v1.8.0: -- Browse by subject and manuscript matching now share one database-first journal pipeline. -- Journal identities, ISSNs, metrics, active status and access flags come from journals.db. -- Hard filters are enforced in code; unsupported top-percent and fee claims are not inferred. -- Results include explainable fit components, recent OpenAlex article evidence, comparison data, - and on-demand Jisc Open Policy Finder checks. -- Accepted-manuscript sharing requirements trigger policy checks for leading candidates. - -Changes in v1.7.0: -- Chat knowledge is stored as source-aware topic records and retrieved per question. -- Educational journal, metrics, publishing, OA, and RDM questions are answered in chat. -- Fast routing is limited to explicit tool requests so it no longer hides useful guidance. -- The chat prompt no longer claims unrestricted web browsing. - -Changes in v1.6.0: -- Journal-level licence checks use the authenticated Jisc Open Policy Finder API. -- The DOI-first OA.Works + Khazna path remains unchanged and primary. -- The former public-page renderer has been removed. -- Open Policy Finder policy facts are displayed as source data and are never generated by an LLM. - -Changes from v1.3 (verified-sources release): -- /api/check-license is now DOI-first and GROUNDED. Policy facts come from the - OA.Works Permissions API (the dataset behind cOAlition S's Journal Checker - Tool), never from the LLM. The LLM writes advisory prose only. -- Khazna deposit status via a nightly OAI-PMH harvest (three-valued: checked / - in_khazna / needs_deposit — "could not check" is never reported as "absent"). -- Every policy answer carries a provenance label. Any LLM-derived - policy_status is forcibly downgraded to "Not confirmed". -- Repository finder constrained to a curated registry; the LLM selects and - explains, the server supplies the facts. - -Changes from v1.2: -- Rate limiting middleware (sliding window, 20 req/min global, 10/min chat per IP) -- /api/chat now streams via SSE (StreamingResponse) -- /api/openalex-works — related literature per journal (OpenAlex) -- /api/altmetric — journal attention score badge (Altmetric free API) -""" -import os -import json -import re -import asyncio -import logging -import secrets -from trusted_links import with_links, discover -import time -from contextlib import asynccontextmanager -from typing import Optional, List, AsyncGenerator, Any, Sequence, Literal - -# Process boot time, captured at import so it is set before any heavy -# module-level work and before lifespan runs. Exposed on /api/health so -# the uptime monitor can distinguish "still running" from "died and came -# back between polls" — a restarted container reports RUNNING and 200 at -# both polls, but its boot_ts moves. -BOOT_TS = time.time() - -from fastapi import FastAPI, HTTPException, Request -from fastapi.middleware.cors import CORSMiddleware -from fastapi.staticfiles import StaticFiles -from fastapi.responses import FileResponse, StreamingResponse -from pydantic import BaseModel, Field, model_validator - -from openai import AsyncOpenAI, RateLimitError, APIStatusError -import httpx - -# journal_lookup wraps scimago_index and serves lookups from journals.db -# (Scopus source list x SCImago 2025). It forwards every other name straight -# through to scimago_index, so the subject-vocabulary functions below behave -# exactly as before. If journals.db is unreadable it falls back to the CSV, -# so this import cannot take the Space down. -from journal_lookup import ( - lookup_any, load_scimago, search_by_subject, load_subject_index, - subject_vocabulary, is_real_category, is_real_area, suggest_categories, - closest_subject_term, get_index, -) -import openalex -import openpolicyfinder -from prompts import ( - JOURNAL_SYSTEM_PROMPT, JOURNAL_PROFILE_PROMPT, JOURNAL_RERANK_PROMPT, - REPO_SYSTEM_PROMPT, SUBJECT_NORMALISE_PROMPT, COVER_LETTER_PROMPT, - CHAT_SYSTEM_PROMPT_BASE, -) -from chat_knowledge import get_chat_knowledge, knowledge_status -from utils import norm_issn -from journal_discovery import ( - DiscoveryPreferences, build_candidate_pool, resolve_asjc_rows, - matching_subjects, quartile_for, access_model, ranking_component, - practical_component, policy_requirement_state, clamp_int, normalise_title, -) - -# ── Verified-source modules (no LLM inside any of these) ────────────────── -import oaworks -import doi_utils -import repositories -from khazna_loader import khazna_index -from repositories import REGISTRY, registry_for_prompt, hydrate_selection -from prompts import ADVISORY_NOTE_PROMPT - -# ── Usage counting + admin dashboard ────────────────────────────────── -# usage.py counts how often each tool is used. It stores COUNTS ONLY — -# no question text, no abstracts, no DOIs — see the module docstring. -import usage -from admin import router as admin_router -from runtime_security import RuntimeSecurityMiddleware, check_rate_limit, get_client_ip -from policy_evaluation import evaluate_policy - -logger = logging.getLogger(__name__) - -# ── Constants ────────────────────────────────────────────────────────────── -# Models are server-controlled; browser input cannot select an arbitrary model. -MODEL = os.getenv("RESEARCHBEE_MODEL", "gpt-4.1-mini") -MODEL_LIGHT = os.getenv("RESEARCHBEE_MODEL_LIGHT", "gpt-4.1-nano") -RELEASE_VERSION = "1.8.2" - -# ── KU publisher-agreement candidates ───────────────────────────────────── -# Publisher matching is a discovery signal only. It does not establish title- -# level APC eligibility, author eligibility, quota availability, article-type -# coverage, or an exact top-percentile requirement. -# Aliases that are ordinary English and would over-match as substrings. -# These only count when they lead the publisher string. -AMBIGUOUS_ALIASES = { - "academic press", "saunders", "masson", "adis", "informa", - "nature research", "nature publishing", -} - -KU_APC_PUBLISHER_GROUPS = [ - # ACS - {"american chemical society"}, - # Elsevier — all regional entities - {"elsevier", "elsevier b.v.", "elsevier bv", "elsevier ltd", "elsevier inc.", - "elsevier inc", "elsevier ireland ltd", "elsevier gmbh", "elsevier masson", - "elsevier singapore", "elsevier australia", "elsevier beijing", - "elsevier editora", "elsevier espana", - # Imprints whose SCImago string never contains "Elsevier". - # NB: normalisation strips dots, so "W.B. Saunders" becomes "wb saunders" - # — that variant must be listed explicitly or ~108 titles are missed. - "academic press", "cell press", "wb saunders", "w b saunders", - "churchill livingstone", "mosby", "pergamon", "saunders", - "masson", "urban and fischer", "hanley and belfus"}, - # IEEE — all society variants - {"ieee", "ieee advancing technology", "ieee communications society", - "ieee computational intelligence", "ieee computer society", - "ieee electron devices", "ieee microwave"}, - # Springer Nature — all Springer variants - {"springer", "springer nature", "springernature", "nature portfolio", - "nature publishing", "springer international publishing", - "springer international publishing ag", "springer berlin", "springer boston", - "springer china", "springer netherlands", "springer new york", - "springer london", "springer japan", "springer verlag", - "springer science and business media", "springer publishing", - "springer science and business media llc", "springer nature america", - "biomed central", "palgrave", "palgrave macmillan", "adis", - "birkhauser", "nature research"}, - # Taylor & Francis — all T&F variants (note: & in SCImago CSV) - {"taylor and francis", "taylor & francis", "taylor & francis", - "routledge", "routledge, taylor", "taylor and francis ltd", - "taylor and francis inc", "taylor and francis a.s", - "taylor and francis asia pacific", "informa", "informa uk", - "informa uk limited", "psychology press", "crc press", - "dove medical press", "taylor and francis group"}, - # Wiley — all Wiley variants - {"wiley", "wiley-blackwell", "wiley-blackwell publishing", - "john wiley and sons", "john wiley & sons", "john wiley & sons", - "wiley-liss", "wiley-vch", "wiley - vch", - "blackwell", "blackwell publishing", "blackwell science", - "john wiley and sons inc", "john wiley and sons ltd"}, -] - -def is_ku_agreement_publisher(publisher: str) -> bool: - """Whether the publisher string matches a KU agreement publisher group. - - This is only a publisher-level candidate signal. It is never treated as - title-level APC eligibility, quota availability, or a top-percentile check. - """ - pub = (publisher or "").lower().strip() - if not pub: - return False - - def norm(value: str) -> str: - return (value.lower() - .replace("&", "&").replace("&", "and") - .replace(".", "").replace(",", "") - .replace("-", " ").strip()) - - pub_norm = norm(pub) - pub_tokens = pub_norm.split() - for group in KU_APC_PUBLISHER_GROUPS: - for alias in group: - alias_norm = norm(alias) - alias_tokens = alias_norm.split() - if pub_norm == alias_norm: - return True - if alias_norm in AMBIGUOUS_ALIASES: - if pub_norm.startswith(alias_norm + " "): - return True - continue - n = len(alias_tokens) - for i in range(len(pub_tokens) - n + 1): - if pub_tokens[i:i + n] == alias_tokens: - return True - return False - - -def is_ku_apc_covered(publisher: str, quartile: str = "") -> bool: - """Backward-compatible candidate flag; not a coverage determination.""" - return quartile == "Q1" and is_ku_agreement_publisher(publisher) - - -# Request identity and bounded quotas are shared with runtime_security. - -KHAZNA_RELOAD_HEADER = "X-ResearchBee-Reload-Token" - - -def require_khazna_reload_auth(request: Request) -> None: - """Require the Space secret before allowing a Khazna index reload.""" - expected = os.environ.get("KHAZNA_RELOAD_TOKEN", "").strip() - if not expected: - raise HTTPException( - 503, - "Khazna reload authentication is not configured.", - ) - - supplied = request.headers.get(KHAZNA_RELOAD_HEADER, "").strip() - if not supplied: - auth = request.headers.get("Authorization", "").strip() - if auth.lower().startswith("bearer "): - supplied = auth[7:].strip() - - if not supplied: - raise HTTPException( - 401, - "Khazna reload token required.", - headers={"WWW-Authenticate": "Bearer"}, - ) - if not secrets.compare_digest(supplied, expected): - raise HTTPException(403, "Invalid Khazna reload token.") - - - -# ── Khazna article workflow contact card ───────────────────────────────── -KHAZNA_DEPOSIT_CARD = { - "title": "🏛️ Deposit to Khazna — KU Institutional Repository", - "url": "https://khazna.ku.ac.ae", - "contact": "khazna@ku.ac.ae", - "library_url": "https://library.ku.ac.ae/lib", - "message": "Register your publication with Khazna. Deposit a manuscript file only when the checked version, repository location, licence and embargo conditions permit it; contact KU Library when permission is unclear.", - "metadata_tip": "Even if depositing data in a domain-specific repository, always register metadata in Khazna so your work appears in KU's research portfolio.", -} - -FAST_ROUTE_PATTERNS = { - "journals": [ - r"\b(?:find|recommend|suggest|match|identify|show me|give me)\b.{0,100}\bjournal(?:s)?\b", - r"\bwhere (?:should|can) i publish\b", - r"\banaly[sz]e my (?:paper|manuscript|abstract)\b", - r"\bbrowse journals?\b", - r"\bopen (?:the )?journal (?:finder|tool|tab)\b", - ], - "license": [ - r"\b(?:open|use|show|take me to)\b.{0,50}\b(?:license|sharing|self[- ]?archiv) (?:checker|tool|tab)\b", - r"\bopen policy checker\b", - ], - "data": [ - r"\b(?:find|recommend|suggest|match|identify|show me|give me)\b.{0,100}\b(?:data )?repositor(?:y|ies)\b", - r"\bwhere (?:should|can) i deposit (?:my )?(?:data|dataset)\b", - r"\bwhich repository (?:should|can) i use for (?:my )?(?:data|dataset)\b", - r"\bopen (?:the )?(?:data )?repository (?:finder|tool|tab)\b", - ], -} - -_INFORMATIONAL_OPENERS = ( - "what is", "what are", "what does", "how does", "how do", "how can", - "why is", "why are", "explain", "tell me about", "difference between", - "compare", "which metric", "when should", "tips for", "criteria for", -) - - -def quick_route(message: str) -> Optional[str]: - """Return a tab only for an explicit structured-tool request. - - The previous broad keyword router sent questions such as "What is FAIR - data?" or "How do I choose the best journal?" straight to a tab. That made - an expanded knowledge base unreachable. Informational questions now go to - the assistant, while clear action requests still receive an instant route. - """ - msg = " ".join(message.lower().split()) - if not msg: - return None - if msg.startswith(_INFORMATIONAL_OPENERS): - return None - for tab, patterns in FAST_ROUTE_PATTERNS.items(): - if any(re.search(pattern, msg, flags=re.IGNORECASE) for pattern in patterns): - return tab - return None - - -# ── Open Policy Finder (Jisc) authenticated API ────────────────────────── -async def fetch_opf_policy(journal_name: str, issn: Optional[str] = None) -> str: - """Return a grounded journal-level summary for the chat assistant.""" - record = await openpolicyfinder.lookup_policy(journal_name, issn or "") - return openpolicyfinder.format_for_chat(record) - - -def detect_journal_policy_question(message: str) -> Optional[str]: - """ - Detect if the user is asking about a specific journal's self-archiving policy. - Returns the journal name if detected, else None. - Uses simple heuristics — good enough for 90% of cases. - """ - msg = message.lower() - policy_triggers = [ - "self-archiv", "self archiv", "can i deposit", "can i post", - "embargo", "postprint", "accepted manuscript", "preprint policy", - "archiving rights", "green oa", "repository policy", - "version of record", "aam", "open access rights", - ] - if not any(t in msg for t in policy_triggers): - return None - - # Try to extract journal name — look for quoted names or "in X journal" patterns - quoted = re.findall(r'["\']([^"\']{3,80})["\']', message) - if quoted: - return quoted[0] - - patterns = [ - r'\bin\s+([A-Z][A-Za-z &\-]{3,60}(?:Journal|Review|Letters|Reports|Science|Nature|Communications|Research)?)', - r'\bfor\s+([A-Z][A-Za-z &\-]{3,60}(?:Journal|Review|Letters|Reports|Science|Nature|Communications|Research)?)', - r'\b([A-Z][A-Za-z &\-]{3,60}(?:Journal|Review|Letters|Reports|Science|Nature|Communications|Research))\b', - ] - for pat in patterns: - m = re.search(pat, message) - if m: - candidate = m.group(1).strip() - if len(candidate) > 3: - return candidate - return None - - -# ── Singleton OpenAI client ──────────────────────────────────────────────── -_openai_client: Optional[AsyncOpenAI] = None - - -def get_client() -> AsyncOpenAI: - global _openai_client - key = os.environ.get("OPENAI_API_KEY") - if not key: - raise HTTPException(500, "OPENAI_API_KEY not configured") - if _openai_client is None: - _openai_client = AsyncOpenAI(api_key=key, timeout=45.0, max_retries=0) - return _openai_client - - -# ── Lifespan ─────────────────────────────────────────────────────────────── -async def refresh_khazna() -> None: - """A native coroutine is scheduled on the event loop by APScheduler.""" - await asyncio.to_thread(khazna_index.reload) - - -@asynccontextmanager -async def lifespan(app: FastAPI): - load_scimago() - load_subject_index() - openalex.init_client() - await openpolicyfinder.init_client() - usage.usage.start() - refresh_task = asyncio.create_task(refresh_khazna()) - scheduler = None - try: - from apscheduler.schedulers.asyncio import AsyncIOScheduler - from apscheduler.triggers.cron import CronTrigger - scheduler = AsyncIOScheduler(timezone="Asia/Dubai") - scheduler.add_job( - refresh_khazna, - CronTrigger(hour=23, minute=30, timezone="Asia/Dubai"), - id="khazna_refresh", replace_existing=True, max_instances=1, - ) - scheduler.start() - logger.info("Khazna refresh scheduled for 23:30 Asia/Dubai") - except Exception as exc: - logger.warning("Khazna scheduler unavailable (%s)", type(exc).__name__) - app.state.khazna_scheduler = scheduler - try: - yield - finally: - if scheduler: - scheduler.shutdown(wait=False) - if not refresh_task.done(): - refresh_task.cancel() - await asyncio.gather(refresh_task, return_exceptions=True) - await usage.usage.stop() - await openpolicyfinder.close_client() - await openalex.close_client() - global _openai_client - if _openai_client is not None: - await _openai_client.close() - _openai_client = None - - -# ── App ──────────────────────────────────────────────────────────────────── -app = FastAPI(title="ResearchBee API", version=RELEASE_VERSION, lifespan=lifespan) - -app.add_middleware(RuntimeSecurityMiddleware) -app.add_middleware( - CORSMiddleware, - allow_origins=[ - "https://ku-library.github.io", - # The ResearchBee fragment is embedded in LibGuides and calls this - # API from the guide's own origin. Without these entries every - # request from the guide fails in the browser and the tools appear - # to do nothing when submitted. - "https://kustar.libguides.com", - "https://libguides.ku.ac.ae", - "http://localhost", - "http://127.0.0.1", - "http://localhost:3000", - "http://127.0.0.1:3000", - ], - expose_headers=["Retry-After"], - allow_methods=["GET", "POST", "OPTIONS"], - allow_headers=["Content-Type", "Authorization", "X-ResearchBee-Reload-Token"], -) - -# Counting observes completed responses; lifespan owns the usage task lifecycle. -usage.install(app) - -# Admin dashboard at /admin. Set RESEARCHBEE_ADMIN_PASSWORD in the Space -# secrets; without it the dashboard cannot be opened at all. -app.include_router(admin_router, prefix="/admin") - - -# ── Helpers ──────────────────────────────────────────────────────────────── -def lang_instruction(language: str) -> str: - return "Please respond entirely in Arabic." if language == "arabic" else "" - - -def build_verify_links(name: str, issn: Optional[str], sourceid: str = "") -> dict: - q = re.sub(r"[^\w\s]", "", name).strip() - q_encoded = q.replace(" ", "%20") - ni = norm_issn(issn) - return { - "scimago": (f"https://www.scimagojr.com/journalsearch.php?q={ni}&tip=iss" - if ni else f"https://www.scimagojr.com/journalsearch.php?q={q_encoded}&tip=jou"), - "sherpa_romeo": (f"https://openpolicyfinder.jisc.ac.uk/search?search={q_encoded}" - "&per_page=10&publication_page=1&publisher_page=1&funder_page=1"), - "doaj": (f"https://doaj.org/toc/{ni.replace('-', '')}" - if ni else f"https://doaj.org/search/journals?source=%7B%22query%22%3A%7B%22query_string%22%3A%7B%22query%22%3A%22{q_encoded}%22%7D%7D%7D"), - "openalex": (f"https://openalex.org/sources/issn:{ni}" - if ni else f"https://openalex.org/sources?filter=display_name.search:{q_encoded}"), - "scopus_sources": "https://www.scopus.com/sources", - "issn_display": ni or "", - "name_display": name.strip(), - } - - -def _index_extras(scimago: Optional[dict]) -> dict: - """ - journals.db-only fields, guarded. - - Additive metadata must never be able to fail a request. If anything in - here raises, the caller still gets a complete ranking block from the - fields that predate the journals.db work. The warning names the journal so - a recurring failure is traceable rather than silent. - """ - if not scimago: - return {} - try: - return { - "quartiles_by_field": scimago.get("quartiles_by_field") or [], - "sjr": scimago.get("sjr"), - "indexation_state": scimago.get("state"), - "indexation_message": scimago.get("state_message"), - "warn": bool(scimago.get("warn")), - "currently_indexed": scimago.get("currently_indexed", True), - "successor_title": scimago.get("successor_title"), - "oa_diamond": bool(scimago.get("oa_diamond")), - "from_scopus": bool(scimago.get("from_scopus")), - "from_scimago": bool(scimago.get("from_scimago", True)), - } - except Exception as e: # noqa: BLE001 - logger.warning("[JournalIndex] extras unavailable (%s)", type(e).__name__) - return {} - - -def _quartile_interpretation(scimago: Optional[dict]) -> str: - """ - Per-field quartile sentence, guarded, with the original wording as the - fallback. 46% of ranked journals rank differently across their own subject - areas, so the spread is worth showing — but never at the cost of the call. - """ - try: - by_field = (scimago or {}).get("quartiles_by_field") or [] - if len(by_field) > 1: - spread = ", ".join(f"{f['quartile']} in {f['field']}" for f in by_field) - return f"Quartile varies by subject area: {spread}." - except Exception as e: # noqa: BLE001 - logger.warning("[JournalIndex] quartile spread unavailable (%s)", type(e).__name__) - return (f"Quartile verified from journal metadata " - f"(2025 dataset, matched by ISSN {(scimago or {}).get('matched_issn', '')}).") - - -def enrich_ranking(journal: dict, scimago: Optional[dict], oa_pct: Optional[dict]) -> dict: - if scimago: - q = scimago.get("quartile", "") - out = { - "requested": True, "source": "Verified from journal data", "year": "2025", - "category": scimago.get("categories", ""), "metric_value": None, - "percentile": f"Rank #{scimago.get('rank', '—')} globally" if scimago.get("rank") else None, - "quartile": q if q and q != "-" else None, - "interpretation": _quartile_interpretation(scimago), - "verification_status": "Confirmed", "h_index": scimago.get("h_index"), - } - # journals.db extras are merged in, never inlined. If the index or its - # shape misbehaves the response is still complete and correct — the - # caller simply does not get the extra keys. - out.update(_index_extras(scimago)) - return out - elif oa_pct: - return { - "requested": True, "source": "OpenAlex-derived percentile", "year": "2025", - "category": oa_pct.get("concept"), "metric_value": None, - "percentile": f"{oa_pct['percentile']}th percentile within {oa_pct['peer_count']} {oa_pct['concept']} journals", - "quartile": oa_pct.get("quartile"), - "interpretation": "OpenAlex-derived percentile — NOT a Scopus/SCImago quartile.", - "verification_status": "OpenAlex-derived (not a Scopus/SCImago metric)", - } - else: - return { - "requested": True, "source": "Not available", "year": None, - "category": None, "metric_value": None, "percentile": None, "quartile": None, - "interpretation": "No journal metrics match found. Use verify links to check Scopus Sources directly.", - "verification_status": "Not confirmed — verify manually via Scopus Sources link", - } - - -def _scimago_title_fallback(title: str) -> Optional[dict]: - """ - Fallback: fuzzy title match in SCImago when ISSN lookup fails. - Used when LLM returns a wrong/hallucinated ISSN. - """ - if not title: - return None - rows = load_subject_index() - title_lower = title.lower().strip() - # Exact title match first - for r in rows: - if r["title"].lower() == title_lower: - ni = norm_issn(r.get("issn", "")) - if ni: - return lookup_any([ni]) - # Partial match — title contains all significant words - words = [w for w in title_lower.split() if len(w) > 3] - if len(words) >= 2: - for r in rows: - rt = r["title"].lower() - if all(w in rt for w in words): - ni = norm_issn(r.get("issn", "")) - if ni: - result = lookup_any([ni]) - if result: - return result - return None - - -async def enrich_journals(journals: list) -> list: - enriched_oa = await openalex.enrich_all(journals) - result = [] - for j in enriched_oa: - issn = j.get("issn") - oa = j.get("openalex") or {} - all_issns = [issn, oa.get("issn_l")] + (oa.get("issns") or []) - scimago = lookup_any(all_issns) - - # ── Title fallback: if ISSN lookup fails, try matching by journal name ── - if not scimago: - scimago = _scimago_title_fallback(j.get("name", "")) - if scimago: - logger.info(f"[Enrich] Title fallback matched: {j.get('name')}") - - oa_pct = j.get("oa_percentile") - final_issn = (scimago or {}).get("matched_issn") or oa.get("issn_l") or issn - j["issn"] = final_issn - j["ranking"] = enrich_ranking(j, scimago, oa_pct) - sourceid = (scimago or {}).get("sourceid", "") - j["verify_links"] = build_verify_links(j.get("name", ""), final_issn, sourceid) - j.pop("oa_percentile", None) - result.append(j) - return result - - -def enrich_extended(extended: list) -> list: - result = [] - for j in extended: - issn = j.get("issn") - scimago = lookup_any([issn]) if issn else None - q = (scimago or {}).get("quartile", "") - h_index = (scimago or {}).get("h_index", "") - publisher= (scimago or {}).get("publisher", "") or j.get("publisher", "") - oa = (scimago or {}).get("open_access", "No") - country = (scimago or {}).get("country", "") - sourceid = (scimago or {}).get("sourceid", "") - j["quartile"] = q if q and q != "-" else None - j["h_index"] = int(h_index) if str(h_index).isdigit() else None - j["publisher"] = publisher - j["open_access"] = oa - j["country"] = country - - # warn is True ONLY for the 967 titles Scopus discontinued for cause. - # It is deliberately False for the 6,855 that were merely renamed: - # telling a researcher that a journal which changed its name is a - # quality risk is worse than saying nothing at all. - # - # Merged through the guarded helper for the same reason as above: a - # quick-match row must not be able to fail the whole request. - j.update(_index_extras(scimago)) - - j["verify_links"] = build_verify_links(j.get("name", ""), issn, sourceid) - j["ku_apc_covered"] = is_ku_apc_covered(publisher, q) - result.append(j) - return result - -_POLICY_ACTION_RE = re.compile( - r"(?:self[- ]?archiv|embargo|deposit|repository|licen[cs]e|preprint|postprint|" - r"accepted manuscript|version of record|green oa|\baam\b)", - re.IGNORECASE, -) - - -def enforce_journal_policy_guard(parsed: dict) -> dict: - """ - Make the journal-finder workflow incapable of returning LLM policy facts. - - The model may rank and explain journals, but every policy-bearing field is - replaced with a canonical, three-valued "unknown" response. This guard is - deliberately server-side so a future prompt regression cannot reintroduce - apparent permissions, embargoes, or licences. - """ - if not isinstance(parsed, dict): - return {} - - for journal in parsed.get("journals") or []: - if not isinstance(journal, dict): - continue - green = oaworks.not_confirmed( - journal.get("name", ""), - journal.get("issn", ""), - reason="Journal Finder evaluates fit only; no article-level permission lookup was performed", - ) - green["repository_action_note"] = ( - "Use ResearchBee License Checking with the article DOI before depositing any manuscript version." - ) - green["evidence_note"] = ( - "No article-level permission lookup is performed by Journal Finder. " - "Policy values from the language model are discarded by the server." - ) - green["risk_flag"] = "Deposit rights not checked — verify with the article DOI." - journal["green_oa"] = green - journal["oa_compliance_note"] = ( - "Journal Finder does not determine self-archiving rights. Use the DOI-based " - "License Checking tool before depositing any version." - ) - - unknown = oaworks.not_confirmed( - reason="Journal Finder does not perform an article-level permission lookup" - ) - parsed["repository_recommendation"] = oaworks.recommendation_from(unknown) - - safe_actions = [] - for action in parsed.get("next_actions") or []: - action = " ".join(str(action or "").split()) - if action and not _POLICY_ACTION_RE.search(action): - safe_actions.append(action[:600]) - safe_actions.extend([ - "Verify the journal scope, author guidelines, current fees, and indexation before submission.", - "Use ResearchBee License Checking with the article DOI before depositing any manuscript version.", - ]) - # Stable de-duplication. - parsed["next_actions"] = list(dict.fromkeys(safe_actions)) - parsed["global_notes"] = ( - "Journal Finder evaluates publication fit and verified journal metrics only. " - "Self-archiving permissions, embargoes, and licences require a DOI-based check." - ) - parsed["policy_source_mode"] = "not_checked_in_journal_finder" - return parsed - - -def build_subject_journal_result(journal: dict) -> dict: - """Build one subject-browse row, including all DB-backed field quartiles.""" - issn = journal.get("issn") - scimago = lookup_any([issn]) if issn else None - if not scimago: - scimago = _scimago_title_fallback(journal.get("title", "")) - - source = scimago or journal - final_issn = source.get("matched_issn") or issn or "" - quartile = source.get("quartile") or journal.get("quartile", "") - h_index = source.get("h_index", journal.get("h_index")) - try: - h_index = int(h_index) if h_index not in (None, "") else None - except (TypeError, ValueError): - h_index = None - - result = { - "name": journal.get("title", ""), - "issn": final_issn, - "quartile": quartile if quartile and quartile != "-" else None, - "h_index": h_index, - "publisher": source.get("publisher", "") or journal.get("publisher", ""), - "open_access": source.get("open_access", journal.get("open_access", "No")), - "country": source.get("country", journal.get("country", "")), - "categories": source.get("categories", journal.get("categories", "")), - "verify_links": build_verify_links( - journal.get("title", ""), - final_issn, - source.get("sourceid", journal.get("sourceid", "")), - ), - } - result.update(_index_extras(scimago)) - result["ku_apc_covered"] = is_ku_apc_covered(result["publisher"], quartile) - return result - - -# ── OpenAI calls: bounded retries, validated top-level JSON, redacted logs ── -async def call_openai(client: AsyncOpenAI, system: str, user: str, max_tokens: int = 4096, model: str = None) -> dict: - chosen_model = model or MODEL - for attempt in range(3): - try: - response = await client.chat.completions.create( - model=chosen_model, - response_format={"type": "json_object"}, - messages=[{"role": "system", "content": system}, {"role": "user", "content": user}], - max_tokens=max_tokens, - temperature=0.3, - ) - content = response.choices[0].message.content or "" - if getattr(response.choices[0], "finish_reason", None) == "length": - logger.warning("Model response truncated: model=%s chars=%d", chosen_model, len(content)) - return {"error": "The answer was cut off before it could be completed. Please try again."} - try: - parsed = json.loads(content) - except (json.JSONDecodeError, TypeError): - logger.warning("Invalid model JSON: model=%s chars=%d", chosen_model, len(content)) - return {"error": "The AI service returned an unreadable answer. Please try again."} - if not isinstance(parsed, dict) or parsed.get("error"): - return {"error": "The AI service returned an unexpected answer. Please try again."} - return parsed - except RateLimitError as exc: - # Inspect only the machine code, never expose/log provider response text. - code = str(getattr(exc, "code", "") or "") - if code in {"insufficient_quota", "billing_hard_limit_reached"}: - raise HTTPException(503, "AI capacity is unavailable. Please contact the library.") from None - if attempt == 2: - raise HTTPException(429, "AI service is busy. Please try again shortly.", headers={"Retry-After": "30"}) from None - except APIStatusError as exc: - status = getattr(exc, "status_code", 500) - if status < 500 or attempt == 2: - logger.warning("AI upstream HTTP status=%s", status) - raise HTTPException(503, "AI service is temporarily unavailable. Please try again.") from None - except Exception as exc: - logger.warning("AI request failed (%s)", type(exc).__name__) - raise HTTPException(503, "AI service is temporarily unavailable. Please try again.") from None - await asyncio.sleep(2 ** attempt) - raise HTTPException(503, "AI service is temporarily unavailable.") - - -def _text_list(value: Any, limit: int = 10, width: int = 300) -> list[str]: - """Validate model-authored list shape before downstream use.""" - if not isinstance(value, list): - return [] - return [" ".join(v.split())[:width] for v in value if isinstance(v, str) and v.strip()][:limit] - - -# ── Request models ───────────────────────────────────────────────────────── -_T300 = Field("", max_length=300) -_T500 = Field("", max_length=500) - - -class ManuscriptInput(BaseModel): - title: str = Field(..., max_length=300) - abstract: str = Field(..., max_length=5000) - keywords: str = _T500 - discipline: str = _T300 - article_type: str = Field("Original research article", max_length=100) - methods: str = _T500 - audience: str = _T300 - funder: str = _T300 - institution: str = _T300 - country: str = _T300 - apc_budget: str = _T300 - # v1.8 names. Access model and manuscript-sharing rights are intentionally - # separate. The legacy fields remain accepted for older LibGuides copies. - access_preference: str = Field("Any journal", max_length=80) - sharing_requirement: bool = False - intended_repository: str = Field("Institutional repository", max_length=100) - ranking_preference: str = Field("No filter", max_length=50) - preferred_journals: str = _T500 - avoid_journals: str = _T500 - oa_preference: str = Field("", max_length=50) - ranking_source: str = Field("", max_length=50) - speed_preference: str = Field("", max_length=50) - repository_target: str = Field("", max_length=100) - - -class JournalRequest(BaseModel): - manuscript: ManuscriptInput - model: str = Field("gpt-4o-mini", max_length=30) - language: str = Field("english", max_length=10) - - -class LicenseInput(BaseModel): - # journal_name is no longer required: a DOI alone is a complete request, - # and the endpoint raises 422 if neither is supplied. - journal_name: str = Field("", max_length=300) - doi: str = Field("", max_length=400) # DOI or article URL - issn: str = Field("", max_length=20) - publisher: str = _T300 - manuscript_version: str = Field("All versions", max_length=50) - funder: str = _T300 - intended_repository: str = Field("Institutional repository", max_length=100) - intended_licence: str = Field("CC BY", max_length=50) - notes: str = _T500 - - -class LicenseRequest(BaseModel): - license_input: LicenseInput - model: str = Field("gpt-4o-mini", max_length=30) - language: str = Field("english", max_length=10) - - -class DatasetInput(BaseModel): - title: str = Field(..., max_length=300) - description: str = Field(..., max_length=2000) - discipline: str = Field(..., max_length=300) - data_types: str = Field(..., max_length=300) - file_formats: str = _T300 - approx_size: str = _T300 - sensitivity: str = Field("Open", max_length=50) - contains_personal_data: str = Field("No", max_length=50) - licence_intent: str = Field("CC BY", max_length=50) - funder: str = _T300 - institution: str = _T300 - country: str = _T300 - needs_doi: str = Field("Yes", max_length=10) - needs_versioning: str = Field("Yes", max_length=10) - embargo_required: Literal["Yes", "No", "Unsure"] = "No" - embargo_months: Optional[int] = Field(None, ge=0, le=120) - preferred_repository: str = _T300 - related_publication: str = _T300 - notes: str = _T500 - - -class RepoRequest(BaseModel): - dataset: DatasetInput - model: str = Field("gpt-4o-mini", max_length=30) - language: str = Field("english", max_length=10) - - -class SubjectRequest(BaseModel): - subject: str = Field(..., min_length=1, max_length=200) - ranking_preference: str = Field("No filter", max_length=50) - access_preference: str = Field("Any journal", max_length=80) - sharing_requirement: bool = False - intended_repository: str = Field("Institutional repository", max_length=100) - apc_budget: str = _T300 - funder: str = _T300 - preferred_journals: str = _T500 - avoid_journals: str = _T500 - model: str = Field("gpt-4o-mini", max_length=30) - language: str = Field("english", max_length=10) - - -class JournalPolicyItem(BaseModel): - key: str = Field("", max_length=200) - name: str = Field(..., min_length=1, max_length=300) - issn: str = Field("", max_length=20) - publisher: str = _T300 - - -class JournalPolicyBatchRequest(BaseModel): - journals: List[JournalPolicyItem] = Field(..., min_length=1, max_length=5) - manuscript_version: str = Field("Accepted manuscript (AAM / postprint)", max_length=80) - intended_repository: str = Field("Institutional repository", max_length=100) - funder: str = _T300 - language: str = Field("english", max_length=10) - - -class CoverLetterRequest(BaseModel): - manuscript_title: str = Field(..., max_length=300) - abstract: str = Field(..., max_length=5000) - journal_name: str = Field(..., max_length=300) - publisher: str = Field("", max_length=300) - article_type: str = Field("Original research article", max_length=100) - author_name: str = Field("", max_length=200) - discipline: str = Field("", max_length=300) - language: str = Field("english", max_length=10) - model: str = Field("gpt-4o-mini", max_length=30) - - -class ExportRanking(BaseModel): - quartile: Optional[str] = Field(None, max_length=10) - - -class ExportJournal(BaseModel): - name: str = Field(..., min_length=1, max_length=300) - publisher: str = Field("", max_length=500) - issn: Optional[str] = Field("", max_length=50) - quartile: Optional[str] = Field(None, max_length=10) - ranking: Optional[ExportRanking] = None - - -class ExportRequest(BaseModel): - journals: List[ExportJournal] = Field(..., min_length=1, max_length=30) - format: Literal["bibtex", "ris"] = "bibtex" - - -class ChatMessage(BaseModel): - role: str = Field(..., pattern="^(user|assistant)$") - content: str = Field(..., max_length=12000) - - @model_validator(mode="after") - def validate_role_limit(self): - if self.role == "user" and len(self.content) > 2000: - raise ValueError("User messages must contain at most 2000 characters.") - return self - - -class ChatRequest(BaseModel): - messages: List[ChatMessage] = Field(..., max_length=20) - language: str = Field("english", max_length=10) - - @model_validator(mode="after") - def validate_history(self): - if self.messages and self.messages[-1].role != "user": - raise ValueError("The final chat message must be from the user.") - return self - - -# ── Routes ───────────────────────────────────────────────────────────────── -def _journal_index_status() -> dict: - """Whether journals.db opened, for /api/health. Never raises.""" - try: - import journal_lookup - ok = journal_lookup.index_available() - if not ok: - return {"loaded": False, "fallback": "scimago CSV"} - meta = journal_lookup._get_index().meta() - return { - "loaded": True, - "journals": int(meta.get("n_journals", 0)), - "active": int(meta.get("n_active", 0)), - "discontinued": int(meta.get("n_discontinued", 0)), - "built": meta.get("built_at", "")[:10], - } - except Exception as e: # noqa: BLE001 - return {"loaded": False, "error": "Journal index could not be opened"} - - -@app.get("/api/health") -async def health(): - """ - Service status plus data-source provenance. - - The `sources` block distinguishes local readiness, configuration and - observed upstream status. It does not actively probe external services. - """ - now = time.time() - index_status = _journal_index_status() - try: - scimago_count = len(load_subject_index()) - except Exception: - scimago_count = 0 - ready = bool(index_status.get("loaded")) and scimago_count > 0 - return { - "status": "ok" if ready else "degraded", - "ready": ready, - "liveness": "ok", - "service": "ResearchBee", - "version": RELEASE_VERSION, - # Restart detection for the uptime monitor. boot_ts is the stable - # identity of this process; when it changes between two polls the - # container was replaced, whatever the HTTP status said. - "boot_ts": round(BOOT_TS, 3), - "uptime_s": round(now - BOOT_TS, 1), - "knowledge_base": knowledge_status(), - "sources": { - "oaworks": { - "endpoint": oaworks.OA_WORKS_BASE + oaworks.PERMISSIONS_PATH, - "auth": "none required", - "ku_ror_set": bool(oaworks.KU_ROR), - }, - "khazna": khazna_index.status, - "repository_registry": {"entries": len(REGISTRY)}, - "open_policy_finder": openpolicyfinder.status(), - "scimago": {"loaded": scimago_count > 0, "records": scimago_count, "metric_year": 2025}, - "openalex": openalex.status(), - "openai": {"configured": bool(os.getenv("OPENAI_API_KEY")), "availability": "not_probed"}, - # False here means journals.db did not open and every lookup is - # falling back to the CSV — the Space still works, but none of the - # Scopus data is reaching users. Worth alerting on. - "journal_index": index_status, - }, - } - - -@app.post("/api/admin/reload-khazna") -async def reload_khazna(request: Request): - """ - Force an immediate reload of the Khazna index. - - Called by the harvest workflow after a successful publish, so the Space - picks up new data within seconds instead of waiting for the 23:30 refresh. - Idempotent and harmless: it only re-reads a public dataset. - """ - require_khazna_reload_auth(request) - ip = get_client_ip(request) - if not check_rate_limit(ip, "global"): - raise HTTPException(429, "Too many requests.") - ok = await asyncio.to_thread(khazna_index.reload) - return {"reloaded": ok, "khazna": khazna_index.status} - - -# ── OpenAlex related works per journal ──────────────────────────────────── -@app.get("/api/openalex-works") -async def openalex_works(request: Request, source_id: str, per_page: int = 5): - """ - Fetch recent highly-cited works from an OpenAlex source (journal). - source_id: OpenAlex source ID e.g. S137773608 - """ - ip = get_client_ip(request) - if not check_rate_limit(ip, "global"): - raise HTTPException(429, "Too many requests. Please wait a moment.") - - clean_id = str(source_id or "").strip().rsplit("/", 1)[-1] - if clean_id.startswith("S"): - clean_id = clean_id[1:] - if not re.fullmatch(r"[0-9]{1,20}", clean_id): - raise HTTPException(400, "Invalid OpenAlex source_id") - - try: - client = openalex._get_client() - r = await client.get( - "https://api.openalex.org/works", - params={ - "filter": f"primary_location.source.id:S{clean_id},is_retracted:false", - "sort": "cited_by_count:desc", - "per_page": min(max(per_page, 1), 8), - "select": "id,title,publication_year,doi,cited_by_count,authorships", - }, - timeout=6.0, - ) - if not r.is_success: - raise HTTPException(503, "OpenAlex is temporarily unavailable. Please try again.") - - works = [] - for w in r.json().get("results", []): - authors = [ - a.get("author", {}).get("display_name", "") - for a in (w.get("authorships") or [])[:3] - ] - works.append({ - "title": w.get("title", ""), - "year": w.get("publication_year"), - "doi": w.get("doi", ""), - "cited_by": w.get("cited_by_count", 0), - "authors": authors, - "openalex_url": w.get("id", ""), - }) - return {"works": works, "status": "available" if works else "no_results"} - except HTTPException: - raise - except Exception as e: - logger.warning("OpenAlex works unavailable (%s)", type(e).__name__) - raise HTTPException(503, "OpenAlex is temporarily unavailable. Please try again.") from None - - -@app.get("/api/openalex-cited-by") -async def openalex_cited_by(request: Request, work_id: str, per_page: int = 5): - """Return the most-cited papers that cite one validated OpenAlex work id.""" - ip = get_client_ip(request) - if not check_rate_limit(ip, "global"): - raise HTTPException(429, "Too many requests. Please wait a moment.") - - clean_id = str(work_id or "").strip().replace("https://openalex.org/", "") - if not re.fullmatch(r"W\d+", clean_id): - raise HTTPException(400, "Invalid OpenAlex work_id") - - try: - client = openalex._get_client() - response = await client.get( - "https://api.openalex.org/works", - params={ - "filter": f"cites:{clean_id},is_retracted:false", - "sort": "cited_by_count:desc", - "per_page": min(max(per_page, 1), 8), - "select": "id,title,publication_year,doi,cited_by_count,authorships", - }, - timeout=6.0, - ) - if not response.is_success: - raise HTTPException(503, "OpenAlex is temporarily unavailable. Please try again.") - - works = [] - for work in response.json().get("results", []): - authors = [ - authorship.get("author", {}).get("display_name", "") - for authorship in (work.get("authorships") or [])[:3] - ] - works.append({ - "title": work.get("title", ""), - "year": work.get("publication_year"), - "doi": work.get("doi", ""), - "cited_by": work.get("cited_by_count", 0), - "authors": [author for author in authors if author], - "openalex_url": work.get("id", ""), - }) - return {"works": works, "status": "available" if works else "no_results"} - except HTTPException: - raise - except Exception as exc: - logger.warning("OpenAlex cited-by unavailable (%s)", type(exc).__name__) - raise HTTPException(503, "OpenAlex is temporarily unavailable. Please try again.") from None - - -# ── Trending papers by concept/topic ───────────────────────────────────── -@app.get("/api/trending-papers") -async def trending_papers(request: Request, concept: str = "", concept_id: str = "", per_page: int = 5): - """ - Fetch most-cited papers in last 3 years for a given concept/topic via OpenAlex. - concept_id: OpenAlex concept ID (e.g. C41008148) — preferred - concept: display name fallback (e.g. "Machine Learning") - """ - ip = get_client_ip(request) - if not check_rate_limit(ip, "global"): - raise HTTPException(429, "Too many requests.") - - if not concept and not concept_id: - return {"works": [], "status": "no_results"} - - if len(concept) > 200 or (concept_id and not re.fullmatch(r"C?[0-9]{1,20}", concept_id)): - raise HTTPException(400, "Invalid subject identifier") - try: - from datetime import datetime - year_cutoff = datetime.now().year - 3 - client = openalex._get_client() - - # Build filter — prefer concept_id for accuracy - if concept_id: - clean_id = concept_id.strip().lstrip("C") - filter_str = f"concepts.id:C{clean_id},publication_year:>{year_cutoff},is_retracted:false" - else: - # Use keyword search via default_search - filter_str = f"publication_year:>{year_cutoff},is_retracted:false" - - params = { - "filter": filter_str, - "sort": "cited_by_count:desc", - "per_page": min(max(per_page, 1), 8), - "select": "id,title,publication_year,doi,cited_by_count,authorships,primary_location", - } - - # If only concept name, use search param instead of filter - if not concept_id and concept: - params["search"] = concept - params["filter"] = f"publication_year:>{year_cutoff},is_retracted:false" - - r = await client.get( - "https://api.openalex.org/works", - params=params, - timeout=8.0, - ) - if not r.is_success: - raise HTTPException(503, "OpenAlex is temporarily unavailable. Please try again.") - - works = [] - for w in r.json().get("results", []): - authors = [ - a.get("author", {}).get("display_name", "") - for a in (w.get("authorships") or [])[:3] - ] - journal = (w.get("primary_location") or {}).get("source", {}) or {} - works.append({ - "title": w.get("title", ""), - "year": w.get("publication_year"), - "doi": w.get("doi", ""), - "cited_by": w.get("cited_by_count", 0), - "authors": [a for a in authors if a], - "journal": journal.get("display_name", ""), - "openalex_url": w.get("id", ""), - }) - return {"works": works, "status": "available" if works else "no_results"} - except HTTPException: - raise - except Exception as e: - logger.warning("OpenAlex recent papers unavailable (%s)", type(e).__name__) - raise HTTPException(503, "OpenAlex is temporarily unavailable. Please try again.") from None - - -# ── Altmetric journal attention ─────────────────────────────────────────── -@app.get("/api/altmetric") -async def altmetric_score(request: Request, issn: str): - """Legacy endpoint: no verified journal-level aggregate is available.""" - return {"score": None, "url": None, "issn": norm_issn(issn), "status": "not_supported", - "message": "Article-level attention cannot be reported as a journal-level score."} - - -# ── Unified verified journal discovery (v1.8) ───────────────────────────── -_JOURNAL_PROFILE_VOCAB_BLOCK: Optional[str] = None - - -def journal_profile_prompt_with_vocabulary() -> str: - global _JOURNAL_PROFILE_VOCAB_BLOCK - if _JOURNAL_PROFILE_VOCAB_BLOCK is None: - vocabulary = subject_vocabulary() - categories = sorted(vocabulary["categories"]) - areas = sorted(vocabulary["areas"]) - _JOURNAL_PROFILE_VOCAB_BLOCK = ( - "\n\nVALID AREAS (" + str(len(areas)) + "):\n" - + "\n".join("- " + value.title() for value in areas) - + "\n\nVALID CATEGORIES (" + str(len(categories)) + "):\n" - + "\n".join("- " + value.title() for value in categories) - ) - return JOURNAL_PROFILE_PROMPT + _JOURNAL_PROFILE_VOCAB_BLOCK - - -def _legacy_access_and_sharing(manuscript: ManuscriptInput) -> tuple[str, bool, str]: - access = " ".join((manuscript.access_preference or "").split()) - sharing = bool(manuscript.sharing_requirement) - repository = " ".join((manuscript.intended_repository or "").split()) - legacy = " ".join((manuscript.oa_preference or "").split()).lower() - if not access or access.lower() == "any journal": - if "diamond" in legacy or "no apc" in legacy: - access = "Diamond / no APC" - elif "gold" in legacy: - access = "Open-access journals" - else: - access = "Any journal" - if "green" in legacy: - sharing = True - if (not repository or repository == "Institutional repository") and manuscript.repository_target: - repository = manuscript.repository_target - return access, sharing, repository or "Institutional repository" - - -def _preferences_from_manuscript(manuscript: ManuscriptInput) -> DiscoveryPreferences: - access, sharing, repository = _legacy_access_and_sharing(manuscript) - return DiscoveryPreferences.from_values( - ranking_preference=manuscript.ranking_preference, - access_preference=access, - sharing_requirement=sharing, - intended_repository=repository, - apc_budget=manuscript.apc_budget, - funder=manuscript.funder, - preferred_journals=manuscript.preferred_journals, - avoid_journals=manuscript.avoid_journals, - ) - - -def _preferences_from_subject(req: SubjectRequest) -> DiscoveryPreferences: - return DiscoveryPreferences.from_values( - ranking_preference=req.ranking_preference, - access_preference=req.access_preference, - sharing_requirement=req.sharing_requirement, - intended_repository=req.intended_repository, - apc_budget=req.apc_budget, - funder=req.funder, - preferred_journals=req.preferred_journals, - avoid_journals=req.avoid_journals, - ) - - -def _fallback_manuscript_profile(manuscript: ManuscriptInput, note: str = "") -> dict: - keywords = [k.strip() for k in re.split(r"[,;]", manuscript.keywords or "") if k.strip()] - return { - "manuscript_understanding": { - "summary": manuscript.title, - "discipline": manuscript.discipline, - "article_type": manuscript.article_type, - "methods": manuscript.methods, - "audience": manuscript.audience, - "keywords": keywords[:10], - "inferred_criteria": [], - "assumptions": [ - "The manuscript profile could not be fully inferred; supplied fields were used directly." - ], - }, - "primary_category": manuscript.discipline if is_real_category(manuscript.discipline) else "", - "additional_categories": [], - "area": manuscript.discipline if is_real_area(manuscript.discipline) else "", - "confidence": "near" if manuscript.discipline else "not_found", - "classification_note": note or "Subject classification used the author-supplied fields only.", - } - - -async def _profile_manuscript(manuscript: ManuscriptInput, language: str) -> dict: - payload = manuscript.model_dump() - # Legacy form fields must not steer the classifier as if they were evidence - # about the manuscript's subject. - for key in ("oa_preference", "ranking_source", "speed_preference", "repository_target"): - payload.pop(key, None) - try: - parsed = await call_openai( - get_client(), - journal_profile_prompt_with_vocabulary(), - f"{lang_instruction(language)}Classify this manuscript.\n\n{json.dumps(payload, ensure_ascii=False, indent=2)}", - max_tokens=1400, - model=MODEL_LIGHT, - ) - if parsed.get("error"): - return _fallback_manuscript_profile(manuscript, parsed["error"]) - except Exception as exc: - logger.warning("[JournalProfile] using supplied fields (%s)", type(exc).__name__) - return _fallback_manuscript_profile(manuscript, "AI classification was unavailable; supplied fields were used.") - - understanding = parsed.get("manuscript_understanding") or {} - if not isinstance(understanding, dict): - return _fallback_manuscript_profile(manuscript, "AI profile format was invalid; supplied fields were used.") - understanding = { - "summary": " ".join(str(understanding.get("summary") or manuscript.title).split())[:1200], - "discipline": " ".join(str(understanding.get("discipline") or manuscript.discipline).split())[:300], - "article_type": " ".join(str(understanding.get("article_type") or manuscript.article_type).split())[:100], - "methods": " ".join(str(understanding.get("methods") or manuscript.methods).split())[:500], - "audience": " ".join(str(understanding.get("audience") or manuscript.audience).split())[:300], - "keywords": _text_list(understanding.get("keywords"), 10, 80), - "inferred_criteria": _text_list(understanding.get("inferred_criteria"), 8, 300), - "assumptions": _text_list(understanding.get("assumptions"), 8, 300), - } - primary = str(parsed.get("primary_category") or "").strip() - additional = _text_list(parsed.get("additional_categories"), 3, 200) - area = str(parsed.get("area") or "").strip() - # Closed-list validation. Invalid names are discarded rather than fuzzily - # corrected into a different field. - valid_categories = subject_vocabulary()["categories"] - valid_areas = subject_vocabulary()["areas"] - primary = primary if primary.lower() in valid_categories else "" - additional = [v for v in additional if v.lower() in valid_categories and v.lower() != primary.lower()][:2] - area = area if area.lower() in valid_areas else "" - return { - "manuscript_understanding": understanding, - "primary_category": primary, - "additional_categories": additional, - "area": area, - "confidence": parsed.get("confidence") if isinstance(parsed.get("confidence"), str) and parsed.get("confidence") in {"exact", "near", "not_found"} else "near", - "classification_note": " ".join(str(parsed.get("classification_note") or "").split())[:500], - } - - -def _candidate_query_text(manuscript: ManuscriptInput, profile: dict) -> str: - understanding = profile.get("manuscript_understanding") or {} - parts = [ - manuscript.title, - manuscript.abstract, - manuscript.keywords, - manuscript.discipline, - understanding.get("discipline", ""), - " ".join(understanding.get("keywords") or []), - understanding.get("methods", ""), - ] - return " ".join(str(part or "") for part in parts) - - -def _subject_query_text(subject: str, normalised: str, category: str, area: str) -> str: - return " ".join(value for value in (subject, normalised, category, area) if value) - - -def _candidate_base_dict(rec_obj, asjc_codes: Sequence[str], score_ratio: float) -> dict: - record = rec_obj.journal - matched_q, matched_subject_rows = quartile_for(record, asjc_codes) - matched_subjects = [ - { - "code": row.get("asjc", ""), - "field": row.get("name", ""), - "parent": row.get("parent", ""), - "quartile": row.get("quartile"), - } - for row in matched_subject_rows - ] - all_fields = [ - {"code": row.get("asjc", ""), "field": row.get("name", ""), "quartile": row.get("quartile")} - for row in record.subjects if row.get("quartile") - ] - issn = record.issn_e or record.issn_print or "" - publisher = record.publisher_group or record.publisher or "" - access = access_model(record) - agreement_publisher = is_ku_agreement_publisher(publisher) - ku_candidate = bool(agreement_publisher and matched_q == "Q1") - return { - "id": record.scopus_id, - "scopus_id": record.scopus_id, - "name": record.title, - "issn": issn, - "publisher": publisher, - "publisher_raw": record.publisher or "", - "identity_verification": { - "status": "Confirmed", - "method": "Canonical Scopus source record", - "scopus_id": record.scopus_id, - "issn_print": record.issn_print or "", - "issn_electronic": record.issn_e or "", - }, - "ranking": { - "source": "Scopus source list + SCImago 2025", - "year": "2025", - "quartile": matched_q, - "category": "; ".join(row["field"] for row in matched_subjects if row.get("field"))[:500], - "h_index": record.h_index, - "sjr": record.sjr, - "rank": record.sjr_rank, - "verification_status": "Confirmed", - "verification_method": "Canonical Scopus source ID and bundled SCImago match", - "interpretation": ( - f"{matched_q} in a matched subject field; compare within the relevant category." - if matched_q else "Currently indexed, but a matched-field quartile was not available." - ), - }, - # Top-level mirrors keep the shared LibGuides renderers and exports - # consistent across manuscript and subject discovery modes. - "quartile": matched_q, - "h_index": record.h_index, - "sjr": record.sjr, - "quartiles_by_field": all_fields, - "matched_subjects": matched_subjects, - "state": record.state, - "state_message": record.state_message, - "indexation_state": record.state, - "indexation_message": record.state_message, - "currently_indexed": record.currently_indexed, - "warn": record.warn, - "successor_title": record.successor_title, - "from_scopus": True, - "source_type": record.source_type, - "coverage_start": record.coverage_start, - "coverage_end": record.coverage_end, - "coverage_has_gap": record.coverage_has_gap, - "open_access": "Yes" if access["code"] in {"open", "diamond"} else "No/unknown", - "oa_diamond": record.oa_diamond, - "access_model": access, - "ku_agreement_publisher": agreement_publisher, - "ku_apc_candidate": ku_candidate, - "ku_apc_status": ( - "Potential KU publisher-agreement candidate; exact title, author, article type, quota and top-percentile eligibility are not confirmed." - if ku_candidate else - "Publisher-level KU agreement signal only; title-level APC eligibility is not confirmed." - if agreement_publisher else - "No KU publisher-agreement signal found in the configured publisher list." - ), - # Backward-compatible flag, now explicitly a candidate signal only. - "ku_apc_covered": ku_candidate, - "candidate_method": rec_obj.method, - "candidate_reasons": list(rec_obj.reasons), - "base_score": round(float(rec_obj.score), 4), - "base_score_ratio": round(float(score_ratio), 4), - "similar_articles": [], - "article_similarity_score": 0, - "article_evidence_status": "not_checked", - "sharing_policy": { - "status": "not_checked", - "requirement_status": "not_requested", - "message": "Journal-level sharing policy has not been checked." - }, - "verify_links": build_verify_links(record.title, issn, record.scimago_id or record.scopus_id), - } - - -async def _attach_article_evidence(candidates: list[dict], query: str, limit: int = 12) -> None: - if not candidates: - return - try: - evidence = await openalex.evidence_for_journals(candidates, query, limit=limit, per_page=3) - except Exception as exc: - logger.warning("[JournalEvidence] batch unavailable (%s)", type(exc).__name__) - return - for journal in candidates: - got = evidence.get(str(journal.get("id"))) - if not got: - continue - journal["similar_articles"] = got.get("works") or [] - journal["article_similarity_score"] = clamp_int(got.get("similarity_score"), 0, 100) - journal["article_evidence_status"] = got.get("status") or "unavailable" - journal["openalex"] = got.get("source") - if journal.get("openalex") and journal["openalex"].get("display_name"): - journal["identity_verification"]["openalex_title"] = journal["openalex"].get("display_name") - - -def _retain_preferred(journals: list[dict], preferred_titles: Sequence[str], limit: int = 15) -> list[dict]: - selected = list(journals[:limit]) - wanted = {normalise_title(title) for title in preferred_titles} - if not wanted: - return selected - selected_ids = {str(j.get("id")) for j in selected} - for journal in journals: - if normalise_title(journal.get("name", "")) not in wanted or str(journal.get("id")) in selected_ids: - continue - replace_at = next((i for i in range(len(selected)-1, -1, -1) if normalise_title(selected[i].get("name", "")) not in wanted), None) - if len(selected) < limit: - selected.append(journal) - elif replace_at is not None: - selected_ids.discard(str(selected[replace_at].get("id"))) - selected[replace_at] = journal - else: - break - selected_ids.add(str(journal.get("id"))) - order = {str(j.get("id")): i for i, j in enumerate(journals)} - return sorted(selected, key=lambda j: order.get(str(j.get("id")), len(journals))) - - -def _compact_candidate_for_rerank(journal: dict) -> dict: - return { - "id": journal.get("id"), - "title": journal.get("name"), - "publisher": journal.get("publisher"), - "matched_subjects": journal.get("matched_subjects", [])[:6], - "quartile": (journal.get("ranking") or {}).get("quartile"), - "h_index": (journal.get("ranking") or {}).get("h_index"), - "access_model": (journal.get("access_model") or {}).get("label"), - "ku_agreement_candidate": journal.get("ku_apc_candidate", False), - "base_score_ratio": journal.get("base_score_ratio", 0), - "article_similarity_score": journal.get("article_similarity_score", 0), - "recent_article_titles": [ - {"title": item.get("title"), "year": item.get("year")} - for item in (journal.get("similar_articles") or [])[:3] - ], - } - - -async def _rerank_manuscript_candidates( - manuscript: ManuscriptInput, - profile: dict, - candidates: list[dict], - record_map: dict[str, Any], - asjc_codes: Sequence[str], - preferences: DiscoveryPreferences, - language: str, -) -> tuple[list[dict], list[str], str]: - if not candidates: - return [], [], "No verified journal candidates met the applied filters." - candidate_payload = [_compact_candidate_for_rerank(j) for j in _retain_preferred(candidates, preferences.preferred_journals, 35)] - user_payload = { - "manuscript": { - "title": manuscript.title, - "abstract": manuscript.abstract, - "article_type": manuscript.article_type, - "methods": manuscript.methods, - "audience": manuscript.audience, - }, - "profile": profile, - "verified_candidates": candidate_payload, - } - parsed: dict = {} - try: - parsed = await call_openai( - get_client(), - JOURNAL_RERANK_PROMPT, - f"{lang_instruction(language)}Rerank these verified candidates.\n\n{json.dumps(user_payload, ensure_ascii=False)}", - max_tokens=7000, - model=MODEL, - ) - except Exception as exc: - logger.warning("[JournalRerank] subject-based fallback (%s)", type(exc).__name__) - parsed = {"selected": [], "next_actions": [], "global_notes": "AI reranking was unavailable. These are subject-based suggestions; methods and audience fit were not evaluated."} - - by_id = {str(j["id"]): j for j in candidates} - selected_rows = [] - seen: set[str] = set() - model_rows = parsed.get("selected") if isinstance(parsed.get("selected"), list) else [] - for item in model_rows: - if not isinstance(item, dict): - continue - sid = str(item.get("id") or "") - if sid in by_id and sid not in seen: - seen.add(sid) - selected_rows.append((by_id[sid], item)) - if len(selected_rows) >= 15: - break - # Fewer is better than padding, but always retain at least five verified - # candidates when the deterministic engine found them. - for journal in candidates: - sid = str(journal["id"]) - if len(selected_rows) >= min(15, len(candidates)): - break - if sid not in seen: - selected_rows.append((journal, {})); seen.add(sid) - - # Include eligible preferred records before scoring and final truncation. - selected_by_id = {str(j["id"]): item for j, item in selected_rows} - selected_order = [j for j, _ in selected_rows] + [j for j in candidates if str(j["id"]) not in selected_by_id] - selected_rows = [(j, selected_by_id.get(str(j["id"]), {})) for j in _retain_preferred(selected_order, preferences.preferred_journals, 15)] - output: list[dict] = [] - for journal, item in selected_rows: - record = record_map[str(journal["id"])].journal - topic = clamp_int(item.get("topic_scope_score"), 0, 40, - default=20 if journal.get("matched_subjects") else 0) - # Recent article-title overlap is a modest, transparent supporting - # signal. It can strengthen a scope score but never dominate it. - evidence_bonus = min(5, round(clamp_int(journal.get("article_similarity_score"), 0, 100) / 20)) - topic = min(40, topic + evidence_bonus) - article = clamp_int(item.get("article_type_methods_score"), 0, 20, - default=0) - audience = clamp_int(item.get("audience_score"), 0, 15, - default=0) - rank = ranking_component(record, asjc_codes, 15) - practical = practical_component( - record, preferences, 10, ku_agreement_candidate=journal.get("ku_apc_candidate", False) - ) - journal["fit_components"] = [ - {"key": "topic_scope", "label": "Topic and scope evidence", "score": topic, "max": 40}, - {"key": "article_methods", "label": "Article type and methods", "score": article, "max": 20}, - {"key": "audience", "label": "Audience alignment", "score": audience, "max": 15}, - {"key": "ranking", "label": "Indexing and subject ranking", "score": rank, "max": 15}, - {"key": "practical", "label": "Practical preferences", "score": practical, "max": 10}, - ] - journal["scoring_mode"] = "model_assisted" if item else "subject_fallback" - journal["scoring_note"] = ( - "Heuristic comparison score; not an acceptance probability." - if item else - "Subject-based fallback: methods and audience fit were not evaluated. Zero for these components means not evaluated." - ) - journal["fit_score"] = topic + article + audience + rank + practical - journal["fit_reason"] = " ".join(str(item.get("fit_reason") or "").split()) or ( - "Selected from the verified subject-filtered journal index; review the matched fields and recent articles before submission." - ) - journal["audience_match"] = " ".join(str(item.get("audience_match") or "").split()) or ( - "The journal's indexed subject fields overlap the inferred manuscript discipline." - ) - journal["submission_strategy"] = " ".join(str(item.get("submission_strategy") or "").split()) or ( - "Review the current aims and scope, recent articles, author guidelines, fees, and article-type requirements." - ) - journal["open_access_planning"] = " ".join(str(item.get("open_access_planning") or "").split()) or ( - "Use the journal access flag for initial planning, then check current fees and the Open Policy Finder sharing options." - ) - journal["article_evidence_summary"] = " ".join(str(item.get("article_evidence_summary") or "").split()) or ( - "Recent article-title evidence was unavailable." - if not journal.get("similar_articles") else - "Recent OpenAlex article titles provide supporting, not conclusive, scope evidence." - ) - output.append(journal) - - output.sort(key=lambda j: (-j.get("fit_score", 0), j.get("name", ""))) - next_actions = _text_list(parsed.get("next_actions"), 4, 500) - global_notes = " ".join(str(parsed.get("global_notes") or "").split())[:800] - return output, next_actions, global_notes - - -def _score_subject_candidates( - candidates: list[dict], - record_map: dict[str, Any], - asjc_codes: Sequence[str], - preferences: DiscoveryPreferences, - *, - exact_category: bool, -) -> list[dict]: - output = [] - for journal in candidates: - record = record_map[str(journal["id"])].journal - ratio = float(journal.get("base_score_ratio") or 0) - subject_score = round((42 if exact_category else 32) + (18 if exact_category else 20) * ratio) - evidence_bonus = min(5, round(clamp_int(journal.get("article_similarity_score"), 0, 100) / 20)) - subject_score = min(60, subject_score + evidence_bonus) - rank = ranking_component(record, asjc_codes, 25) - practical = practical_component( - record, preferences, 15, ku_agreement_candidate=journal.get("ku_apc_candidate", False) - ) - journal["fit_components"] = [ - {"key": "subject", "label": "Subject relevance", "score": subject_score, "max": 60}, - {"key": "ranking", "label": "Indexing and subject ranking", "score": rank, "max": 25}, - {"key": "practical", "label": "Practical preferences", "score": practical, "max": 15}, - ] - journal["fit_score"] = subject_score + rank + practical - fields = [row.get("field") for row in journal.get("matched_subjects") or [] if row.get("field")] - field_text = ", ".join(fields[:3]) or "the selected subject area" - journal["fit_reason"] = f"Indexed in {field_text}; ranked using verified subject and journal metadata." - journal["audience_match"] = f"Relevant to researchers and readers working in {field_text}." - journal["submission_strategy"] = "Use this as a discovery lead; review current aims and scope, recent articles, article types, and author instructions." - journal["open_access_planning"] = "The access-model flag is journal-level. Check current fees and run the Open Policy Finder policy check before planning manuscript sharing." - journal["article_evidence_summary"] = ( - "Recent OpenAlex article titles support the subject match." - if journal.get("similar_articles") else - "Recent article-title evidence was unavailable; rely on the verified subject classification and inspect the journal site." - ) - output.append(journal) - output.sort(key=lambda j: (-j.get("fit_score", 0), j.get("name", ""))) - return output - - -async def _lookup_policy_for_journal(journal: dict, *, - manuscript_version: str = "Accepted manuscript (AAM / postprint)", - intended_repository: str = "Institutional repository", intended_licence: str = "", funder: str = "", -) -> dict: - try: - record = await openpolicyfinder.lookup_policy( - journal.get("name", ""), journal.get("issn", ""), journal.get("publisher", "") - ) - if record.get("lookup_status") == "ok": - green = openpolicyfinder.to_green_oa(record, manuscript_version=manuscript_version, - intended_repository=intended_repository, intended_licence=intended_licence, funder=funder) - return { - "status": "confirmed", - "requirement_status": policy_requirement_state(green, manuscript_version=manuscript_version, - intended_repository=intended_repository, intended_licence=intended_licence, funder=funder), - "policy_evaluation": evaluate_policy(green, manuscript_version=manuscript_version, - intended_repository=intended_repository, intended_licence=intended_licence, funder=funder), - "green_oa": green, - "record": { - "title": record.get("journal_title", ""), - "publisher": record.get("publisher", ""), - "record_url": record.get("record_url", ""), - "match_quality": record.get("match_quality", ""), - "retrieved_at": record.get("retrieved_at", ""), - }, - "message": "Journal-level options retrieved from Jisc Open Policy Finder.", - } - return { - "status": "unavailable", - "requirement_status": "unknown", - "lookup_status": record.get("lookup_status", "upstream_error"), - "message": record.get("user_message") or "No matching Open Policy Finder record was retrieved.", - "verify_url": openpolicyfinder.public_search_url(journal.get("name", ""), journal.get("issn", "")), - } - except Exception as exc: - logger.warning("[JournalPolicy] lookup unavailable (%s)", type(exc).__name__) - return { - "status": "unavailable", "requirement_status": "unknown", - "message": "The journal-level policy could not be retrieved.", - "verify_url": openpolicyfinder.public_search_url(journal.get("name", ""), journal.get("issn", "")), - } - - -async def _lookup_policies(journals: Sequence[dict], limit: int = 5, *, - manuscript_version: str = "Accepted manuscript (AAM / postprint)", - intended_repository: str = "Institutional repository", intended_licence: str = "", funder: str = "", -) -> dict[str, dict]: - semaphore = asyncio.Semaphore(4) - - async def one(journal: dict) -> tuple[str, dict]: - async with semaphore: - return str(journal.get("id") or journal.get("issn") or journal.get("name")), await _lookup_policy_for_journal(journal, manuscript_version=manuscript_version, - intended_repository=intended_repository, intended_licence=intended_licence, funder=funder) - - selected = list(journals)[: max(0, limit)] - pairs = await asyncio.gather(*(one(j) for j in selected)) if selected else [] - return dict(pairs) - - -async def _apply_sharing_requirement( - journals: list[dict], - preferences: DiscoveryPreferences, -) -> tuple[list[dict], list[str]]: - if not preferences.sharing_requirement or not journals: - for journal in journals: - journal["sharing_policy"]["requirement_status"] = "not_requested" - return journals, [] - - policies = await _lookup_policies(journals, limit=min(15, len(journals)), - intended_repository=preferences.intended_repository, funder=preferences.funder) - for journal in journals: - key = str(journal.get("id")) - if key in policies: - journal["sharing_policy"] = policies[key] - else: - journal["sharing_policy"]["requirement_status"] = "unknown" - - acceptable = [j for j in journals if (j.get("sharing_policy") or {}).get("requirement_status") in {"meets", "conditional"}] - unknown = [j for j in journals if (j.get("sharing_policy") or {}).get("requirement_status") == "unknown"] - denied = [j for j in journals if (j.get("sharing_policy") or {}).get("requirement_status") == "does_not_meet"] - notes = [] - if denied: - notes.append(f"Removed {len(denied)} candidate(s) whose retrieved accepted-manuscript option did not meet the sharing requirement.") - if unknown: - notes.append(f"{len(unknown)} candidate(s) could not be confirmed and are retained after confirmed/conditional matches; missing data is not treated as permission or prohibition.") - if not acceptable: - notes.append("No candidate had a clearly applicable accepted-manuscript option; results are shown for manual review rather than claiming the requirement was satisfied.") - ordered = acceptable + unknown - return ordered, notes - - -def _finalise_discovery_result( - *, - mode: str, - ranked: list[dict], - applied_criteria: list[str], - unverified_criteria: list[str], - legacy_notes: list[str], - next_actions: list[str], - global_notes: str, - understanding: Optional[dict] = None, - subject_understanding: Optional[dict] = None, - classification_note: str = "", - preferred_titles: Sequence[str] = (), -) -> dict: - # Five evidence-rich cards plus up to ten additional verified leads. - shortlist = _retain_preferred(ranked, preferred_titles, 15) - detailed = shortlist[:5] - extended = shortlist[5:15] - actions = [a for a in next_actions if a] - actions.extend([ - "Compare two to five shortlisted journals using the decision table.", - "Check the current journal website for aims and scope, article types, fees and author instructions.", - "Run the Open Policy Finder check for shortlisted journals; use a DOI-based article check after publication.", - ]) - actions = list(dict.fromkeys(actions))[:6] - note_parts = [value for value in (global_notes, classification_note) if value] - return { - "schema_version": "journal_discovery_v2", - "mode": mode, - "manuscript_understanding": understanding, - "subject_understanding": subject_understanding, - "applied_criteria": applied_criteria, - "unverified_criteria": unverified_criteria + legacy_notes, - "journals": detailed, - "extended_list": extended, - "total": len(detailed) + len(extended), - "candidate_count_before_shortlist": len(ranked), - "next_actions": actions, - "global_notes": " ".join(note_parts) or ( - "Recommendations are decision support, not guarantees of acceptance or policy eligibility." - ), - "policy_source_mode": "on_demand_open_policy_finder", - "comparison_enabled": True, - } - - -@app.post("/api/analyze-journal") -@with_links -async def analyze_journal(req: JournalRequest, request: Request): - ip = get_client_ip(request) - if not check_rate_limit(ip, "global"): - raise HTTPException(429, "Too many requests. Please wait a moment and try again.") - - index = get_index() - if index is None: - raise HTTPException( - 503, - "The verified journal index is unavailable. ResearchBee will not generate unverified journal names as a fallback.", - ) - - manuscript = req.manuscript - preferences = _preferences_from_manuscript(manuscript) - profile = await _profile_manuscript(manuscript, req.language) - asjc_rows = resolve_asjc_rows( - index, - category=profile.get("primary_category", ""), - area=profile.get("area", ""), - additional_categories=profile.get("additional_categories") or [], - ) - query_text = _candidate_query_text(manuscript, profile) - pool = build_candidate_pool( - index, - title=manuscript.title, - abstract=query_text, - asjc_rows=asjc_rows, - preferences=preferences, - limit=50, - candidate_pool=600, - ) - - record_map = {rec.journal.scopus_id: rec for rec in pool.recommendations} - asjc_codes = [str(row.get("code")) for row in asjc_rows if row.get("code")] - candidates = [ - _candidate_base_dict(rec, asjc_codes, pool.score_ratios.get(rec.journal.scopus_id, 0.0)) - for rec in pool.recommendations - ] - evidence_query = " ".join(str(row.get("label") or row.get("name") or "") for row in asjc_rows) - if not evidence_query: - evidence_query = " ".join(v for v in [profile.get("primary_category", ""), profile.get("area", "")] if v) - await _attach_article_evidence(candidates, evidence_query, limit=12) - ranked, next_actions, global_notes = await _rerank_manuscript_candidates( - manuscript, profile, candidates, record_map, asjc_codes, preferences, req.language - ) - ranked, policy_notes = await _apply_sharing_requirement(ranked, preferences) - - unverified = list(pool.unverified_criteria) - if policy_notes: - # Replace the provisional note added by build_candidate_pool with the - # actual outcome of the Open Policy Finder pass. - unverified = [ - note for note in unverified - if not note.startswith("Accepted-manuscript sharing will be checked") - ] - unverified.extend(policy_notes) - if not asjc_rows: - unverified.append( - "No exact closed-vocabulary subject category was resolved, so candidate generation used verified full-index text matching and should be reviewed carefully." - ) - - result = _finalise_discovery_result( - mode="manuscript", - preferred_titles=preferences.preferred_journals, - ranked=ranked, - applied_criteria=pool.applied_criteria, - unverified_criteria=unverified, - legacy_notes=pool.legacy_notes, - next_actions=next_actions, - global_notes=global_notes, - understanding=profile.get("manuscript_understanding"), - classification_note=profile.get("classification_note", ""), - ) - result["classification"] = { - "confidence": profile.get("confidence"), - "primary_category": profile.get("primary_category"), - "additional_categories": profile.get("additional_categories") or [], - "area": profile.get("area"), - "asjc_codes": asjc_codes, - } - return {"result": result} - - -# ── Khazna deposit status ───────────────────────────────────────────────── -def khazna_status_for(doi: str) -> dict: - """ - THREE-VALUED on purpose. - - "could not check" and "not in Khazna" are different answers. Conflating them - would tell a researcher to deposit something already deposited, or imply an - absence we never actually verified. - """ - if not doi: - return {"checked": False, "reason": "no DOI supplied"} - source = khazna_index.status - snapshot = { - "checked_scope": "loaded_snapshot", - "snapshot_loaded_at": source.get("loaded_at"), - "snapshot_freshness": source.get("snapshot_freshness", "unknown"), - "freshness_note": source.get("freshness_note", "Source harvest date is unknown; confirm current status in Khazna."), - "retaining_previous_snapshot": source.get("retaining_previous_snapshot", False), - } - if not khazna_index.available: - return {**snapshot, "checked": False, "reason": "Khazna index unavailable"} - rec = khazna_index.get(doi) - if rec is None: - return {**snapshot, "checked": True, "in_khazna": False, - "message": "This DOI was not found in the loaded Khazna snapshot. Confirm in the live portal before creating another record."} - state = rec.get("deposit_state", "undetermined") - return { - **snapshot, - "checked": True, - "in_khazna": True, - "deposit_state": state, - "embargo_end": rec.get("embargo_end"), - "portal_url": rec.get("portal_url", ""), - "needs_deposit": state in ("metadata_only", "closed"), - "message": { - "open": "The loaded Khazna record reports an open full text. Check the portal for its current status.", - "embargoed": "In the KU repository, Khazna, with an embargoed file.", - "restricted": "In the KU repository, Khazna, but the attached file is restricted.", - "closed": "In the KU repository, Khazna, as a closed record — an open version may be permitted.", - # Khazna is currently a metadata-only CRIS: records exist for KU - # output but no files are attached. This is therefore the normal - # case, and the message is framed as an opportunity rather than a - # problem with the author's record. - "metadata_only": ("Your publication is already recorded in the KU repository, " - "Khazna, but no full text is attached. Adding the permitted " - "version makes it openly readable and discoverable."), - "undetermined": "In the KU repository, Khazna; access status could not be determined.", - }.get(state, "In the KU repository, Khazna."), - } - - -async def _advisory_note(green: dict, li: "LicenseInput", lang: str) -> str: - """The LLM writes prose from verified facts only. It never supplies a value.""" - facts = { - "permission_status": green.get("permission_status", "unknown"), - "embargo_status": green.get("embargo_status", "unknown"), - "can_archive": green.get("_can_archive"), - "versions_permitted": green.get("_versions"), - "best_version": green.get("_best_version"), - "locations": green.get("_locations"), - "embargo_months": green.get("_embargo_months"), - "embargo_stated": green.get("embargo_status") != "unknown", - "embargo_end": green.get("embargo_end"), - "licence": green.get("_licence"), - "journal_oa_type": green.get("journal_oa_type"), - "institutional_repository_permitted": green.get("_institutional_ok"), - "author_intent": { - "repository": li.intended_repository, - "licence": li.intended_licence, - "version": li.manuscript_version, - "funder": li.funder, - }, - } - try: - out = await call_openai( - get_client(), ADVISORY_NOTE_PROMPT, - lang + json.dumps(facts, ensure_ascii=False), - max_tokens=280, model=MODEL_LIGHT, - ) - return out.get("note", "") if isinstance(out, dict) else "" - except Exception as e: - logger.warning("Advisory note unavailable (%s)", type(e).__name__) - return "" - - -@app.post("/api/check-journal-policies") -async def check_journal_policies(req: JournalPolicyBatchRequest, request: Request): - """On-demand journal-level policy checks for either discovery mode. - - Up to five shortlist items are accepted per call. Every item is resolved - again against ``journals.db`` before Jisc is queried, so browser-supplied - titles, ISSNs, and publishers cannot replace the canonical journal identity. - Facts come only from the authenticated Jisc Open Policy Finder API; a miss - remains unknown. - """ - ip = get_client_ip(request) - if not check_rate_limit(ip, "global"): - raise HTTPException(429, "Too many requests. Please wait a moment and try again.") - - index = get_index() - if index is None: - raise HTTPException( - 503, - "The verified journal index is unavailable, so journal identity cannot be checked safely.", - ) - - prepared: list[dict] = [] - verified_journals: list[dict] = [] - for item in req.journals: - response_key = item.key or item.issn or item.name - record = None - try: - if item.key: - record = index.lookup_exact(scopus_id=item.key) - if record is None and item.issn: - record = index.lookup_exact(issn=item.issn) - if record is None and item.name: - record = index.lookup_exact(title=item.name) - except Exception as exc: - logger.warning("[JournalPolicy] identity lookup unavailable (%s)", type(exc).__name__) - - if ( - record is None - or not record.currently_indexed - or record.source_type != "Journal" - ): - prepared.append({ - "key": response_key, - "journal": None, - "result": { - "key": response_key, - "name": item.name, - "issn": item.issn, - "publisher": item.publisher, - "identity_verification": { - "status": "Not confirmed", - "method": "No active canonical journal record matched the submitted selection", - }, - "policy": { - "status": "unavailable", - "requirement_status": "unknown", - "message": "Policy was not queried because the journal identity could not be re-verified.", - }, - "context": { - "manuscript_version": req.manuscript_version, - "intended_repository": req.intended_repository, - "funder": req.funder, - }, - }, - }) - continue - - canonical = { - "id": response_key, - "name": record.title, - "issn": record.issn_e or record.issn_print or "", - "publisher": record.publisher_group or record.publisher or "", - "scopus_id": record.scopus_id, - } - prepared.append({"key": response_key, "journal": canonical, "result": None}) - verified_journals.append(canonical) - - policies = await _lookup_policies(verified_journals, limit=5, manuscript_version=req.manuscript_version, - intended_repository=req.intended_repository, funder=req.funder) - results = [] - for entry in prepared: - if entry["result"] is not None: - results.append(entry["result"]) - continue - journal = entry["journal"] - key = str(entry["key"]) - policy = policies.get(key) or { - "status": "unavailable", - "requirement_status": "unknown", - "message": "The journal-level policy could not be retrieved.", - } - results.append({ - "key": key, - "name": journal["name"], - "issn": journal["issn"], - "publisher": journal["publisher"], - "identity_verification": { - "status": "Confirmed", - "method": "Canonical journals.db record re-resolved before policy lookup", - "scopus_id": journal["scopus_id"], - }, - "policy": policy, - "context": { - "manuscript_version": req.manuscript_version, - "intended_repository": req.intended_repository, - "funder": req.funder, - }, - }) - return {"results": results, "source_mode": "journal_api"} - - -@app.post("/api/check-license") -@with_links -async def check_license(req: LicenseRequest, request: Request): - ip = get_client_ip(request) - if not check_rate_limit(ip, "global"): - raise HTTPException(429, "Too many requests. Please wait a moment and try again.") - - li = req.license_input - policy_context = {"manuscript_version": li.manuscript_version, - "intended_repository": li.intended_repository, - "intended_licence": li.intended_licence, "funder": li.funder} - lang = lang_instruction(req.language) - doi = doi_utils.extract_doi(li.doi) or doi_utils.extract_doi(li.notes) - if li.doi.strip() and not doi: - raise HTTPException(422, "Enter a valid DOI or a URL containing a DOI.") - - # ══ PATH A — DOI supplied: verified, article-level, no LLM policy facts ══ - if doi: - permissions, khazna = await asyncio.gather( - oaworks.fetch_permissions(doi, ror=oaworks.KU_ROR), - asyncio.to_thread(khazna_status_for, doi), - ) - green = oaworks.parse_permissions(permissions, **policy_context) - - if green is None: - # The DOI resolved but OA.Works holds no policy for it, so the - # researcher leaves without the verified answer they came for. - usage.usage.record_gap("licence_deposit", "oaworks_404") - request.state.usage_recorded = True - green = oaworks.not_confirmed( - li.journal_name, li.issn, - reason="no permission record for this DOI in OA.Works") - else: - green["repository_action_note"] = await _advisory_note(green, li, lang) - - journal = { - "name": li.journal_name or green.get("copyright_name", ""), - "issn": li.issn or ((green.get("issuer_issns") or [""])[0]), - "publisher": li.publisher or green.get("policy_issuer", ""), - "green_oa": green, - } - enriched = await enrich_journals([journal]) - - # Subject repositories, but ONLY when OA.Works says one is permitted. - # Suggesting arXiv for a journal that forbids subject-repository - # deposit would be exactly the kind of confident-but-wrong advice this - # build exists to prevent. - subject_repos = [] - if any("subject repository" in loc.lower() - for loc in green.get("_locations", [])): - sci = lookup_any([enriched[0].get("issn"), li.issn]) or {} - subject_repos = repositories.suggest_subject_repositories( - sci.get("areas", ""), sci.get("categories", "")) - - result = { - "journals": enriched, - "subject_repositories": subject_repos, - "repository_recommendation": oaworks.recommendation_from(green, **policy_context), - "next_actions": oaworks.next_actions_from(green, doi, khazna, **policy_context), - "global_notes": "", - "doi": doi, - "doi_url": doi_utils.doi_url(doi), - "shareyourpaper_url": oaworks.share_link(doi), - "khazna_status": khazna, - "source_mode": "doi", - "khazna": KHAZNA_DEPOSIT_CARD, - } - return {"result": result} - - # ══ PATH B — no DOI: authenticated Open Policy Finder journal record ══ - if not li.journal_name and not li.issn: - raise HTTPException(422, "Provide an article DOI/URL, or a journal name or ISSN.") - - issn = doi_utils.normalise_issn(li.issn) or li.issn - opf_record = await openpolicyfinder.lookup_policy( - li.journal_name, - issn, - li.publisher, - ) - lookup_status = opf_record.get("lookup_status", "upstream_error") - - if lookup_status == "ok": - green = openpolicyfinder.to_green_oa(opf_record, **policy_context) - source_mode = "journal_api" - display_name = opf_record.get("journal_title") or li.journal_name - display_issn = issn or ((opf_record.get("issns") or [""])[0]) - display_publisher = opf_record.get("publisher") or li.publisher - next_actions = oaworks.next_actions_from(green, "", **policy_context) - global_notes = ( - "This is a journal-level policy record from the authenticated Jisc " - "Open Policy Finder API. It may contain several routes with different " - "conditions. Use the DOI search above for an article-level permission " - "check and a Khazna status check." - ) - else: - reason_by_status = { - "unconfigured": "Open Policy Finder API access is not configured", - "authentication_error": "Open Policy Finder rejected the configured API key", - "rate_limited": "Open Policy Finder is temporarily rate limiting requests", - "not_found": "no matching journal record was returned by Open Policy Finder", - "identity_mismatch": "Open Policy Finder returned records that did not match the supplied journal identity", - "invalid_response": "Open Policy Finder returned an unexpected response", - "upstream_error": "Open Policy Finder is temporarily unavailable", - } - reason = reason_by_status.get(lookup_status, reason_by_status["upstream_error"]) - green = oaworks.not_confirmed(li.journal_name, issn, reason=reason) - green["source_type"] = "open_policy_finder_api" - green["policy_scope"] = "journal" - green["repository_action_note"] = ( - "Open the linked Open Policy Finder search result and check the current " - "journal record. No policy determination was made by ResearchBee." - ) - green["_verify_url"] = opf_record.get("record_url") or openpolicyfinder.public_search_url( - li.journal_name, issn - ) - green["evidence_note"] = opf_record.get("message") or ( - "No authenticated journal-level policy record was available for this request." - ) - source_mode = "journal" - display_name = li.journal_name - display_issn = issn - display_publisher = li.publisher - next_actions = oaworks.next_actions_from(green, "", **policy_context) - global_notes = green["evidence_note"] - - gap_code = { - "unconfigured": "opf_unconfigured", - "authentication_error": "opf_auth_error", - "rate_limited": "opf_rate_limited", - "not_found": "opf_not_found", - "identity_mismatch": "opf_identity_mismatch", - "invalid_response": "opf_invalid_response", - "upstream_error": "opf_upstream_error", - }.get(lookup_status, "opf_upstream_error") - usage.usage.record_gap("licence_deposit", gap_code) - request.state.usage_recorded = True - - journal = { - "name": display_name, - "issn": display_issn, - "publisher": display_publisher, - "green_oa": green, - } - enriched = await enrich_journals([journal]) - if source_mode == "journal_api" and enriched: - enriched[0].setdefault("verify_links", {})["sherpa_romeo"] = green.get( - "_verify_url", - openpolicyfinder.public_search_url(display_name, display_issn), - ) - - parsed = { - "journals": enriched, - "repository_recommendation": oaworks.recommendation_from(green, **policy_context), - "next_actions": next_actions, - "global_notes": global_notes, - "khazna_status": {"checked": False, "reason": "no DOI supplied"}, - "source_mode": source_mode, - "opf_lookup": { - "status": lookup_status, - "source": "Jisc Open Policy Finder API", - "scope": "journal", - "record_url": opf_record.get("record_url", ""), - "record_id": opf_record.get("record_id", ""), - "retrieved_at": opf_record.get("retrieved_at", ""), - }, - "khazna": KHAZNA_DEPOSIT_CARD, - } - return {"result": parsed} - - -@app.post("/api/find-repository") -async def find_repository(req: RepoRequest, request: Request): - ip = get_client_ip(request) - if not check_rate_limit(ip, "global"): - raise HTTPException(429, "Too many requests. Please wait a moment and try again.") - client = get_client() - lang = lang_instruction(req.language) - user_msg = ( - f"{lang}Select suitable repositories for this dataset FROM THE REGISTRY BELOW. " - f"Return JSON only. Use the exact \"id\" values. Do not invent repositories " - f"and do not state factual repository attributes.\n\n" - f"AVAILABLE REGISTRY:\n{registry_for_prompt()}\n\n" - f"DATASET INPUT:\n{json.dumps(req.dataset.model_dump(), indent=2)}" - ) - parsed = await call_openai(client, REPO_SYSTEM_PROMPT, user_msg, max_tokens=2000) - if "error" in parsed: - raise HTTPException(500, parsed["error"]) - - # The model selected ids and wrote fit_reasons; the SERVER supplies every - # factual field. Unknown ids are silently dropped. - constraints = req.dataset.model_dump() - if not isinstance(parsed.get("repositories", []), list): - raise HTTPException(503, "The AI service returned an invalid repository selection. Please try again.") - parsed["selection_exclusions"] = repositories.selection_exclusions( - parsed.get("repositories", []), constraints=constraints - ) - repos = hydrate_selection(parsed.get("repositories", []), constraints=constraints) - - # Khazna always leads, and is never a model choice. - repos = [r for r in repos if not r.get("is_khazna")] - if not repos: - # Only Khazna will be offered — the registry had nothing for this - # dataset. A recurring code here is a prompt to widen the registry. - usage.usage.record_gap("data_repository", "no_repository_match") - request.state.usage_recorded = True - repos.insert(0, repositories.khazna_entry(constraints=constraints)) - parsed["deposit_strategy"] = repositories.hydrate_deposit_strategy( - parsed.get("deposit_strategy"), repos, constraints=constraints - ) - for key in ("dataset_understanding", "fair_assessment"): - if key in parsed and not isinstance(parsed[key], dict): - parsed[key] = {} - for key in ("next_actions", "warnings"): - if key in parsed: - parsed[key] = _text_list(parsed[key], 8, 500) - - for r in repos: - if not r.get("is_khazna"): - clean_name = re.sub(r"[^\w\s]", "", r.get("name", "")) - r["verify_links"] = { - "re3data": f"https://www.re3data.org/search?query={clean_name}", - "fairsharing": f"https://fairsharing.org/search?q={clean_name}", - } - - parsed["repositories"] = repos - parsed["selection_schema_version"] = "release-a-id-only-v1" - parsed["registry_note"] = ( - "Repositories are selected from a registry curated by KU Library. " - "Factual details are served from verified records, not generated." - ) - return {"result": parsed} - - -# ── Subject vocabulary injection ─────────────────────────────────────────── -# -# The LLM used to be asked for an "Exact SCImago category name" without being -# shown what the exact names are, so it generated plausible ones. "Crime and -# Society" reads like a SCImago category; SCImago has no category containing -# the word "crime". The invented name matched nothing, the search fell through -# to a substring test on the area, and a criminology query returned the -# Astrophysical Journal. -# -# Same pattern as the repository finder: the model chooses from a closed list -# and the server owns the facts. The validated vocabulary is cached locally. - -_SUBJECT_VOCAB_BLOCK: Optional[str] = None - - -def subject_prompt_with_vocabulary() -> str: - """SUBJECT_NORMALISE_PROMPT plus the real category and area names.""" - global _SUBJECT_VOCAB_BLOCK - if _SUBJECT_VOCAB_BLOCK is None: - v = subject_vocabulary() - cats = sorted(v["categories"]) - areas = sorted(v["areas"]) - _SUBJECT_VOCAB_BLOCK = ( - "\n\nVALID AREAS (" + str(len(areas)) + "):\n" - + "\n".join("- " + a.title() for a in areas) - + "\n\nVALID CATEGORIES (" + str(len(cats)) + "):\n" - + "\n".join("- " + c.title() for c in cats) - ) - return SUBJECT_NORMALISE_PROMPT + _SUBJECT_VOCAB_BLOCK - - -@app.post("/api/browse-subject") -@with_links -async def browse_subject(req: SubjectRequest, request: Request): - ip = get_client_ip(request) - if not check_rate_limit(ip, "global"): - raise HTTPException(429, "Too many requests. Please wait a moment and try again.") - - from scimago_index import _subject_cache, _scimago_cache - if not _subject_cache or not _scimago_cache: - load_scimago() - load_subject_index() - - subject_text = req.subject.strip() - typed_category = subject_text if is_real_category(subject_text) else "" - typed_area = subject_text if is_real_area(subject_text) else "" - spelling_match = None if (typed_category or typed_area) else closest_subject_term(subject_text) - - # Exact vocabulary terms and conservative spelling repairs are resolved - # locally. The LLM is reserved for synonym and semantic nearest-category - # mapping, and may only return names from the closed vocabulary. - if typed_category or typed_area: - parsed = { - "confidence": "exact", - "normalised": subject_text, - "scimago_category": typed_category, - "scimago_area": typed_area, - "nearest_categories": [], - "mapping_type": "exact", - "use_nearest_category": False, - "user_message": "Matched directly to the verified ASJC/SCImago vocabulary.", - } - elif spelling_match: - corrected = str(spelling_match["name"]) - is_category = spelling_match["kind"] == "category" - parsed = { - "confidence": "near", - "normalised": corrected, - "scimago_category": corrected if is_category else "", - "scimago_area": corrected if not is_category else "", - "nearest_categories": suggest_categories(corrected, limit=4), - "mapping_type": "spelling_correction", - "use_nearest_category": True, - "user_message": ( - f'Spelling corrected: "{subject_text}" → "{corrected.title()}". ' - f'Results use this verified {spelling_match["kind"]}.' - ), - } - else: - try: - parsed = await call_openai( - get_client(), - subject_prompt_with_vocabulary(), - f"{lang_instruction(req.language)}Subject input: {subject_text}", - max_tokens=520, - model=MODEL_LIGHT, - ) - except (RateLimitError, APIStatusError, HTTPException) as exc: - logger.warning("[BrowseSubject] classifier unavailable (%s)", type(exc).__name__) - parsed = { - "confidence": "not_found", - "normalised": subject_text, - "scimago_category": "", - "scimago_area": "", - "nearest_categories": [], - "mapping_type": "none", - "use_nearest_category": False, - "user_message": "The subject classifier was unavailable. Choose a verified category suggestion or try again.", - } - except Exception as exc: - logger.warning("[BrowseSubject] classifier failed (%s)", type(exc).__name__) - parsed = { - "confidence": "not_found", - "normalised": subject_text, - "scimago_category": "", - "scimago_area": "", - "nearest_categories": [], - "mapping_type": "none", - "use_nearest_category": False, - "user_message": "The subject could not be classified safely. Choose a verified category suggestion or try again.", - } - - confidence = parsed.get("confidence", "not_found") - if not isinstance(confidence, str) or confidence not in {"exact", "near", "not_found"}: - confidence = "not_found" - mapping_type = str(parsed.get("mapping_type") or "").strip().lower() - if mapping_type not in {"exact", "synonym", "spelling_correction", "nearest_category", "none"}: - mapping_type = "nearest_category" if confidence == "near" else "none" - use_nearest_category = parsed.get("use_nearest_category") is True - normalised = str(parsed.get("normalised") or subject_text).strip() - category = str(parsed.get("scimago_category") or "").strip() - area = str(parsed.get("scimago_area") or "").strip() - user_message = " ".join(str(parsed.get("user_message") or "").split()) - - nearest_categories: list[str] = [] - nearest_seen: set[str] = set() - for name in _text_list(parsed.get("nearest_categories"), 6, 200): - value = str(name).strip() - key = value.lower() - if key and key not in nearest_seen and is_real_category(value): - nearest_seen.add(key) - nearest_categories.append(value) - - bad_category = bool(category) and not is_real_category(category) - bad_area = bool(area) and not is_real_area(area) - if bad_category: - logger.warning("[BrowseSubject] discarded unlisted category %r", category) - category = "" - if bad_area: - logger.warning("[BrowseSubject] discarded unlisted area %r", area) - area = "" - - # Exact typed terms always win over a model's alternative valid term. - if typed_category: - category = typed_category - area = "" - confidence = "exact" - mapping_type = "exact" - if typed_area: - area = typed_area - category = "" - confidence = "exact" - mapping_type = "exact" - - # A model may conservatively return the chosen nearest category only in - # nearest_categories. Use it only when the model explicitly says the - # fallback is safe, or when it labelled the response "near". A true - # not_found with use_nearest_category=false remains a no-result response. - used_nearest_fallback = False - if not category and not area and nearest_categories and ( - use_nearest_category or confidence == "near" - ): - category = nearest_categories[0] - confidence = "near" - mapping_type = "nearest_category" - used_nearest_fallback = True - if not normalised or normalised == subject_text: - normalised = category - - selected_term = category or area - if selected_term: - display_term = selected_term.title() - if mapping_type == "spelling_correction": - disclosure = ( - f'Spelling corrected: "{subject_text}" → "{display_term}". ' - f'Results use this verified {"category" if category else "broad area"}.' - ) - elif mapping_type == "nearest_category" or used_nearest_fallback: - disclosure = ( - f'SCImago and Scopus have no exact subject category for "{subject_text}". ' - f'ResearchBee mapped it to the nearest verified category "{display_term}".' - ) - elif mapping_type == "synonym" and selected_term.lower() != subject_text.lower(): - disclosure = ( - f'Interpreted "{subject_text}" as the verified ' - f'{"category" if category else "area"} "{display_term}".' - ) - elif confidence == "near" and selected_term.lower() != subject_text.lower(): - disclosure = ( - f'Interpreted "{subject_text}" using the closest verified ' - f'{"category" if category else "area"} "{display_term}".' - ) - else: - disclosure = "" - if disclosure and disclosure.lower() not in user_message.lower(): - user_message = f"{disclosure} {user_message}".strip() - - def suggestions() -> list[dict]: - out: list[str] = [] - seen: set[str] = set() - for name in nearest_categories: - key = str(name).strip().lower() - if key and key not in seen and is_real_category(key): - seen.add(key) - out.append(key) - for name in suggest_categories(subject_text, limit=8): - key = str(name).strip().lower() - if key and key not in seen and is_real_category(key): - seen.add(key) - out.append(key) - counts = subject_vocabulary()["categories"] - return [{"category": name.title(), "n_journals": counts.get(name, 0)} for name in out[:6]] - - have_category = bool(category) - # Permit a corrected or LLM-resolved broad area when no category is - # available. Exact typed areas continue to take precedence. - have_area = bool(area) and not have_category - if confidence == "not_found" or (not have_category and not have_area): - usage.usage.record_gap( - "journal_subject", - "category_invented" if bad_category else "subject_unrecognised", - ) - request.state.usage_recorded = True - nearby = suggestions() - base = f"SCImago and Scopus have no exact subject category for '{subject_text}'." - hint = ( - " Closest verified categories: " + ", ".join(item["category"] for item in nearby) + "." - if nearby else "" - ) - return {"result": { - "schema_version": "journal_discovery_v2", - "mode": "subject", - "confidence": "not_found", - "normalised": normalised, - "user_message": (user_message + " " if user_message else "") + base + hint, - "subject_mapping": { - "original_input": subject_text, - "normalised": normalised, - "match_type": "none", - "selected_category": "", - "selected_area": "", - "used_nearest_category": False, - "spelling_corrected": False, - "alternatives": nearby, - }, - "suggestions": nearby, - "journals": [], - "extended_list": [], - "total": 0, - }} - - index = get_index() - if index is None: - raise HTTPException( - 503, - "The verified journal index is unavailable. ResearchBee will not generate unverified journal names as a fallback.", - ) - - preferences = _preferences_from_subject(req) - asjc_rows = resolve_asjc_rows( - index, - category=category if have_category else "", - area=area if have_area else "", - ) - query_text = _subject_query_text(req.subject, normalised, category, area) - pool = build_candidate_pool( - index, - title=query_text, - abstract="", - asjc_rows=asjc_rows, - preferences=preferences, - limit=50, - candidate_pool=600, - ) - record_map = {rec.journal.scopus_id: rec for rec in pool.recommendations} - asjc_codes = [str(row.get("code")) for row in asjc_rows if row.get("code")] - candidates = [ - _candidate_base_dict(rec, asjc_codes, pool.score_ratios.get(rec.journal.scopus_id, 0.0)) - for rec in pool.recommendations - ] - await _attach_article_evidence(candidates, query_text, limit=12) - ranked = _score_subject_candidates( - candidates, record_map, asjc_codes, preferences, exact_category=have_category - ) - ranked, policy_notes = await _apply_sharing_requirement(ranked, preferences) - - unverified = list(pool.unverified_criteria) - if policy_notes: - unverified = [ - note for note in unverified - if not note.startswith("Accepted-manuscript sharing will be checked") - ] - unverified.extend(policy_notes) - if have_area and not have_category: - area_note = ( - f"'{subject_text}' was resolved to the broad subject area " - f"'{area.title()}'. Results span that area; choose a narrower category for a more specific list." - ) - user_message = f"{user_message} {area_note}".strip() - usage.usage.record_gap("journal_subject", "area_fallback") - request.state.usage_recorded = True - - all_suggestions = suggestions() - alternatives = [ - item for item in all_suggestions - if item["category"].lower() != str(category).lower() - ][:4] - result = _finalise_discovery_result( - mode="subject", - preferred_titles=preferences.preferred_journals, - ranked=ranked, - applied_criteria=pool.applied_criteria, - unverified_criteria=unverified, - legacy_notes=pool.legacy_notes, - next_actions=[], - global_notes=user_message, - subject_understanding={ - "summary": f"Verified journal discovery for {normalised}.", - "discipline": normalised, - "category": category, - "area": area, - "confidence": confidence, - "mapping_type": mapping_type, - "original_input": subject_text, - "alternatives": alternatives, - }, - classification_note=( - "This is a broad area result, not a manuscript-specific fit assessment." - if have_area and not have_category else - "Subject relevance is based on the verified ASJC/SCImago category mapping." - ), - ) - result.update({ - "confidence": confidence, - "normalised": normalised, - "scimago_category": category, - "scimago_area": area, - "user_message": user_message, - "asjc_codes": asjc_codes, - "subject_mapping": { - "original_input": subject_text, - "normalised": normalised, - "match_type": mapping_type, - "selected_category": category, - "selected_area": area, - "used_nearest_category": mapping_type == "nearest_category" or used_nearest_fallback, - "spelling_corrected": mapping_type == "spelling_correction", - "alternatives": alternatives, - }, - "suggestions": alternatives if confidence == "near" else [], - }) - if not ranked: - result["filter_no_match"] = True - result["global_notes"] = ( - "The subject was recognised, but no active journal met every selected hard filter. " - "ResearchBee did not silently relax those filters." - ) - return {"result": result} - - -@app.post("/api/export-citations") -async def export_citations(req: ExportRequest): - fmt = req.format.lower() - journals = [item.model_dump() for item in req.journals] - - if fmt == "bibtex": - lines = [] - for i, j in enumerate(journals): - key = f"journal{i + 1}" - def bibtext(value): - return str(value or "").replace("\\", " ").replace("{", "(").replace("}", ")").replace("\n", " ").replace("\r", " ") - j = {name: bibtext(value) if name != "ranking" else value for name, value in j.items()} - lines.append( - f"@misc{{{key},\n" - f" title = {{{j.get('name', '')}}},\n" - f" howpublished = {{Journal shortlist record, not an article citation}},\n" - f" publisher = {{{j.get('publisher', '')}}},\n" - f" issn = {{{j.get('issn', '')}}},\n" - f" note = {{Quartile: {j.get('quartile') or (j.get('ranking') or {}).get('quartile', '—')}. Recommended by ResearchBee.}}\n}}" - ) - return {"content": "\n\n".join(lines), "filename": "journals.bib", "mimetype": "text/plain"} - - elif fmt == "ris": - lines = [] - for j in journals: - j = {key: " ".join(value.split()) if isinstance(value, str) else value for key, value in j.items()} - q = j.get("quartile") or (j.get("ranking") or {}).get("quartile", "") - lines.append( - f"TY - GEN\nTI - {j.get('name', '')}\n" - f"PB - {j.get('publisher', '')}\nSN - {j.get('issn', '')}\n" - f"N1 - Journal shortlist record, not an article citation. Quartile: {q}. Recommended by ResearchBee.\nER - " - ) - return {"content": "\n\n".join(lines), "filename": "journals.ris", "mimetype": "text/plain"} - - raise HTTPException(400, "Format must be bibtex or ris") - - -@app.post("/api/cover-letter") -async def generate_cover_letter(req: CoverLetterRequest): - client = get_client() - lang = "Respond in Arabic." if req.language == "arabic" else "Respond in English." - user_msg = ( - f"{lang}\n\nGenerate a cover letter for the following submission:\n\n" - f"Manuscript title: {req.manuscript_title}\nAbstract: {req.abstract}\n" - f"Article type: {req.article_type}\nTarget journal: {req.journal_name}\n" - f"Publisher: {req.publisher or 'Not specified'}\nDiscipline: {req.discipline or 'Not specified'}\n" - f"Author name: {req.author_name or 'The Author(s)'}\n\nReturn JSON only." - ) - parsed = await call_openai(client, COVER_LETTER_PROMPT, user_msg, max_tokens=800, model=MODEL_LIGHT) - if parsed.get("error"): - raise HTTPException(503, parsed["error"]) - if not isinstance(parsed.get("cover_letter"), str) or not parsed["cover_letter"].strip(): - raise HTTPException(503, "The AI service returned an invalid cover-letter draft. Please try again.") - parsed["cover_letter"] = parsed["cover_letter"][:12000] - parsed["subject_line"] = str(parsed.get("subject_line") or "Manuscript submission")[:500] - parsed["requires_author_review"] = True - parsed["review_note"] = "Verify every submission declaration and replace the bracketed author-confirmation placeholders before use." - return {"result": parsed} - - -# ── Chat endpoint — streaming SSE ───────────────────────────────────────── -@app.post("/api/chat") -async def chat(req: ChatRequest, request: Request): - """ - Three-layer resolution with streaming: - 1. Keyword pre-filter — instant routing, no LLM call - 2. Open Policy Finder — live policy lookup for specific journals - 3. LLM stream — tokens stream via SSE; route extracted from full reply at end - """ - ip = get_client_ip(request) - if not check_rate_limit(ip, "chat"): - raise HTTPException(429, "Chat limit reached. Please wait a minute before sending more messages.") - - if not req.messages: - async def empty_stream(): - greeting = "Hi! How can I help you with publishing or Open Access today?" - yield f'data: {json.dumps({"token": greeting, "done": False})}\n\n' - yield f'data: {json.dumps({"token": "", "done": True, "reply": greeting, "route": None, "prefill": {}})}\n\n' - return StreamingResponse(empty_stream(), media_type="text/event-stream", - headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}) - - last_msg = req.messages[-1].content.strip() - - # ── Layer 1: keyword pre-filter — instant, no streaming needed ──────── - fast_route = quick_route(last_msg) - if fast_route: - TAB_DESCRIPTIONS = { - "journals": "Journal Submission tool", - "license": "License Checking tool", - "data": "Data Repository tool", - } - reply = ( - f"The {TAB_DESCRIPTIONS[fast_route]} above is perfect for this! " - f"It'll give you accurate, structured results in seconds." - ) - if req.language == "arabic": - reply = f"أداة {TAB_DESCRIPTIONS[fast_route]} المتاحة أعلاه مناسبة تماماً لهذا! ستمنحك نتائج دقيقة ومنظمة في ثوانٍ." - # Return as SSE with route - async def fast_stream(): - payload = json.dumps({"token": reply, "done": False}) - yield f"data: {payload}\n\n" - done = json.dumps({"token": "", "done": True, "route": fast_route, "prefill": {}}) - yield f"data: {done}\n\n" - return StreamingResponse(fast_stream(), media_type="text/event-stream", - headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}) - - # ── Layer 2: Open Policy Finder ─────────────────────────────────────── - opf_data = "" - journal_name = detect_journal_policy_question(last_msg) - if journal_name: - opf_data = await fetch_opf_policy(journal_name) - - # ── Layer 3: LLM streaming ──────────────────────────────────────────── - client = get_client() - - try: - retrieved_knowledge = get_chat_knowledge(last_msg, context_query=" ".join(m.content for m in req.messages[-5:-1] if m.role == "user")[-2000:]) - except Exception as exc: # noqa: BLE001 - chat should degrade safely - logger.warning("[Chat] Knowledge retrieval unavailable (%s)", type(exc).__name__) - retrieved_knowledge = ( - "The curated knowledge index is temporarily unavailable. Give only stable general " - "guidance, do not invent current policies or links, and direct KU-specific questions " - "to the KU Library." - ) - - system = CHAT_SYSTEM_PROMPT_BASE.replace( - "{RETRIEVED_KNOWLEDGE}", retrieved_knowledge - ).replace( - "{OPF_DATA}", opf_data if opf_data else "No live Open Policy Finder data was retrieved for this query." - ) - - if req.language == "arabic": - system += "\n\nRespond entirely in Arabic." - - messages = [{"role": "system", "content": system}] - # Preserve the latest turn while bounding total model context independently - # of the browser's history representation. - history = [] - remaining = 24000 - for message in reversed(req.messages): - text = message.content[-min(remaining, 12000):] - if not text or remaining <= 0: - break - history.append({"role": message.role, "content": text}) - remaining -= len(text) - messages += list(reversed(history)) - - def extract_route(text: str): - """Brace-counting extraction of routing JSON from end of reply.""" - starts = [m.start() for m in re.finditer(r'\{', text)] - for start in reversed(starts): - candidate = text[start:] - depth = 0; end = None - for i, ch in enumerate(candidate): - if ch == '{': depth += 1 - elif ch == '}': - depth -= 1 - if depth == 0: end = i + 1; break - if end is None: continue - try: - parsed = json.loads(candidate[:end]) - if "route" in parsed: - return text[:start].strip(), parsed.get("route"), parsed.get("prefill", {}) - except Exception: - continue - return text.strip(), None, {} - - # Create streaming response outside generator to catch connection errors early - try: - openai_stream = await client.chat.completions.create( - model=MODEL, - messages=messages, - max_tokens=600, - temperature=0.3, - stream=True, - ) - except Exception as e: - logger.warning("[Chat] AI request failed (%s)", type(e).__name__) - raise HTTPException(503, "AI service is temporarily unavailable. Please try again.") from None - - request.state.usage_recorded = True - async def stream_response() -> AsyncGenerator[str, None]: - full_text = "" - try: - async for chunk in openai_stream: - token = chunk.choices[0].delta.content if chunk.choices else None - if token: - full_text += token - yield f'data: {json.dumps({"token": token, "done": False})}\n\n' - - reply, route, prefill = extract_route(full_text) - if not isinstance(route, (str, type(None))) or route not in ("journals", "license", "data", None): - route = None - - done_payload = json.dumps({ - "token": "", "done": True, - "reply": reply, - "route": route, - "prefill": prefill if isinstance(prefill, dict) else {}, - }) - usage.usage.record("chat", "ok") - yield f"data: {done_payload}\n\n" - - except asyncio.CancelledError: - usage.usage.record("chat", "miss", "cancelled") - raise - except Exception as e: - usage.usage.record("chat", "error", "stream_error") - logger.warning("[Chat] Stream failed (%s)", type(e).__name__) - err = json.dumps({"token": "", "error": "The answer was interrupted. Please try again.", "done": True, "route": None, "prefill": {}}) - yield f"data: {err}\n\n" - finally: - await openai_stream.close() - - return StreamingResponse( - stream_response(), - media_type="text/event-stream", - headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}, - ) - - -# ── Journal meta lookup — multi-suggestion with ISSN auto-fill ─────────── -@app.get("/api/lookup-journal-meta") -async def lookup_journal_meta(request: Request, name: str): - """ - Multi-suggestion journal lookup: - Layer 1a: SCImago — up to 4 ranked title matches (name + ISSN + publisher) - Layer 1b: OpenAlex — up to 4 results, merged + deduplicated with SCImago - Layer 2: LLM — only if nothing found; returns up to 3 name suggestions only - (NO ISSN from LLM — those come from re-running Layer 1 on acceptance) - """ - ip = get_client_ip(request) - if not check_rate_limit(ip, "global"): - raise HTTPException(429, "Too many requests.") - - name = name.strip() - if len(name) > 300: - raise HTTPException(422, "Journal name must contain at most 300 characters.") - if not name or len(name) < 3: - return {"suggestions": [], "llm_suggestions": []} - - name_lower = name.lower() - words = [w for w in name_lower.split() if len(w) > 2] - - def score_title(title: str) -> float: - """Score a title against the query — higher = better match.""" - t = title.lower() - if t == name_lower: - return 1.0 - # Word overlap score - overlap = sum(1 for w in words if w in t) - # Bonus for starts-with - starts = 0.2 if t.startswith(name_lower[:6]) else 0 - # Penalty for very different lengths - len_ratio = min(len(name_lower), len(t)) / max(len(name_lower), len(t)) - return (overlap / max(len(words), 1)) * 0.6 + starts + len_ratio * 0.2 - - # ── Layer 1a: SCImago — collect top 4 scored matches ───────────────── - rows = load_subject_index() - scored = [] - for r in rows: - s = score_title(r["title"]) - if s > 0.35: # threshold — avoid noise - scored.append((s, r)) - scored.sort(key=lambda x: -x[0]) - top_scimago = scored[:4] - - suggestions = [] - seen_titles = set() - - for score, r in top_scimago: - t = r["title"] - if t.lower() in seen_titles: - continue - seen_titles.add(t.lower()) - ni = norm_issn(r.get("issn", "")) - suggestions.append({ - "title": t, - "issn": ni or r.get("issn", ""), - "publisher": r.get("publisher", ""), - "source": "SCImago", - "confidence": "high" if score > 0.8 else "medium", - "score": round(score, 3), - }) - - # ── Layer 1b: OpenAlex — up to 4, merge with SCImago results ───────── - try: - oa_client = openalex._get_client() - r = await oa_client.get( - "https://api.openalex.org/sources", - params={"search": name, "per_page": 4, "filter": "type:journal"}, - timeout=6.0, - ) - if r.is_success: - for s in r.json().get("results", []): - title = s.get("display_name", "") - if not title or title.lower() in seen_titles: - continue - issns = s.get("issn", []) - issn_l = s.get("issn_l", "") - publisher = s.get("host_organization_name", "") - best_issn = issn_l or (issns[0] if issns else "") - ni = norm_issn(best_issn) - sc = score_title(title) - seen_titles.add(title.lower()) - suggestions.append({ - "title": title, - "issn": ni or best_issn, - "publisher": publisher, - "source": "OpenAlex", - "confidence": "high" if sc > 0.8 else "medium", - "score": round(sc, 3), - }) - except Exception as e: - logger.warning("[LookupJournal] OpenAlex unavailable (%s)", type(e).__name__) - - # Re-sort merged list by score, keep top 4 - suggestions.sort(key=lambda x: -x.get("score", 0)) - suggestions = suggestions[:4] - - # If we have good suggestions, return them - if suggestions: - return {"suggestions": suggestions, "llm_suggestions": []} - - # ── Layer 2: LLM — name suggestions only, NO ISSN ──────────────────── - llm_suggestions = [] - try: - client = get_client() - resp = await client.chat.completions.create( - model=MODEL, - response_format={"type": "json_object"}, - messages=[ - { - "role": "system", - "content": ( - "You are an academic journal name expert. " - "The user has typed a journal name that may be misspelled or abbreviated. " - "Return ONLY valid JSON: " - "{suggestions: [{name: string, confidence: high|medium|low}]} " - "Return up to 3 most likely correct journal names, ranked by confidence. " - "Only include journals you are confident exist. " - "Do NOT return ISSN, publisher, or any other fields. " - "Return JSON only." - ), - }, - {"role": "user", "content": f"Journal name typed by user: {name}"}, - ], - max_tokens=150, - temperature=0.1, - ) - raw = resp.choices[0].message.content or "{}" - parsed = json.loads(raw) - for s in parsed.get("suggestions", [])[:3]: - n = s.get("name", "").strip() - c = s.get("confidence", "low") - if n and n.lower() != name_lower and c in ("high", "medium"): - llm_suggestions.append({"name": n, "confidence": c}) - except Exception as e: - logger.warning("[LookupJournal] AI lookup unavailable (%s)", type(e).__name__) - - return {"suggestions": [], "llm_suggestions": llm_suggestions} - - -# ── Static frontend ──────────────────────────────────────────────────────── -@app.get("/") -async def root(): - if os.path.exists("static/index.html"): - return FileResponse("static/index.html") - return {"status": "ResearchBee API running", "docs": "/docs"} - - -if os.path.exists("static"): - if os.path.exists("static/css"): - app.mount("/css", StaticFiles(directory="static/css"), name="css") - if os.path.exists("static/js"): - app.mount("/js", StaticFiles(directory="static/js"), name="js") - - -class JournalLinksRequest(BaseModel): - scopus_id: str = Field("", max_length=30) - issn: str = Field("", max_length=20) - -@app.post("/api/journal-links") -async def find_author_guidelines(req: JournalLinksRequest, request: Request): - if not check_rate_limit(get_client_ip(request), "global"): - raise HTTPException(429, "Too many requests. Please try again later.") - return await discover(req.model_dump()) +""" +ResearchBee - FastAPI backend v1.8.3 (nearest-subject matching, re3data supplement) + +Changes in v1.8.1: +- Subject browsing applies conservative closed-vocabulary spelling correction before the LLM. +- Meaningful unlisted subjects may be mapped to the nearest verified category with explicit disclosure. +- Subject responses include structured mapping provenance and alternative verified categories. +- The backend version aligns with the unified comparison/OA front end. + +Changes in v1.8.0: +- Browse by subject and manuscript matching now share one database-first journal pipeline. +- Journal identities, ISSNs, metrics, active status and access flags come from journals.db. +- Hard filters are enforced in code; unsupported top-percent and fee claims are not inferred. +- Results include explainable fit components, recent OpenAlex article evidence, comparison data, + and on-demand Jisc Open Policy Finder checks. +- Accepted-manuscript sharing requirements trigger policy checks for leading candidates. + +Changes in v1.7.0: +- Chat knowledge is stored as source-aware topic records and retrieved per question. +- Educational journal, metrics, publishing, OA, and RDM questions are answered in chat. +- Fast routing is limited to explicit tool requests so it no longer hides useful guidance. +- The chat prompt no longer claims unrestricted web browsing. + +Changes in v1.6.0: +- Journal-level licence checks use the authenticated Jisc Open Policy Finder API. +- The DOI-first OA.Works + Khazna path remains unchanged and primary. +- The former public-page renderer has been removed. +- Open Policy Finder policy facts are displayed as source data and are never generated by an LLM. + +Changes from v1.3 (verified-sources release): +- /api/check-license is now DOI-first and GROUNDED. Policy facts come from the + OA.Works Permissions API (the dataset behind cOAlition S's Journal Checker + Tool), never from the LLM. The LLM writes advisory prose only. +- Khazna deposit status via a nightly OAI-PMH harvest (three-valued: checked / + in_khazna / needs_deposit — "could not check" is never reported as "absent"). +- Every policy answer carries a provenance label. Any LLM-derived + policy_status is forcibly downgraded to "Not confirmed". +- Repository finder constrained to a curated registry; the LLM selects and + explains, the server supplies the facts. + +Changes from v1.2: +- Rate limiting middleware (sliding window, 20 req/min global, 10/min chat per IP) +- /api/chat now streams via SSE (StreamingResponse) +- /api/openalex-works — related literature per journal (OpenAlex) +- /api/altmetric — journal attention score badge (Altmetric free API) +""" +import os +import json +import re +import asyncio +import logging +import secrets +from trusted_links import with_links, discover +import time +from contextlib import asynccontextmanager +from typing import Optional, List, AsyncGenerator, Any, Sequence, Literal + +# Process boot time, captured at import so it is set before any heavy +# module-level work and before lifespan runs. Exposed on /api/health so +# the uptime monitor can distinguish "still running" from "died and came +# back between polls" — a restarted container reports RUNNING and 200 at +# both polls, but its boot_ts moves. +BOOT_TS = time.time() + +from fastapi import FastAPI, HTTPException, Request +from fastapi.middleware.cors import CORSMiddleware +from fastapi.staticfiles import StaticFiles +from fastapi.responses import FileResponse, StreamingResponse +from pydantic import BaseModel, Field, model_validator + +from openai import AsyncOpenAI, RateLimitError, APIStatusError +import httpx +from urllib.parse import quote_plus + +# journal_lookup wraps scimago_index and serves lookups from journals.db +# (Scopus source list x SCImago 2025). It forwards every other name straight +# through to scimago_index, so the subject-vocabulary functions below behave +# exactly as before. If journals.db is unreadable it falls back to the CSV, +# so this import cannot take the Space down. +from journal_lookup import ( + lookup_any, load_scimago, search_by_subject, load_subject_index, + subject_vocabulary, is_real_category, is_real_area, suggest_categories, + closest_subject_term, nearest_subject_term, get_index, +) +import openalex +import openpolicyfinder +from prompts import ( + JOURNAL_SYSTEM_PROMPT, JOURNAL_PROFILE_PROMPT, JOURNAL_RERANK_PROMPT, + REPO_SYSTEM_PROMPT, SUBJECT_NORMALISE_PROMPT, COVER_LETTER_PROMPT, + CHAT_SYSTEM_PROMPT_BASE, +) +from chat_knowledge import get_chat_knowledge, knowledge_status +from utils import norm_issn +from journal_discovery import ( + DiscoveryPreferences, build_candidate_pool, resolve_asjc_rows, + matching_subjects, quartile_for, access_model, ranking_component, + practical_component, policy_requirement_state, clamp_int, normalise_title, +) + +# ── Verified-source modules (no LLM inside any of these) ────────────────── +import oaworks +import doi_utils +import repositories +import re3data +from khazna_loader import khazna_index +from repositories import REGISTRY, registry_for_prompt, hydrate_selection +from prompts import ADVISORY_NOTE_PROMPT + +# ── Usage counting + admin dashboard ────────────────────────────────── +# usage.py counts how often each tool is used. It stores COUNTS ONLY — +# no question text, no abstracts, no DOIs — see the module docstring. +import usage +from admin import router as admin_router +from runtime_security import RuntimeSecurityMiddleware, check_rate_limit, get_client_ip +from policy_evaluation import evaluate_policy + +logger = logging.getLogger(__name__) + +# ── Constants ────────────────────────────────────────────────────────────── +# Models are server-controlled; browser input cannot select an arbitrary model. +MODEL = os.getenv("RESEARCHBEE_MODEL", "gpt-4.1-mini") +MODEL_LIGHT = os.getenv("RESEARCHBEE_MODEL_LIGHT", "gpt-4.1-nano") +RELEASE_VERSION = "1.8.3" + +# ── KU publisher-agreement candidates ───────────────────────────────────── +# Publisher matching is a discovery signal only. It does not establish title- +# level APC eligibility, author eligibility, quota availability, article-type +# coverage, or an exact top-percentile requirement. +# Aliases that are ordinary English and would over-match as substrings. +# These only count when they lead the publisher string. +AMBIGUOUS_ALIASES = { + "academic press", "saunders", "masson", "adis", "informa", + "nature research", "nature publishing", +} + +KU_APC_PUBLISHER_GROUPS = [ + # ACS + {"american chemical society"}, + # Elsevier — all regional entities + {"elsevier", "elsevier b.v.", "elsevier bv", "elsevier ltd", "elsevier inc.", + "elsevier inc", "elsevier ireland ltd", "elsevier gmbh", "elsevier masson", + "elsevier singapore", "elsevier australia", "elsevier beijing", + "elsevier editora", "elsevier espana", + # Imprints whose SCImago string never contains "Elsevier". + # NB: normalisation strips dots, so "W.B. Saunders" becomes "wb saunders" + # — that variant must be listed explicitly or ~108 titles are missed. + "academic press", "cell press", "wb saunders", "w b saunders", + "churchill livingstone", "mosby", "pergamon", "saunders", + "masson", "urban and fischer", "hanley and belfus"}, + # IEEE — all society variants + {"ieee", "ieee advancing technology", "ieee communications society", + "ieee computational intelligence", "ieee computer society", + "ieee electron devices", "ieee microwave"}, + # Springer Nature — all Springer variants + {"springer", "springer nature", "springernature", "nature portfolio", + "nature publishing", "springer international publishing", + "springer international publishing ag", "springer berlin", "springer boston", + "springer china", "springer netherlands", "springer new york", + "springer london", "springer japan", "springer verlag", + "springer science and business media", "springer publishing", + "springer science and business media llc", "springer nature america", + "biomed central", "palgrave", "palgrave macmillan", "adis", + "birkhauser", "nature research"}, + # Taylor & Francis — all T&F variants (note: & in SCImago CSV) + {"taylor and francis", "taylor & francis", "taylor & francis", + "routledge", "routledge, taylor", "taylor and francis ltd", + "taylor and francis inc", "taylor and francis a.s", + "taylor and francis asia pacific", "informa", "informa uk", + "informa uk limited", "psychology press", "crc press", + "dove medical press", "taylor and francis group"}, + # Wiley — all Wiley variants + {"wiley", "wiley-blackwell", "wiley-blackwell publishing", + "john wiley and sons", "john wiley & sons", "john wiley & sons", + "wiley-liss", "wiley-vch", "wiley - vch", + "blackwell", "blackwell publishing", "blackwell science", + "john wiley and sons inc", "john wiley and sons ltd"}, +] + +def is_ku_agreement_publisher(publisher: str) -> bool: + """Whether the publisher string matches a KU agreement publisher group. + + This is only a publisher-level candidate signal. It is never treated as + title-level APC eligibility, quota availability, or a top-percentile check. + """ + pub = (publisher or "").lower().strip() + if not pub: + return False + + def norm(value: str) -> str: + return (value.lower() + .replace("&", "&").replace("&", "and") + .replace(".", "").replace(",", "") + .replace("-", " ").strip()) + + pub_norm = norm(pub) + pub_tokens = pub_norm.split() + for group in KU_APC_PUBLISHER_GROUPS: + for alias in group: + alias_norm = norm(alias) + alias_tokens = alias_norm.split() + if pub_norm == alias_norm: + return True + if alias_norm in AMBIGUOUS_ALIASES: + if pub_norm.startswith(alias_norm + " "): + return True + continue + n = len(alias_tokens) + for i in range(len(pub_tokens) - n + 1): + if pub_tokens[i:i + n] == alias_tokens: + return True + return False + + +def is_ku_apc_covered(publisher: str, quartile: str = "") -> bool: + """Backward-compatible candidate flag; not a coverage determination.""" + return quartile == "Q1" and is_ku_agreement_publisher(publisher) + + +# Request identity and bounded quotas are shared with runtime_security. + +KHAZNA_RELOAD_HEADER = "X-ResearchBee-Reload-Token" + + +def require_khazna_reload_auth(request: Request) -> None: + """Require the Space secret before allowing a Khazna index reload.""" + expected = os.environ.get("KHAZNA_RELOAD_TOKEN", "").strip() + if not expected: + raise HTTPException( + 503, + "Khazna reload authentication is not configured.", + ) + + supplied = request.headers.get(KHAZNA_RELOAD_HEADER, "").strip() + if not supplied: + auth = request.headers.get("Authorization", "").strip() + if auth.lower().startswith("bearer "): + supplied = auth[7:].strip() + + if not supplied: + raise HTTPException( + 401, + "Khazna reload token required.", + headers={"WWW-Authenticate": "Bearer"}, + ) + if not secrets.compare_digest(supplied, expected): + raise HTTPException(403, "Invalid Khazna reload token.") + + + +# ── Khazna article workflow contact card ───────────────────────────────── +KHAZNA_DEPOSIT_CARD = { + "title": "🏛️ Deposit to Khazna — KU Institutional Repository", + "url": "https://khazna.ku.ac.ae", + "contact": "khazna@ku.ac.ae", + "library_url": "https://library.ku.ac.ae/lib", + "message": "Register your publication with Khazna. Deposit a manuscript file only when the checked version, repository location, licence and embargo conditions permit it; contact KU Library when permission is unclear.", + "metadata_tip": "Even if depositing data in a domain-specific repository, always register metadata in Khazna so your work appears in KU's research portfolio.", +} + +FAST_ROUTE_PATTERNS = { + "journals": [ + r"\b(?:find|recommend|suggest|match|identify|show me|give me)\b.{0,100}\bjournal(?:s)?\b", + r"\bwhere (?:should|can) i publish\b", + r"\banaly[sz]e my (?:paper|manuscript|abstract)\b", + r"\bbrowse journals?\b", + r"\bopen (?:the )?journal (?:finder|tool|tab)\b", + ], + "license": [ + r"\b(?:open|use|show|take me to)\b.{0,50}\b(?:license|sharing|self[- ]?archiv) (?:checker|tool|tab)\b", + r"\bopen policy checker\b", + ], + "data": [ + r"\b(?:find|recommend|suggest|match|identify|show me|give me)\b.{0,100}\b(?:data )?repositor(?:y|ies)\b", + r"\bwhere (?:should|can) i deposit (?:my )?(?:data|dataset)\b", + r"\bwhich repository (?:should|can) i use for (?:my )?(?:data|dataset)\b", + r"\bopen (?:the )?(?:data )?repository (?:finder|tool|tab)\b", + ], +} + +_INFORMATIONAL_OPENERS = ( + "what is", "what are", "what does", "how does", "how do", "how can", + "why is", "why are", "explain", "tell me about", "difference between", + "compare", "which metric", "when should", "tips for", "criteria for", +) + + +def quick_route(message: str) -> Optional[str]: + """Return a tab only for an explicit structured-tool request. + + The previous broad keyword router sent questions such as "What is FAIR + data?" or "How do I choose the best journal?" straight to a tab. That made + an expanded knowledge base unreachable. Informational questions now go to + the assistant, while clear action requests still receive an instant route. + """ + msg = " ".join(message.lower().split()) + if not msg: + return None + if msg.startswith(_INFORMATIONAL_OPENERS): + return None + for tab, patterns in FAST_ROUTE_PATTERNS.items(): + if any(re.search(pattern, msg, flags=re.IGNORECASE) for pattern in patterns): + return tab + return None + + +# ── Open Policy Finder (Jisc) authenticated API ────────────────────────── +async def fetch_opf_policy(journal_name: str, issn: Optional[str] = None) -> str: + """Return a grounded journal-level summary for the chat assistant.""" + record = await openpolicyfinder.lookup_policy(journal_name, issn or "") + return openpolicyfinder.format_for_chat(record) + + +def detect_journal_policy_question(message: str) -> Optional[str]: + """ + Detect if the user is asking about a specific journal's self-archiving policy. + Returns the journal name if detected, else None. + Uses simple heuristics — good enough for 90% of cases. + """ + msg = message.lower() + policy_triggers = [ + "self-archiv", "self archiv", "can i deposit", "can i post", + "embargo", "postprint", "accepted manuscript", "preprint policy", + "archiving rights", "green oa", "repository policy", + "version of record", "aam", "open access rights", + ] + if not any(t in msg for t in policy_triggers): + return None + + # Try to extract journal name — look for quoted names or "in X journal" patterns + quoted = re.findall(r'["\']([^"\']{3,80})["\']', message) + if quoted: + return quoted[0] + + patterns = [ + r'\bin\s+([A-Z][A-Za-z &\-]{3,60}(?:Journal|Review|Letters|Reports|Science|Nature|Communications|Research)?)', + r'\bfor\s+([A-Z][A-Za-z &\-]{3,60}(?:Journal|Review|Letters|Reports|Science|Nature|Communications|Research)?)', + r'\b([A-Z][A-Za-z &\-]{3,60}(?:Journal|Review|Letters|Reports|Science|Nature|Communications|Research))\b', + ] + for pat in patterns: + m = re.search(pat, message) + if m: + candidate = m.group(1).strip() + if len(candidate) > 3: + return candidate + return None + + +# ── Singleton OpenAI client ──────────────────────────────────────────────── +_openai_client: Optional[AsyncOpenAI] = None + + +def get_client() -> AsyncOpenAI: + global _openai_client + key = os.environ.get("OPENAI_API_KEY") + if not key: + raise HTTPException(500, "OPENAI_API_KEY not configured") + if _openai_client is None: + _openai_client = AsyncOpenAI(api_key=key, timeout=45.0, max_retries=0) + return _openai_client + + +# ── Lifespan ─────────────────────────────────────────────────────────────── +async def refresh_khazna() -> None: + """A native coroutine is scheduled on the event loop by APScheduler.""" + await asyncio.to_thread(khazna_index.reload) + + +@asynccontextmanager +async def lifespan(app: FastAPI): + load_scimago() + load_subject_index() + openalex.init_client() + re3data.init_client() + await openpolicyfinder.init_client() + usage.usage.start() + refresh_task = asyncio.create_task(refresh_khazna()) + scheduler = None + try: + from apscheduler.schedulers.asyncio import AsyncIOScheduler + from apscheduler.triggers.cron import CronTrigger + scheduler = AsyncIOScheduler(timezone="Asia/Dubai") + scheduler.add_job( + refresh_khazna, + CronTrigger(hour=23, minute=30, timezone="Asia/Dubai"), + id="khazna_refresh", replace_existing=True, max_instances=1, + ) + scheduler.start() + logger.info("Khazna refresh scheduled for 23:30 Asia/Dubai") + except Exception as exc: + logger.warning("Khazna scheduler unavailable (%s)", type(exc).__name__) + app.state.khazna_scheduler = scheduler + try: + yield + finally: + if scheduler: + scheduler.shutdown(wait=False) + if not refresh_task.done(): + refresh_task.cancel() + await asyncio.gather(refresh_task, return_exceptions=True) + await usage.usage.stop() + await openpolicyfinder.close_client() + await openalex.close_client() + await re3data.close_client() + global _openai_client + if _openai_client is not None: + await _openai_client.close() + _openai_client = None + + +# ── App ──────────────────────────────────────────────────────────────────── +app = FastAPI(title="ResearchBee API", version=RELEASE_VERSION, lifespan=lifespan) + +app.add_middleware(RuntimeSecurityMiddleware) +app.add_middleware( + CORSMiddleware, + allow_origins=[ + "https://ku-library.github.io", + # The ResearchBee fragment is embedded in LibGuides and calls this + # API from the guide's own origin. Without these entries every + # request from the guide fails in the browser and the tools appear + # to do nothing when submitted. + "https://kustar.libguides.com", + "https://libguides.ku.ac.ae", + "http://localhost", + "http://127.0.0.1", + "http://localhost:3000", + "http://127.0.0.1:3000", + ], + expose_headers=["Retry-After"], + allow_methods=["GET", "POST", "OPTIONS"], + allow_headers=["Content-Type", "Authorization", "X-ResearchBee-Reload-Token"], +) + +# Counting observes completed responses; lifespan owns the usage task lifecycle. +usage.install(app) + +# Admin dashboard at /admin. Set RESEARCHBEE_ADMIN_PASSWORD in the Space +# secrets; without it the dashboard cannot be opened at all. +app.include_router(admin_router, prefix="/admin") + + +# ── Helpers ──────────────────────────────────────────────────────────────── +def lang_instruction(language: str) -> str: + return "Please respond entirely in Arabic." if language == "arabic" else "" + + +def build_verify_links(name: str, issn: Optional[str], sourceid: str = "") -> dict: + q = re.sub(r"[^\w\s]", "", name).strip() + q_encoded = q.replace(" ", "%20") + ni = norm_issn(issn) + return { + "scimago": (f"https://www.scimagojr.com/journalsearch.php?q={ni}&tip=iss" + if ni else f"https://www.scimagojr.com/journalsearch.php?q={q_encoded}&tip=jou"), + "sherpa_romeo": (f"https://openpolicyfinder.jisc.ac.uk/search?search={q_encoded}" + "&per_page=10&publication_page=1&publisher_page=1&funder_page=1"), + "doaj": (f"https://doaj.org/toc/{ni.replace('-', '')}" + if ni else f"https://doaj.org/search/journals?source=%7B%22query%22%3A%7B%22query_string%22%3A%7B%22query%22%3A%22{q_encoded}%22%7D%7D%7D"), + "openalex": (f"https://openalex.org/sources/issn:{ni}" + if ni else f"https://openalex.org/sources?filter=display_name.search:{q_encoded}"), + "scopus_sources": "https://www.scopus.com/sources", + "issn_display": ni or "", + "name_display": name.strip(), + } + + +def _index_extras(scimago: Optional[dict]) -> dict: + """ + journals.db-only fields, guarded. + + Additive metadata must never be able to fail a request. If anything in + here raises, the caller still gets a complete ranking block from the + fields that predate the journals.db work. The warning names the journal so + a recurring failure is traceable rather than silent. + """ + if not scimago: + return {} + try: + return { + "quartiles_by_field": scimago.get("quartiles_by_field") or [], + "sjr": scimago.get("sjr"), + "indexation_state": scimago.get("state"), + "indexation_message": scimago.get("state_message"), + "warn": bool(scimago.get("warn")), + "currently_indexed": scimago.get("currently_indexed", True), + "successor_title": scimago.get("successor_title"), + "oa_diamond": bool(scimago.get("oa_diamond")), + "from_scopus": bool(scimago.get("from_scopus")), + "from_scimago": bool(scimago.get("from_scimago", True)), + } + except Exception as e: # noqa: BLE001 + logger.warning("[JournalIndex] extras unavailable (%s)", type(e).__name__) + return {} + + +def _quartile_interpretation(scimago: Optional[dict]) -> str: + """ + Per-field quartile sentence, guarded, with the original wording as the + fallback. 46% of ranked journals rank differently across their own subject + areas, so the spread is worth showing — but never at the cost of the call. + """ + try: + by_field = (scimago or {}).get("quartiles_by_field") or [] + if len(by_field) > 1: + spread = ", ".join(f"{f['quartile']} in {f['field']}" for f in by_field) + return f"Quartile varies by subject area: {spread}." + except Exception as e: # noqa: BLE001 + logger.warning("[JournalIndex] quartile spread unavailable (%s)", type(e).__name__) + return (f"Quartile verified from journal metadata " + f"(2025 dataset, matched by ISSN {(scimago or {}).get('matched_issn', '')}).") + + +def enrich_ranking(journal: dict, scimago: Optional[dict], oa_pct: Optional[dict]) -> dict: + if scimago: + q = scimago.get("quartile", "") + out = { + "requested": True, "source": "Verified from journal data", "year": "2025", + "category": scimago.get("categories", ""), "metric_value": None, + "percentile": f"Rank #{scimago.get('rank', '—')} globally" if scimago.get("rank") else None, + "quartile": q if q and q != "-" else None, + "interpretation": _quartile_interpretation(scimago), + "verification_status": "Confirmed", "h_index": scimago.get("h_index"), + } + # journals.db extras are merged in, never inlined. If the index or its + # shape misbehaves the response is still complete and correct — the + # caller simply does not get the extra keys. + out.update(_index_extras(scimago)) + return out + elif oa_pct: + return { + "requested": True, "source": "OpenAlex-derived percentile", "year": "2025", + "category": oa_pct.get("concept"), "metric_value": None, + "percentile": f"{oa_pct['percentile']}th percentile within {oa_pct['peer_count']} {oa_pct['concept']} journals", + "quartile": oa_pct.get("quartile"), + "interpretation": "OpenAlex-derived percentile — NOT a Scopus/SCImago quartile.", + "verification_status": "OpenAlex-derived (not a Scopus/SCImago metric)", + } + else: + return { + "requested": True, "source": "Not available", "year": None, + "category": None, "metric_value": None, "percentile": None, "quartile": None, + "interpretation": "No journal metrics match found. Use verify links to check Scopus Sources directly.", + "verification_status": "Not confirmed — verify manually via Scopus Sources link", + } + + +def _scimago_title_fallback(title: str) -> Optional[dict]: + """ + Fallback: fuzzy title match in SCImago when ISSN lookup fails. + Used when LLM returns a wrong/hallucinated ISSN. + """ + if not title: + return None + rows = load_subject_index() + title_lower = title.lower().strip() + # Exact title match first + for r in rows: + if r["title"].lower() == title_lower: + ni = norm_issn(r.get("issn", "")) + if ni: + return lookup_any([ni]) + # Partial match — title contains all significant words + words = [w for w in title_lower.split() if len(w) > 3] + if len(words) >= 2: + for r in rows: + rt = r["title"].lower() + if all(w in rt for w in words): + ni = norm_issn(r.get("issn", "")) + if ni: + result = lookup_any([ni]) + if result: + return result + return None + + +async def enrich_journals(journals: list) -> list: + enriched_oa = await openalex.enrich_all(journals) + result = [] + for j in enriched_oa: + issn = j.get("issn") + oa = j.get("openalex") or {} + all_issns = [issn, oa.get("issn_l")] + (oa.get("issns") or []) + scimago = lookup_any(all_issns) + + # ── Title fallback: if ISSN lookup fails, try matching by journal name ── + if not scimago: + scimago = _scimago_title_fallback(j.get("name", "")) + if scimago: + logger.info(f"[Enrich] Title fallback matched: {j.get('name')}") + + oa_pct = j.get("oa_percentile") + final_issn = (scimago or {}).get("matched_issn") or oa.get("issn_l") or issn + j["issn"] = final_issn + j["ranking"] = enrich_ranking(j, scimago, oa_pct) + sourceid = (scimago or {}).get("sourceid", "") + j["verify_links"] = build_verify_links(j.get("name", ""), final_issn, sourceid) + j.pop("oa_percentile", None) + result.append(j) + return result + + +def enrich_extended(extended: list) -> list: + result = [] + for j in extended: + issn = j.get("issn") + scimago = lookup_any([issn]) if issn else None + q = (scimago or {}).get("quartile", "") + h_index = (scimago or {}).get("h_index", "") + publisher= (scimago or {}).get("publisher", "") or j.get("publisher", "") + oa = (scimago or {}).get("open_access", "No") + country = (scimago or {}).get("country", "") + sourceid = (scimago or {}).get("sourceid", "") + j["quartile"] = q if q and q != "-" else None + j["h_index"] = int(h_index) if str(h_index).isdigit() else None + j["publisher"] = publisher + j["open_access"] = oa + j["country"] = country + + # warn is True ONLY for the 967 titles Scopus discontinued for cause. + # It is deliberately False for the 6,855 that were merely renamed: + # telling a researcher that a journal which changed its name is a + # quality risk is worse than saying nothing at all. + # + # Merged through the guarded helper for the same reason as above: a + # quick-match row must not be able to fail the whole request. + j.update(_index_extras(scimago)) + + j["verify_links"] = build_verify_links(j.get("name", ""), issn, sourceid) + j["ku_apc_covered"] = is_ku_apc_covered(publisher, q) + result.append(j) + return result + +_POLICY_ACTION_RE = re.compile( + r"(?:self[- ]?archiv|embargo|deposit|repository|licen[cs]e|preprint|postprint|" + r"accepted manuscript|version of record|green oa|\baam\b)", + re.IGNORECASE, +) + + +def enforce_journal_policy_guard(parsed: dict) -> dict: + """ + Make the journal-finder workflow incapable of returning LLM policy facts. + + The model may rank and explain journals, but every policy-bearing field is + replaced with a canonical, three-valued "unknown" response. This guard is + deliberately server-side so a future prompt regression cannot reintroduce + apparent permissions, embargoes, or licences. + """ + if not isinstance(parsed, dict): + return {} + + for journal in parsed.get("journals") or []: + if not isinstance(journal, dict): + continue + green = oaworks.not_confirmed( + journal.get("name", ""), + journal.get("issn", ""), + reason="Journal Finder evaluates fit only; no article-level permission lookup was performed", + ) + green["repository_action_note"] = ( + "Use ResearchBee License Checking with the article DOI before depositing any manuscript version." + ) + green["evidence_note"] = ( + "No article-level permission lookup is performed by Journal Finder. " + "Policy values from the language model are discarded by the server." + ) + green["risk_flag"] = "Deposit rights not checked — verify with the article DOI." + journal["green_oa"] = green + journal["oa_compliance_note"] = ( + "Journal Finder does not determine self-archiving rights. Use the DOI-based " + "License Checking tool before depositing any version." + ) + + unknown = oaworks.not_confirmed( + reason="Journal Finder does not perform an article-level permission lookup" + ) + parsed["repository_recommendation"] = oaworks.recommendation_from(unknown) + + safe_actions = [] + for action in parsed.get("next_actions") or []: + action = " ".join(str(action or "").split()) + if action and not _POLICY_ACTION_RE.search(action): + safe_actions.append(action[:600]) + safe_actions.extend([ + "Verify the journal scope, author guidelines, current fees, and indexation before submission.", + "Use ResearchBee License Checking with the article DOI before depositing any manuscript version.", + ]) + # Stable de-duplication. + parsed["next_actions"] = list(dict.fromkeys(safe_actions)) + parsed["global_notes"] = ( + "Journal Finder evaluates publication fit and verified journal metrics only. " + "Self-archiving permissions, embargoes, and licences require a DOI-based check." + ) + parsed["policy_source_mode"] = "not_checked_in_journal_finder" + return parsed + + +def build_subject_journal_result(journal: dict) -> dict: + """Build one subject-browse row, including all DB-backed field quartiles.""" + issn = journal.get("issn") + scimago = lookup_any([issn]) if issn else None + if not scimago: + scimago = _scimago_title_fallback(journal.get("title", "")) + + source = scimago or journal + final_issn = source.get("matched_issn") or issn or "" + quartile = source.get("quartile") or journal.get("quartile", "") + h_index = source.get("h_index", journal.get("h_index")) + try: + h_index = int(h_index) if h_index not in (None, "") else None + except (TypeError, ValueError): + h_index = None + + result = { + "name": journal.get("title", ""), + "issn": final_issn, + "quartile": quartile if quartile and quartile != "-" else None, + "h_index": h_index, + "publisher": source.get("publisher", "") or journal.get("publisher", ""), + "open_access": source.get("open_access", journal.get("open_access", "No")), + "country": source.get("country", journal.get("country", "")), + "categories": source.get("categories", journal.get("categories", "")), + "verify_links": build_verify_links( + journal.get("title", ""), + final_issn, + source.get("sourceid", journal.get("sourceid", "")), + ), + } + result.update(_index_extras(scimago)) + result["ku_apc_covered"] = is_ku_apc_covered(result["publisher"], quartile) + return result + + +# ── OpenAI calls: bounded retries, validated top-level JSON, redacted logs ── +async def call_openai(client: AsyncOpenAI, system: str, user: str, max_tokens: int = 4096, model: str = None) -> dict: + chosen_model = model or MODEL + for attempt in range(3): + try: + response = await client.chat.completions.create( + model=chosen_model, + response_format={"type": "json_object"}, + messages=[{"role": "system", "content": system}, {"role": "user", "content": user}], + max_tokens=max_tokens, + temperature=0.3, + ) + content = response.choices[0].message.content or "" + if getattr(response.choices[0], "finish_reason", None) == "length": + logger.warning("Model response truncated: model=%s chars=%d", chosen_model, len(content)) + return {"error": "The answer was cut off before it could be completed. Please try again."} + try: + parsed = json.loads(content) + except (json.JSONDecodeError, TypeError): + logger.warning("Invalid model JSON: model=%s chars=%d", chosen_model, len(content)) + return {"error": "The AI service returned an unreadable answer. Please try again."} + if not isinstance(parsed, dict) or parsed.get("error"): + return {"error": "The AI service returned an unexpected answer. Please try again."} + return parsed + except RateLimitError as exc: + # Inspect only the machine code, never expose/log provider response text. + code = str(getattr(exc, "code", "") or "") + if code in {"insufficient_quota", "billing_hard_limit_reached"}: + raise HTTPException(503, "AI capacity is unavailable. Please contact the library.") from None + if attempt == 2: + raise HTTPException(429, "AI service is busy. Please try again shortly.", headers={"Retry-After": "30"}) from None + except APIStatusError as exc: + status = getattr(exc, "status_code", 500) + if status < 500 or attempt == 2: + logger.warning("AI upstream HTTP status=%s", status) + raise HTTPException(503, "AI service is temporarily unavailable. Please try again.") from None + except Exception as exc: + logger.warning("AI request failed (%s)", type(exc).__name__) + raise HTTPException(503, "AI service is temporarily unavailable. Please try again.") from None + await asyncio.sleep(2 ** attempt) + raise HTTPException(503, "AI service is temporarily unavailable.") + + +def _text_list(value: Any, limit: int = 10, width: int = 300) -> list[str]: + """Validate model-authored list shape before downstream use.""" + if not isinstance(value, list): + return [] + return [" ".join(v.split())[:width] for v in value if isinstance(v, str) and v.strip()][:limit] + + +# ── Request models ───────────────────────────────────────────────────────── +_T300 = Field("", max_length=300) +_T500 = Field("", max_length=500) + + +class ManuscriptInput(BaseModel): + title: str = Field(..., max_length=300) + abstract: str = Field(..., max_length=5000) + keywords: str = _T500 + discipline: str = _T300 + article_type: str = Field("Original research article", max_length=100) + methods: str = _T500 + audience: str = _T300 + funder: str = _T300 + institution: str = _T300 + country: str = _T300 + apc_budget: str = _T300 + # v1.8 names. Access model and manuscript-sharing rights are intentionally + # separate. The legacy fields remain accepted for older LibGuides copies. + access_preference: str = Field("Any journal", max_length=80) + sharing_requirement: bool = False + intended_repository: str = Field("Institutional repository", max_length=100) + ranking_preference: str = Field("No filter", max_length=50) + preferred_journals: str = _T500 + avoid_journals: str = _T500 + oa_preference: str = Field("", max_length=50) + ranking_source: str = Field("", max_length=50) + speed_preference: str = Field("", max_length=50) + repository_target: str = Field("", max_length=100) + + +class JournalRequest(BaseModel): + manuscript: ManuscriptInput + model: str = Field("gpt-4o-mini", max_length=30) + language: str = Field("english", max_length=10) + + +class LicenseInput(BaseModel): + # journal_name is no longer required: a DOI alone is a complete request, + # and the endpoint raises 422 if neither is supplied. + journal_name: str = Field("", max_length=300) + doi: str = Field("", max_length=400) # DOI or article URL + issn: str = Field("", max_length=20) + publisher: str = _T300 + manuscript_version: str = Field("All versions", max_length=50) + funder: str = _T300 + intended_repository: str = Field("Institutional repository", max_length=100) + intended_licence: str = Field("CC BY", max_length=50) + notes: str = _T500 + + +class LicenseRequest(BaseModel): + license_input: LicenseInput + model: str = Field("gpt-4o-mini", max_length=30) + language: str = Field("english", max_length=10) + + +class DatasetInput(BaseModel): + title: str = Field(..., max_length=300) + description: str = Field(..., max_length=2000) + discipline: str = Field(..., max_length=300) + data_types: str = Field(..., max_length=300) + file_formats: str = _T300 + approx_size: str = _T300 + sensitivity: str = Field("Open", max_length=50) + contains_personal_data: str = Field("No", max_length=50) + licence_intent: str = Field("CC BY", max_length=50) + funder: str = _T300 + institution: str = _T300 + country: str = _T300 + needs_doi: str = Field("Yes", max_length=10) + needs_versioning: str = Field("Yes", max_length=10) + embargo_required: Literal["Yes", "No", "Unsure"] = "No" + embargo_months: Optional[int] = Field(None, ge=0, le=120) + preferred_repository: str = _T300 + related_publication: str = _T300 + notes: str = _T500 + + +class RepoRequest(BaseModel): + dataset: DatasetInput + model: str = Field("gpt-4o-mini", max_length=30) + language: str = Field("english", max_length=10) + + +class SubjectRequest(BaseModel): + subject: str = Field(..., min_length=1, max_length=200) + ranking_preference: str = Field("No filter", max_length=50) + access_preference: str = Field("Any journal", max_length=80) + sharing_requirement: bool = False + intended_repository: str = Field("Institutional repository", max_length=100) + apc_budget: str = _T300 + funder: str = _T300 + preferred_journals: str = _T500 + avoid_journals: str = _T500 + model: str = Field("gpt-4o-mini", max_length=30) + language: str = Field("english", max_length=10) + + +class JournalPolicyItem(BaseModel): + key: str = Field("", max_length=200) + name: str = Field(..., min_length=1, max_length=300) + issn: str = Field("", max_length=20) + publisher: str = _T300 + + +class JournalPolicyBatchRequest(BaseModel): + journals: List[JournalPolicyItem] = Field(..., min_length=1, max_length=5) + manuscript_version: str = Field("Accepted manuscript (AAM / postprint)", max_length=80) + intended_repository: str = Field("Institutional repository", max_length=100) + funder: str = _T300 + language: str = Field("english", max_length=10) + + +class CoverLetterRequest(BaseModel): + manuscript_title: str = Field(..., max_length=300) + abstract: str = Field(..., max_length=5000) + journal_name: str = Field(..., max_length=300) + publisher: str = Field("", max_length=300) + article_type: str = Field("Original research article", max_length=100) + author_name: str = Field("", max_length=200) + discipline: str = Field("", max_length=300) + language: str = Field("english", max_length=10) + model: str = Field("gpt-4o-mini", max_length=30) + + +class ExportRanking(BaseModel): + quartile: Optional[str] = Field(None, max_length=10) + + +class ExportJournal(BaseModel): + name: str = Field(..., min_length=1, max_length=300) + publisher: str = Field("", max_length=500) + issn: Optional[str] = Field("", max_length=50) + quartile: Optional[str] = Field(None, max_length=10) + ranking: Optional[ExportRanking] = None + + +class ExportRequest(BaseModel): + journals: List[ExportJournal] = Field(..., min_length=1, max_length=30) + format: Literal["bibtex", "ris"] = "bibtex" + + +class ChatMessage(BaseModel): + role: str = Field(..., pattern="^(user|assistant)$") + content: str = Field(..., max_length=12000) + + @model_validator(mode="after") + def validate_role_limit(self): + if self.role == "user" and len(self.content) > 2000: + raise ValueError("User messages must contain at most 2000 characters.") + return self + + +class ChatRequest(BaseModel): + messages: List[ChatMessage] = Field(..., max_length=20) + language: str = Field("english", max_length=10) + + @model_validator(mode="after") + def validate_history(self): + if self.messages and self.messages[-1].role != "user": + raise ValueError("The final chat message must be from the user.") + return self + + +# ── Routes ───────────────────────────────────────────────────────────────── +def _journal_index_status() -> dict: + """Whether journals.db opened, for /api/health. Never raises.""" + try: + import journal_lookup + ok = journal_lookup.index_available() + if not ok: + return {"loaded": False, "fallback": "scimago CSV"} + meta = journal_lookup._get_index().meta() + return { + "loaded": True, + "journals": int(meta.get("n_journals", 0)), + "active": int(meta.get("n_active", 0)), + "discontinued": int(meta.get("n_discontinued", 0)), + "built": meta.get("built_at", "")[:10], + } + except Exception as e: # noqa: BLE001 + return {"loaded": False, "error": "Journal index could not be opened"} + + +@app.get("/api/health") +async def health(): + """ + Service status plus data-source provenance. + + The `sources` block distinguishes local readiness, configuration and + observed upstream status. It does not actively probe external services. + """ + now = time.time() + index_status = _journal_index_status() + try: + scimago_count = len(load_subject_index()) + except Exception: + scimago_count = 0 + ready = bool(index_status.get("loaded")) and scimago_count > 0 + return { + "status": "ok" if ready else "degraded", + "ready": ready, + "liveness": "ok", + "service": "ResearchBee", + "version": RELEASE_VERSION, + # Restart detection for the uptime monitor. boot_ts is the stable + # identity of this process; when it changes between two polls the + # container was replaced, whatever the HTTP status said. + "boot_ts": round(BOOT_TS, 3), + "uptime_s": round(now - BOOT_TS, 1), + "knowledge_base": knowledge_status(), + "sources": { + "oaworks": { + "endpoint": oaworks.OA_WORKS_BASE + oaworks.PERMISSIONS_PATH, + "auth": "none required", + "ku_ror_set": bool(oaworks.KU_ROR), + }, + "khazna": khazna_index.status, + "repository_registry": {"entries": len(REGISTRY)}, + "re3data": re3data.status(), + "open_policy_finder": openpolicyfinder.status(), + "scimago": {"loaded": scimago_count > 0, "records": scimago_count, "metric_year": 2025}, + "openalex": openalex.status(), + "openai": {"configured": bool(os.getenv("OPENAI_API_KEY")), "availability": "not_probed"}, + # False here means journals.db did not open and every lookup is + # falling back to the CSV — the Space still works, but none of the + # Scopus data is reaching users. Worth alerting on. + "journal_index": index_status, + }, + } + + +@app.post("/api/admin/reload-khazna") +async def reload_khazna(request: Request): + """ + Force an immediate reload of the Khazna index. + + Called by the harvest workflow after a successful publish, so the Space + picks up new data within seconds instead of waiting for the 23:30 refresh. + Idempotent and harmless: it only re-reads a public dataset. + """ + require_khazna_reload_auth(request) + ip = get_client_ip(request) + if not check_rate_limit(ip, "global"): + raise HTTPException(429, "Too many requests.") + ok = await asyncio.to_thread(khazna_index.reload) + return {"reloaded": ok, "khazna": khazna_index.status} + + +# ── OpenAlex related works per journal ──────────────────────────────────── +@app.get("/api/openalex-works") +async def openalex_works(request: Request, source_id: str, per_page: int = 5): + """ + Fetch recent highly-cited works from an OpenAlex source (journal). + source_id: OpenAlex source ID e.g. S137773608 + """ + ip = get_client_ip(request) + if not check_rate_limit(ip, "global"): + raise HTTPException(429, "Too many requests. Please wait a moment.") + + clean_id = str(source_id or "").strip().rsplit("/", 1)[-1] + if clean_id.startswith("S"): + clean_id = clean_id[1:] + if not re.fullmatch(r"[0-9]{1,20}", clean_id): + raise HTTPException(400, "Invalid OpenAlex source_id") + + try: + client = openalex._get_client() + r = await client.get( + "https://api.openalex.org/works", + params={ + "filter": f"primary_location.source.id:S{clean_id},is_retracted:false", + "sort": "cited_by_count:desc", + "per_page": min(max(per_page, 1), 8), + "select": "id,title,publication_year,doi,cited_by_count,authorships", + }, + timeout=6.0, + ) + if not r.is_success: + raise HTTPException(503, "OpenAlex is temporarily unavailable. Please try again.") + + works = [] + for w in r.json().get("results", []): + authors = [ + a.get("author", {}).get("display_name", "") + for a in (w.get("authorships") or [])[:3] + ] + works.append({ + "title": w.get("title", ""), + "year": w.get("publication_year"), + "doi": w.get("doi", ""), + "cited_by": w.get("cited_by_count", 0), + "authors": authors, + "openalex_url": w.get("id", ""), + }) + return {"works": works, "status": "available" if works else "no_results"} + except HTTPException: + raise + except Exception as e: + logger.warning("OpenAlex works unavailable (%s)", type(e).__name__) + raise HTTPException(503, "OpenAlex is temporarily unavailable. Please try again.") from None + + +@app.get("/api/openalex-cited-by") +async def openalex_cited_by(request: Request, work_id: str, per_page: int = 5): + """Return the most-cited papers that cite one validated OpenAlex work id.""" + ip = get_client_ip(request) + if not check_rate_limit(ip, "global"): + raise HTTPException(429, "Too many requests. Please wait a moment.") + + clean_id = str(work_id or "").strip().replace("https://openalex.org/", "") + if not re.fullmatch(r"W\d+", clean_id): + raise HTTPException(400, "Invalid OpenAlex work_id") + + try: + client = openalex._get_client() + response = await client.get( + "https://api.openalex.org/works", + params={ + "filter": f"cites:{clean_id},is_retracted:false", + "sort": "cited_by_count:desc", + "per_page": min(max(per_page, 1), 8), + "select": "id,title,publication_year,doi,cited_by_count,authorships", + }, + timeout=6.0, + ) + if not response.is_success: + raise HTTPException(503, "OpenAlex is temporarily unavailable. Please try again.") + + works = [] + for work in response.json().get("results", []): + authors = [ + authorship.get("author", {}).get("display_name", "") + for authorship in (work.get("authorships") or [])[:3] + ] + works.append({ + "title": work.get("title", ""), + "year": work.get("publication_year"), + "doi": work.get("doi", ""), + "cited_by": work.get("cited_by_count", 0), + "authors": [author for author in authors if author], + "openalex_url": work.get("id", ""), + }) + return {"works": works, "status": "available" if works else "no_results"} + except HTTPException: + raise + except Exception as exc: + logger.warning("OpenAlex cited-by unavailable (%s)", type(exc).__name__) + raise HTTPException(503, "OpenAlex is temporarily unavailable. Please try again.") from None + + +# ── Trending papers by concept/topic ───────────────────────────────────── +@app.get("/api/trending-papers") +async def trending_papers(request: Request, concept: str = "", concept_id: str = "", per_page: int = 5): + """ + Fetch most-cited papers in last 3 years for a given concept/topic via OpenAlex. + concept_id: OpenAlex concept ID (e.g. C41008148) — preferred + concept: display name fallback (e.g. "Machine Learning") + """ + ip = get_client_ip(request) + if not check_rate_limit(ip, "global"): + raise HTTPException(429, "Too many requests.") + + if not concept and not concept_id: + return {"works": [], "status": "no_results"} + + if len(concept) > 200 or (concept_id and not re.fullmatch(r"C?[0-9]{1,20}", concept_id)): + raise HTTPException(400, "Invalid subject identifier") + try: + from datetime import datetime + year_cutoff = datetime.now().year - 3 + client = openalex._get_client() + + # Build filter — prefer concept_id for accuracy + if concept_id: + clean_id = concept_id.strip().lstrip("C") + filter_str = f"concepts.id:C{clean_id},publication_year:>{year_cutoff},is_retracted:false" + else: + # Use keyword search via default_search + filter_str = f"publication_year:>{year_cutoff},is_retracted:false" + + params = { + "filter": filter_str, + "sort": "cited_by_count:desc", + "per_page": min(max(per_page, 1), 8), + "select": "id,title,publication_year,doi,cited_by_count,authorships,primary_location", + } + + # If only concept name, use search param instead of filter + if not concept_id and concept: + params["search"] = concept + params["filter"] = f"publication_year:>{year_cutoff},is_retracted:false" + + r = await client.get( + "https://api.openalex.org/works", + params=params, + timeout=8.0, + ) + if not r.is_success: + raise HTTPException(503, "OpenAlex is temporarily unavailable. Please try again.") + + works = [] + for w in r.json().get("results", []): + authors = [ + a.get("author", {}).get("display_name", "") + for a in (w.get("authorships") or [])[:3] + ] + journal = (w.get("primary_location") or {}).get("source", {}) or {} + works.append({ + "title": w.get("title", ""), + "year": w.get("publication_year"), + "doi": w.get("doi", ""), + "cited_by": w.get("cited_by_count", 0), + "authors": [a for a in authors if a], + "journal": journal.get("display_name", ""), + "openalex_url": w.get("id", ""), + }) + return {"works": works, "status": "available" if works else "no_results"} + except HTTPException: + raise + except Exception as e: + logger.warning("OpenAlex recent papers unavailable (%s)", type(e).__name__) + raise HTTPException(503, "OpenAlex is temporarily unavailable. Please try again.") from None + + +# ── Altmetric journal attention ─────────────────────────────────────────── +@app.get("/api/altmetric") +async def altmetric_score(request: Request, issn: str): + """Legacy endpoint: no verified journal-level aggregate is available.""" + return {"score": None, "url": None, "issn": norm_issn(issn), "status": "not_supported", + "message": "Article-level attention cannot be reported as a journal-level score."} + + +# ── Unified verified journal discovery (v1.8) ───────────────────────────── +_JOURNAL_PROFILE_VOCAB_BLOCK: Optional[str] = None + + +def journal_profile_prompt_with_vocabulary() -> str: + global _JOURNAL_PROFILE_VOCAB_BLOCK + if _JOURNAL_PROFILE_VOCAB_BLOCK is None: + vocabulary = subject_vocabulary() + categories = sorted(vocabulary["categories"]) + areas = sorted(vocabulary["areas"]) + _JOURNAL_PROFILE_VOCAB_BLOCK = ( + "\n\nVALID AREAS (" + str(len(areas)) + "):\n" + + "\n".join("- " + value.title() for value in areas) + + "\n\nVALID CATEGORIES (" + str(len(categories)) + "):\n" + + "\n".join("- " + value.title() for value in categories) + ) + return JOURNAL_PROFILE_PROMPT + _JOURNAL_PROFILE_VOCAB_BLOCK + + +def _legacy_access_and_sharing(manuscript: ManuscriptInput) -> tuple[str, bool, str]: + access = " ".join((manuscript.access_preference or "").split()) + sharing = bool(manuscript.sharing_requirement) + repository = " ".join((manuscript.intended_repository or "").split()) + legacy = " ".join((manuscript.oa_preference or "").split()).lower() + if not access or access.lower() == "any journal": + if "diamond" in legacy or "no apc" in legacy: + access = "Diamond / no APC" + elif "gold" in legacy: + access = "Open-access journals" + else: + access = "Any journal" + if "green" in legacy: + sharing = True + if (not repository or repository == "Institutional repository") and manuscript.repository_target: + repository = manuscript.repository_target + return access, sharing, repository or "Institutional repository" + + +def _preferences_from_manuscript(manuscript: ManuscriptInput) -> DiscoveryPreferences: + access, sharing, repository = _legacy_access_and_sharing(manuscript) + return DiscoveryPreferences.from_values( + ranking_preference=manuscript.ranking_preference, + access_preference=access, + sharing_requirement=sharing, + intended_repository=repository, + apc_budget=manuscript.apc_budget, + funder=manuscript.funder, + preferred_journals=manuscript.preferred_journals, + avoid_journals=manuscript.avoid_journals, + ) + + +def _preferences_from_subject(req: SubjectRequest) -> DiscoveryPreferences: + return DiscoveryPreferences.from_values( + ranking_preference=req.ranking_preference, + access_preference=req.access_preference, + sharing_requirement=req.sharing_requirement, + intended_repository=req.intended_repository, + apc_budget=req.apc_budget, + funder=req.funder, + preferred_journals=req.preferred_journals, + avoid_journals=req.avoid_journals, + ) + + +def _fallback_manuscript_profile(manuscript: ManuscriptInput, note: str = "") -> dict: + keywords = [k.strip() for k in re.split(r"[,;]", manuscript.keywords or "") if k.strip()] + return { + "manuscript_understanding": { + "summary": manuscript.title, + "discipline": manuscript.discipline, + "article_type": manuscript.article_type, + "methods": manuscript.methods, + "audience": manuscript.audience, + "keywords": keywords[:10], + "inferred_criteria": [], + "assumptions": [ + "The manuscript profile could not be fully inferred; supplied fields were used directly." + ], + }, + "primary_category": manuscript.discipline if is_real_category(manuscript.discipline) else "", + "additional_categories": [], + "area": manuscript.discipline if is_real_area(manuscript.discipline) else "", + "confidence": "near" if manuscript.discipline else "not_found", + "classification_note": note or "Subject classification used the author-supplied fields only.", + } + + +async def _profile_manuscript(manuscript: ManuscriptInput, language: str) -> dict: + payload = manuscript.model_dump() + # Legacy form fields must not steer the classifier as if they were evidence + # about the manuscript's subject. + for key in ("oa_preference", "ranking_source", "speed_preference", "repository_target"): + payload.pop(key, None) + try: + parsed = await call_openai( + get_client(), + journal_profile_prompt_with_vocabulary(), + f"{lang_instruction(language)}Classify this manuscript.\n\n{json.dumps(payload, ensure_ascii=False, indent=2)}", + max_tokens=1400, + model=MODEL_LIGHT, + ) + if parsed.get("error"): + return _fallback_manuscript_profile(manuscript, parsed["error"]) + except Exception as exc: + logger.warning("[JournalProfile] using supplied fields (%s)", type(exc).__name__) + return _fallback_manuscript_profile(manuscript, "AI classification was unavailable; supplied fields were used.") + + understanding = parsed.get("manuscript_understanding") or {} + if not isinstance(understanding, dict): + return _fallback_manuscript_profile(manuscript, "AI profile format was invalid; supplied fields were used.") + understanding = { + "summary": " ".join(str(understanding.get("summary") or manuscript.title).split())[:1200], + "discipline": " ".join(str(understanding.get("discipline") or manuscript.discipline).split())[:300], + "article_type": " ".join(str(understanding.get("article_type") or manuscript.article_type).split())[:100], + "methods": " ".join(str(understanding.get("methods") or manuscript.methods).split())[:500], + "audience": " ".join(str(understanding.get("audience") or manuscript.audience).split())[:300], + "keywords": _text_list(understanding.get("keywords"), 10, 80), + "inferred_criteria": _text_list(understanding.get("inferred_criteria"), 8, 300), + "assumptions": _text_list(understanding.get("assumptions"), 8, 300), + } + primary = str(parsed.get("primary_category") or "").strip() + additional = _text_list(parsed.get("additional_categories"), 3, 200) + area = str(parsed.get("area") or "").strip() + # Closed-list validation. Invalid names are discarded rather than fuzzily + # corrected into a different field. + valid_categories = subject_vocabulary()["categories"] + valid_areas = subject_vocabulary()["areas"] + primary = primary if primary.lower() in valid_categories else "" + additional = [v for v in additional if v.lower() in valid_categories and v.lower() != primary.lower()][:2] + area = area if area.lower() in valid_areas else "" + return { + "manuscript_understanding": understanding, + "primary_category": primary, + "additional_categories": additional, + "area": area, + "confidence": parsed.get("confidence") if isinstance(parsed.get("confidence"), str) and parsed.get("confidence") in {"exact", "near", "not_found"} else "near", + "classification_note": " ".join(str(parsed.get("classification_note") or "").split())[:500], + } + + +def _candidate_query_text(manuscript: ManuscriptInput, profile: dict) -> str: + understanding = profile.get("manuscript_understanding") or {} + parts = [ + manuscript.title, + manuscript.abstract, + manuscript.keywords, + manuscript.discipline, + understanding.get("discipline", ""), + " ".join(understanding.get("keywords") or []), + understanding.get("methods", ""), + ] + return " ".join(str(part or "") for part in parts) + + +def _subject_query_text(subject: str, normalised: str, category: str, area: str) -> str: + return " ".join(value for value in (subject, normalised, category, area) if value) + + +def _candidate_base_dict(rec_obj, asjc_codes: Sequence[str], score_ratio: float) -> dict: + record = rec_obj.journal + matched_q, matched_subject_rows = quartile_for(record, asjc_codes) + matched_subjects = [ + { + "code": row.get("asjc", ""), + "field": row.get("name", ""), + "parent": row.get("parent", ""), + "quartile": row.get("quartile"), + } + for row in matched_subject_rows + ] + all_fields = [ + {"code": row.get("asjc", ""), "field": row.get("name", ""), "quartile": row.get("quartile")} + for row in record.subjects if row.get("quartile") + ] + issn = record.issn_e or record.issn_print or "" + publisher = record.publisher_group or record.publisher or "" + access = access_model(record) + agreement_publisher = is_ku_agreement_publisher(publisher) + ku_candidate = bool(agreement_publisher and matched_q == "Q1") + return { + "id": record.scopus_id, + "scopus_id": record.scopus_id, + "name": record.title, + "issn": issn, + "publisher": publisher, + "publisher_raw": record.publisher or "", + "identity_verification": { + "status": "Confirmed", + "method": "Canonical Scopus source record", + "scopus_id": record.scopus_id, + "issn_print": record.issn_print or "", + "issn_electronic": record.issn_e or "", + }, + "ranking": { + "source": "Scopus source list + SCImago 2025", + "year": "2025", + "quartile": matched_q, + "category": "; ".join(row["field"] for row in matched_subjects if row.get("field"))[:500], + "h_index": record.h_index, + "sjr": record.sjr, + "rank": record.sjr_rank, + "verification_status": "Confirmed", + "verification_method": "Canonical Scopus source ID and bundled SCImago match", + "interpretation": ( + f"{matched_q} in a matched subject field; compare within the relevant category." + if matched_q else "Currently indexed, but a matched-field quartile was not available." + ), + }, + # Top-level mirrors keep the shared LibGuides renderers and exports + # consistent across manuscript and subject discovery modes. + "quartile": matched_q, + "h_index": record.h_index, + "sjr": record.sjr, + "quartiles_by_field": all_fields, + "matched_subjects": matched_subjects, + "state": record.state, + "state_message": record.state_message, + "indexation_state": record.state, + "indexation_message": record.state_message, + "currently_indexed": record.currently_indexed, + "warn": record.warn, + "successor_title": record.successor_title, + "from_scopus": True, + "source_type": record.source_type, + "coverage_start": record.coverage_start, + "coverage_end": record.coverage_end, + "coverage_has_gap": record.coverage_has_gap, + "open_access": "Yes" if access["code"] in {"open", "diamond"} else "No/unknown", + "oa_diamond": record.oa_diamond, + "access_model": access, + "ku_agreement_publisher": agreement_publisher, + "ku_apc_candidate": ku_candidate, + "ku_apc_status": ( + "Potential KU publisher-agreement candidate; exact title, author, article type, quota and top-percentile eligibility are not confirmed." + if ku_candidate else + "Publisher-level KU agreement signal only; title-level APC eligibility is not confirmed." + if agreement_publisher else + "No KU publisher-agreement signal found in the configured publisher list." + ), + # Backward-compatible flag, now explicitly a candidate signal only. + "ku_apc_covered": ku_candidate, + "candidate_method": rec_obj.method, + "candidate_reasons": list(rec_obj.reasons), + "base_score": round(float(rec_obj.score), 4), + "base_score_ratio": round(float(score_ratio), 4), + "similar_articles": [], + "article_similarity_score": 0, + "article_evidence_status": "not_checked", + "sharing_policy": { + "status": "not_checked", + "requirement_status": "not_requested", + "message": "Journal-level sharing policy has not been checked." + }, + "verify_links": build_verify_links(record.title, issn, record.scimago_id or record.scopus_id), + } + + +async def _attach_article_evidence(candidates: list[dict], query: str, limit: int = 12) -> None: + if not candidates: + return + try: + evidence = await openalex.evidence_for_journals(candidates, query, limit=limit, per_page=3) + except Exception as exc: + logger.warning("[JournalEvidence] batch unavailable (%s)", type(exc).__name__) + return + for journal in candidates: + got = evidence.get(str(journal.get("id"))) + if not got: + continue + journal["similar_articles"] = got.get("works") or [] + journal["article_similarity_score"] = clamp_int(got.get("similarity_score"), 0, 100) + journal["article_evidence_status"] = got.get("status") or "unavailable" + journal["openalex"] = got.get("source") + if journal.get("openalex") and journal["openalex"].get("display_name"): + journal["identity_verification"]["openalex_title"] = journal["openalex"].get("display_name") + + +def _retain_preferred(journals: list[dict], preferred_titles: Sequence[str], limit: int = 15) -> list[dict]: + selected = list(journals[:limit]) + wanted = {normalise_title(title) for title in preferred_titles} + if not wanted: + return selected + selected_ids = {str(j.get("id")) for j in selected} + for journal in journals: + if normalise_title(journal.get("name", "")) not in wanted or str(journal.get("id")) in selected_ids: + continue + replace_at = next((i for i in range(len(selected)-1, -1, -1) if normalise_title(selected[i].get("name", "")) not in wanted), None) + if len(selected) < limit: + selected.append(journal) + elif replace_at is not None: + selected_ids.discard(str(selected[replace_at].get("id"))) + selected[replace_at] = journal + else: + break + selected_ids.add(str(journal.get("id"))) + order = {str(j.get("id")): i for i, j in enumerate(journals)} + return sorted(selected, key=lambda j: order.get(str(j.get("id")), len(journals))) + + +def _compact_candidate_for_rerank(journal: dict) -> dict: + return { + "id": journal.get("id"), + "title": journal.get("name"), + "publisher": journal.get("publisher"), + "matched_subjects": journal.get("matched_subjects", [])[:6], + "quartile": (journal.get("ranking") or {}).get("quartile"), + "h_index": (journal.get("ranking") or {}).get("h_index"), + "access_model": (journal.get("access_model") or {}).get("label"), + "ku_agreement_candidate": journal.get("ku_apc_candidate", False), + "base_score_ratio": journal.get("base_score_ratio", 0), + "article_similarity_score": journal.get("article_similarity_score", 0), + "recent_article_titles": [ + {"title": item.get("title"), "year": item.get("year")} + for item in (journal.get("similar_articles") or [])[:3] + ], + } + + +async def _rerank_manuscript_candidates( + manuscript: ManuscriptInput, + profile: dict, + candidates: list[dict], + record_map: dict[str, Any], + asjc_codes: Sequence[str], + preferences: DiscoveryPreferences, + language: str, +) -> tuple[list[dict], list[str], str]: + if not candidates: + return [], [], "No verified journal candidates met the applied filters." + candidate_payload = [_compact_candidate_for_rerank(j) for j in _retain_preferred(candidates, preferences.preferred_journals, 35)] + user_payload = { + "manuscript": { + "title": manuscript.title, + "abstract": manuscript.abstract, + "article_type": manuscript.article_type, + "methods": manuscript.methods, + "audience": manuscript.audience, + }, + "profile": profile, + "verified_candidates": candidate_payload, + } + parsed: dict = {} + try: + parsed = await call_openai( + get_client(), + JOURNAL_RERANK_PROMPT, + f"{lang_instruction(language)}Rerank these verified candidates.\n\n{json.dumps(user_payload, ensure_ascii=False)}", + max_tokens=7000, + model=MODEL, + ) + except Exception as exc: + logger.warning("[JournalRerank] subject-based fallback (%s)", type(exc).__name__) + parsed = {"selected": [], "next_actions": [], "global_notes": "AI reranking was unavailable. These are subject-based suggestions; methods and audience fit were not evaluated."} + + by_id = {str(j["id"]): j for j in candidates} + selected_rows = [] + seen: set[str] = set() + model_rows = parsed.get("selected") if isinstance(parsed.get("selected"), list) else [] + for item in model_rows: + if not isinstance(item, dict): + continue + sid = str(item.get("id") or "") + if sid in by_id and sid not in seen: + seen.add(sid) + selected_rows.append((by_id[sid], item)) + if len(selected_rows) >= 15: + break + # Fewer is better than padding, but always retain at least five verified + # candidates when the deterministic engine found them. + for journal in candidates: + sid = str(journal["id"]) + if len(selected_rows) >= min(15, len(candidates)): + break + if sid not in seen: + selected_rows.append((journal, {})); seen.add(sid) + + # Include eligible preferred records before scoring and final truncation. + selected_by_id = {str(j["id"]): item for j, item in selected_rows} + selected_order = [j for j, _ in selected_rows] + [j for j in candidates if str(j["id"]) not in selected_by_id] + selected_rows = [(j, selected_by_id.get(str(j["id"]), {})) for j in _retain_preferred(selected_order, preferences.preferred_journals, 15)] + output: list[dict] = [] + for journal, item in selected_rows: + record = record_map[str(journal["id"])].journal + topic = clamp_int(item.get("topic_scope_score"), 0, 40, + default=20 if journal.get("matched_subjects") else 0) + # Recent article-title overlap is a modest, transparent supporting + # signal. It can strengthen a scope score but never dominate it. + evidence_bonus = min(5, round(clamp_int(journal.get("article_similarity_score"), 0, 100) / 20)) + topic = min(40, topic + evidence_bonus) + article = clamp_int(item.get("article_type_methods_score"), 0, 20, + default=0) + audience = clamp_int(item.get("audience_score"), 0, 15, + default=0) + rank = ranking_component(record, asjc_codes, 15) + practical = practical_component( + record, preferences, 10, ku_agreement_candidate=journal.get("ku_apc_candidate", False) + ) + journal["fit_components"] = [ + {"key": "topic_scope", "label": "Topic and scope evidence", "score": topic, "max": 40}, + {"key": "article_methods", "label": "Article type and methods", "score": article, "max": 20}, + {"key": "audience", "label": "Audience alignment", "score": audience, "max": 15}, + {"key": "ranking", "label": "Indexing and subject ranking", "score": rank, "max": 15}, + {"key": "practical", "label": "Practical preferences", "score": practical, "max": 10}, + ] + journal["scoring_mode"] = "model_assisted" if item else "subject_fallback" + journal["scoring_note"] = ( + "Heuristic comparison score; not an acceptance probability." + if item else + "Subject-based fallback: methods and audience fit were not evaluated. Zero for these components means not evaluated." + ) + journal["fit_score"] = topic + article + audience + rank + practical + journal["fit_reason"] = " ".join(str(item.get("fit_reason") or "").split()) or ( + "Selected from the verified subject-filtered journal index; review the matched fields and recent articles before submission." + ) + journal["audience_match"] = " ".join(str(item.get("audience_match") or "").split()) or ( + "The journal's indexed subject fields overlap the inferred manuscript discipline." + ) + journal["submission_strategy"] = " ".join(str(item.get("submission_strategy") or "").split()) or ( + "Review the current aims and scope, recent articles, author guidelines, fees, and article-type requirements." + ) + journal["open_access_planning"] = " ".join(str(item.get("open_access_planning") or "").split()) or ( + "Use the journal access flag for initial planning, then check current fees and the Open Policy Finder sharing options." + ) + journal["article_evidence_summary"] = " ".join(str(item.get("article_evidence_summary") or "").split()) or ( + "Recent article-title evidence was unavailable." + if not journal.get("similar_articles") else + "Recent OpenAlex article titles provide supporting, not conclusive, scope evidence." + ) + output.append(journal) + + output.sort(key=lambda j: (-j.get("fit_score", 0), j.get("name", ""))) + next_actions = _text_list(parsed.get("next_actions"), 4, 500) + global_notes = " ".join(str(parsed.get("global_notes") or "").split())[:800] + return output, next_actions, global_notes + + +def _score_subject_candidates( + candidates: list[dict], + record_map: dict[str, Any], + asjc_codes: Sequence[str], + preferences: DiscoveryPreferences, + *, + exact_category: bool, +) -> list[dict]: + output = [] + for journal in candidates: + record = record_map[str(journal["id"])].journal + ratio = float(journal.get("base_score_ratio") or 0) + subject_score = round((42 if exact_category else 32) + (18 if exact_category else 20) * ratio) + evidence_bonus = min(5, round(clamp_int(journal.get("article_similarity_score"), 0, 100) / 20)) + subject_score = min(60, subject_score + evidence_bonus) + rank = ranking_component(record, asjc_codes, 25) + practical = practical_component( + record, preferences, 15, ku_agreement_candidate=journal.get("ku_apc_candidate", False) + ) + journal["fit_components"] = [ + {"key": "subject", "label": "Subject relevance", "score": subject_score, "max": 60}, + {"key": "ranking", "label": "Indexing and subject ranking", "score": rank, "max": 25}, + {"key": "practical", "label": "Practical preferences", "score": practical, "max": 15}, + ] + journal["fit_score"] = subject_score + rank + practical + fields = [row.get("field") for row in journal.get("matched_subjects") or [] if row.get("field")] + field_text = ", ".join(fields[:3]) or "the selected subject area" + journal["fit_reason"] = f"Indexed in {field_text}; ranked using verified subject and journal metadata." + journal["audience_match"] = f"Relevant to researchers and readers working in {field_text}." + journal["submission_strategy"] = "Use this as a discovery lead; review current aims and scope, recent articles, article types, and author instructions." + journal["open_access_planning"] = "The access-model flag is journal-level. Check current fees and run the Open Policy Finder policy check before planning manuscript sharing." + journal["article_evidence_summary"] = ( + "Recent OpenAlex article titles support the subject match." + if journal.get("similar_articles") else + "Recent article-title evidence was unavailable; rely on the verified subject classification and inspect the journal site." + ) + output.append(journal) + output.sort(key=lambda j: (-j.get("fit_score", 0), j.get("name", ""))) + return output + + +async def _lookup_policy_for_journal(journal: dict, *, + manuscript_version: str = "Accepted manuscript (AAM / postprint)", + intended_repository: str = "Institutional repository", intended_licence: str = "", funder: str = "", +) -> dict: + try: + record = await openpolicyfinder.lookup_policy( + journal.get("name", ""), journal.get("issn", ""), journal.get("publisher", "") + ) + if record.get("lookup_status") == "ok": + green = openpolicyfinder.to_green_oa(record, manuscript_version=manuscript_version, + intended_repository=intended_repository, intended_licence=intended_licence, funder=funder) + return { + "status": "confirmed", + "requirement_status": policy_requirement_state(green, manuscript_version=manuscript_version, + intended_repository=intended_repository, intended_licence=intended_licence, funder=funder), + "policy_evaluation": evaluate_policy(green, manuscript_version=manuscript_version, + intended_repository=intended_repository, intended_licence=intended_licence, funder=funder), + "green_oa": green, + "record": { + "title": record.get("journal_title", ""), + "publisher": record.get("publisher", ""), + "record_url": record.get("record_url", ""), + "match_quality": record.get("match_quality", ""), + "retrieved_at": record.get("retrieved_at", ""), + }, + "message": "Journal-level options retrieved from Jisc Open Policy Finder.", + } + return { + "status": "unavailable", + "requirement_status": "unknown", + "lookup_status": record.get("lookup_status", "upstream_error"), + "message": record.get("user_message") or "No matching Open Policy Finder record was retrieved.", + "verify_url": openpolicyfinder.public_search_url(journal.get("name", ""), journal.get("issn", "")), + } + except Exception as exc: + logger.warning("[JournalPolicy] lookup unavailable (%s)", type(exc).__name__) + return { + "status": "unavailable", "requirement_status": "unknown", + "message": "The journal-level policy could not be retrieved.", + "verify_url": openpolicyfinder.public_search_url(journal.get("name", ""), journal.get("issn", "")), + } + + +async def _lookup_policies(journals: Sequence[dict], limit: int = 5, *, + manuscript_version: str = "Accepted manuscript (AAM / postprint)", + intended_repository: str = "Institutional repository", intended_licence: str = "", funder: str = "", +) -> dict[str, dict]: + semaphore = asyncio.Semaphore(4) + + async def one(journal: dict) -> tuple[str, dict]: + async with semaphore: + return str(journal.get("id") or journal.get("issn") or journal.get("name")), await _lookup_policy_for_journal(journal, manuscript_version=manuscript_version, + intended_repository=intended_repository, intended_licence=intended_licence, funder=funder) + + selected = list(journals)[: max(0, limit)] + pairs = await asyncio.gather(*(one(j) for j in selected)) if selected else [] + return dict(pairs) + + +async def _apply_sharing_requirement( + journals: list[dict], + preferences: DiscoveryPreferences, +) -> tuple[list[dict], list[str]]: + if not preferences.sharing_requirement or not journals: + for journal in journals: + journal["sharing_policy"]["requirement_status"] = "not_requested" + return journals, [] + + policies = await _lookup_policies(journals, limit=min(15, len(journals)), + intended_repository=preferences.intended_repository, funder=preferences.funder) + for journal in journals: + key = str(journal.get("id")) + if key in policies: + journal["sharing_policy"] = policies[key] + else: + journal["sharing_policy"]["requirement_status"] = "unknown" + + acceptable = [j for j in journals if (j.get("sharing_policy") or {}).get("requirement_status") in {"meets", "conditional"}] + unknown = [j for j in journals if (j.get("sharing_policy") or {}).get("requirement_status") == "unknown"] + denied = [j for j in journals if (j.get("sharing_policy") or {}).get("requirement_status") == "does_not_meet"] + notes = [] + if denied: + notes.append(f"Removed {len(denied)} candidate(s) whose retrieved accepted-manuscript option did not meet the sharing requirement.") + if unknown: + notes.append(f"{len(unknown)} candidate(s) could not be confirmed and are retained after confirmed/conditional matches; missing data is not treated as permission or prohibition.") + if not acceptable: + notes.append("No candidate had a clearly applicable accepted-manuscript option; results are shown for manual review rather than claiming the requirement was satisfied.") + ordered = acceptable + unknown + return ordered, notes + + +def _finalise_discovery_result( + *, + mode: str, + ranked: list[dict], + applied_criteria: list[str], + unverified_criteria: list[str], + legacy_notes: list[str], + next_actions: list[str], + global_notes: str, + understanding: Optional[dict] = None, + subject_understanding: Optional[dict] = None, + classification_note: str = "", + preferred_titles: Sequence[str] = (), +) -> dict: + # Five evidence-rich cards plus up to ten additional verified leads. + shortlist = _retain_preferred(ranked, preferred_titles, 15) + detailed = shortlist[:5] + extended = shortlist[5:15] + actions = [a for a in next_actions if a] + actions.extend([ + "Compare two to five shortlisted journals using the decision table.", + "Check the current journal website for aims and scope, article types, fees and author instructions.", + "Run the Open Policy Finder check for shortlisted journals; use a DOI-based article check after publication.", + ]) + actions = list(dict.fromkeys(actions))[:6] + note_parts = [value for value in (global_notes, classification_note) if value] + return { + "schema_version": "journal_discovery_v2", + "mode": mode, + "manuscript_understanding": understanding, + "subject_understanding": subject_understanding, + "applied_criteria": applied_criteria, + "unverified_criteria": unverified_criteria + legacy_notes, + "journals": detailed, + "extended_list": extended, + "total": len(detailed) + len(extended), + "candidate_count_before_shortlist": len(ranked), + "next_actions": actions, + "global_notes": " ".join(note_parts) or ( + "Recommendations are decision support, not guarantees of acceptance or policy eligibility." + ), + "policy_source_mode": "on_demand_open_policy_finder", + "comparison_enabled": True, + } + + +@app.post("/api/analyze-journal") +@with_links +async def analyze_journal(req: JournalRequest, request: Request): + ip = get_client_ip(request) + if not check_rate_limit(ip, "global"): + raise HTTPException(429, "Too many requests. Please wait a moment and try again.") + + index = get_index() + if index is None: + raise HTTPException( + 503, + "The verified journal index is unavailable. ResearchBee will not generate unverified journal names as a fallback.", + ) + + manuscript = req.manuscript + preferences = _preferences_from_manuscript(manuscript) + profile = await _profile_manuscript(manuscript, req.language) + asjc_rows = resolve_asjc_rows( + index, + category=profile.get("primary_category", ""), + area=profile.get("area", ""), + additional_categories=profile.get("additional_categories") or [], + ) + query_text = _candidate_query_text(manuscript, profile) + pool = build_candidate_pool( + index, + title=manuscript.title, + abstract=query_text, + asjc_rows=asjc_rows, + preferences=preferences, + limit=50, + candidate_pool=600, + ) + + record_map = {rec.journal.scopus_id: rec for rec in pool.recommendations} + asjc_codes = [str(row.get("code")) for row in asjc_rows if row.get("code")] + candidates = [ + _candidate_base_dict(rec, asjc_codes, pool.score_ratios.get(rec.journal.scopus_id, 0.0)) + for rec in pool.recommendations + ] + evidence_query = " ".join(str(row.get("label") or row.get("name") or "") for row in asjc_rows) + if not evidence_query: + evidence_query = " ".join(v for v in [profile.get("primary_category", ""), profile.get("area", "")] if v) + await _attach_article_evidence(candidates, evidence_query, limit=12) + ranked, next_actions, global_notes = await _rerank_manuscript_candidates( + manuscript, profile, candidates, record_map, asjc_codes, preferences, req.language + ) + ranked, policy_notes = await _apply_sharing_requirement(ranked, preferences) + + unverified = list(pool.unverified_criteria) + if policy_notes: + # Replace the provisional note added by build_candidate_pool with the + # actual outcome of the Open Policy Finder pass. + unverified = [ + note for note in unverified + if not note.startswith("Accepted-manuscript sharing will be checked") + ] + unverified.extend(policy_notes) + if not asjc_rows: + unverified.append( + "No exact closed-vocabulary subject category was resolved, so candidate generation used verified full-index text matching and should be reviewed carefully." + ) + + result = _finalise_discovery_result( + mode="manuscript", + preferred_titles=preferences.preferred_journals, + ranked=ranked, + applied_criteria=pool.applied_criteria, + unverified_criteria=unverified, + legacy_notes=pool.legacy_notes, + next_actions=next_actions, + global_notes=global_notes, + understanding=profile.get("manuscript_understanding"), + classification_note=profile.get("classification_note", ""), + ) + result["classification"] = { + "confidence": profile.get("confidence"), + "primary_category": profile.get("primary_category"), + "additional_categories": profile.get("additional_categories") or [], + "area": profile.get("area"), + "asjc_codes": asjc_codes, + } + return {"result": result} + + +# ── Khazna deposit status ───────────────────────────────────────────────── +def khazna_status_for(doi: str) -> dict: + """ + THREE-VALUED on purpose. + + "could not check" and "not in Khazna" are different answers. Conflating them + would tell a researcher to deposit something already deposited, or imply an + absence we never actually verified. + """ + if not doi: + return {"checked": False, "reason": "no DOI supplied"} + source = khazna_index.status + snapshot = { + "checked_scope": "loaded_snapshot", + "snapshot_loaded_at": source.get("loaded_at"), + "snapshot_freshness": source.get("snapshot_freshness", "unknown"), + "freshness_note": source.get("freshness_note", "Source harvest date is unknown; confirm current status in Khazna."), + "retaining_previous_snapshot": source.get("retaining_previous_snapshot", False), + } + if not khazna_index.available: + return {**snapshot, "checked": False, "reason": "Khazna index unavailable"} + rec = khazna_index.get(doi) + if rec is None: + return {**snapshot, "checked": True, "in_khazna": False, + "message": "This DOI was not found in the loaded Khazna snapshot. Confirm in the live portal before creating another record."} + state = rec.get("deposit_state", "undetermined") + return { + **snapshot, + "checked": True, + "in_khazna": True, + "deposit_state": state, + "embargo_end": rec.get("embargo_end"), + "portal_url": rec.get("portal_url", ""), + "needs_deposit": state in ("metadata_only", "closed"), + "message": { + "open": "The loaded Khazna record reports an open full text. Check the portal for its current status.", + "embargoed": "In the KU repository, Khazna, with an embargoed file.", + "restricted": "In the KU repository, Khazna, but the attached file is restricted.", + "closed": "In the KU repository, Khazna, as a closed record — an open version may be permitted.", + # Khazna is currently a metadata-only CRIS: records exist for KU + # output but no files are attached. This is therefore the normal + # case, and the message is framed as an opportunity rather than a + # problem with the author's record. + "metadata_only": ("Your publication is already recorded in the KU repository, " + "Khazna, but no full text is attached. Adding the permitted " + "version makes it openly readable and discoverable."), + "undetermined": "In the KU repository, Khazna; access status could not be determined.", + }.get(state, "In the KU repository, Khazna."), + } + + +async def _advisory_note(green: dict, li: "LicenseInput", lang: str) -> str: + """The LLM writes prose from verified facts only. It never supplies a value.""" + facts = { + "permission_status": green.get("permission_status", "unknown"), + "embargo_status": green.get("embargo_status", "unknown"), + "can_archive": green.get("_can_archive"), + "versions_permitted": green.get("_versions"), + "best_version": green.get("_best_version"), + "locations": green.get("_locations"), + "embargo_months": green.get("_embargo_months"), + "embargo_stated": green.get("embargo_status") != "unknown", + "embargo_end": green.get("embargo_end"), + "licence": green.get("_licence"), + "journal_oa_type": green.get("journal_oa_type"), + "institutional_repository_permitted": green.get("_institutional_ok"), + "author_intent": { + "repository": li.intended_repository, + "licence": li.intended_licence, + "version": li.manuscript_version, + "funder": li.funder, + }, + } + try: + out = await call_openai( + get_client(), ADVISORY_NOTE_PROMPT, + lang + json.dumps(facts, ensure_ascii=False), + max_tokens=280, model=MODEL_LIGHT, + ) + return out.get("note", "") if isinstance(out, dict) else "" + except Exception as e: + logger.warning("Advisory note unavailable (%s)", type(e).__name__) + return "" + + +@app.post("/api/check-journal-policies") +async def check_journal_policies(req: JournalPolicyBatchRequest, request: Request): + """On-demand journal-level policy checks for either discovery mode. + + Up to five shortlist items are accepted per call. Every item is resolved + again against ``journals.db`` before Jisc is queried, so browser-supplied + titles, ISSNs, and publishers cannot replace the canonical journal identity. + Facts come only from the authenticated Jisc Open Policy Finder API; a miss + remains unknown. + """ + ip = get_client_ip(request) + if not check_rate_limit(ip, "global"): + raise HTTPException(429, "Too many requests. Please wait a moment and try again.") + + index = get_index() + if index is None: + raise HTTPException( + 503, + "The verified journal index is unavailable, so journal identity cannot be checked safely.", + ) + + prepared: list[dict] = [] + verified_journals: list[dict] = [] + for item in req.journals: + response_key = item.key or item.issn or item.name + record = None + try: + if item.key: + record = index.lookup_exact(scopus_id=item.key) + if record is None and item.issn: + record = index.lookup_exact(issn=item.issn) + if record is None and item.name: + record = index.lookup_exact(title=item.name) + except Exception as exc: + logger.warning("[JournalPolicy] identity lookup unavailable (%s)", type(exc).__name__) + + if ( + record is None + or not record.currently_indexed + or record.source_type != "Journal" + ): + prepared.append({ + "key": response_key, + "journal": None, + "result": { + "key": response_key, + "name": item.name, + "issn": item.issn, + "publisher": item.publisher, + "identity_verification": { + "status": "Not confirmed", + "method": "No active canonical journal record matched the submitted selection", + }, + "policy": { + "status": "unavailable", + "requirement_status": "unknown", + "message": "Policy was not queried because the journal identity could not be re-verified.", + }, + "context": { + "manuscript_version": req.manuscript_version, + "intended_repository": req.intended_repository, + "funder": req.funder, + }, + }, + }) + continue + + canonical = { + "id": response_key, + "name": record.title, + "issn": record.issn_e or record.issn_print or "", + "publisher": record.publisher_group or record.publisher or "", + "scopus_id": record.scopus_id, + } + prepared.append({"key": response_key, "journal": canonical, "result": None}) + verified_journals.append(canonical) + + policies = await _lookup_policies(verified_journals, limit=5, manuscript_version=req.manuscript_version, + intended_repository=req.intended_repository, funder=req.funder) + results = [] + for entry in prepared: + if entry["result"] is not None: + results.append(entry["result"]) + continue + journal = entry["journal"] + key = str(entry["key"]) + policy = policies.get(key) or { + "status": "unavailable", + "requirement_status": "unknown", + "message": "The journal-level policy could not be retrieved.", + } + results.append({ + "key": key, + "name": journal["name"], + "issn": journal["issn"], + "publisher": journal["publisher"], + "identity_verification": { + "status": "Confirmed", + "method": "Canonical journals.db record re-resolved before policy lookup", + "scopus_id": journal["scopus_id"], + }, + "policy": policy, + "context": { + "manuscript_version": req.manuscript_version, + "intended_repository": req.intended_repository, + "funder": req.funder, + }, + }) + return {"results": results, "source_mode": "journal_api"} + + +@app.post("/api/check-license") +@with_links +async def check_license(req: LicenseRequest, request: Request): + ip = get_client_ip(request) + if not check_rate_limit(ip, "global"): + raise HTTPException(429, "Too many requests. Please wait a moment and try again.") + + li = req.license_input + policy_context = {"manuscript_version": li.manuscript_version, + "intended_repository": li.intended_repository, + "intended_licence": li.intended_licence, "funder": li.funder} + lang = lang_instruction(req.language) + doi = doi_utils.extract_doi(li.doi) or doi_utils.extract_doi(li.notes) + if li.doi.strip() and not doi: + raise HTTPException(422, "Enter a valid DOI or a URL containing a DOI.") + + # ══ PATH A — DOI supplied: verified, article-level, no LLM policy facts ══ + if doi: + permissions, khazna = await asyncio.gather( + oaworks.fetch_permissions(doi, ror=oaworks.KU_ROR), + asyncio.to_thread(khazna_status_for, doi), + ) + green = oaworks.parse_permissions(permissions, **policy_context) + + if green is None: + # The DOI resolved but OA.Works holds no policy for it, so the + # researcher leaves without the verified answer they came for. + usage.usage.record_gap("licence_deposit", "oaworks_404") + request.state.usage_recorded = True + green = oaworks.not_confirmed( + li.journal_name, li.issn, + reason="no permission record for this DOI in OA.Works") + else: + green["repository_action_note"] = await _advisory_note(green, li, lang) + + journal = { + "name": li.journal_name or green.get("copyright_name", ""), + "issn": li.issn or ((green.get("issuer_issns") or [""])[0]), + "publisher": li.publisher or green.get("policy_issuer", ""), + "green_oa": green, + } + enriched = await enrich_journals([journal]) + + # Subject repositories, but ONLY when OA.Works says one is permitted. + # Suggesting arXiv for a journal that forbids subject-repository + # deposit would be exactly the kind of confident-but-wrong advice this + # build exists to prevent. + subject_repos = [] + if any("subject repository" in loc.lower() + for loc in green.get("_locations", [])): + sci = lookup_any([enriched[0].get("issn"), li.issn]) or {} + subject_repos = repositories.suggest_subject_repositories( + sci.get("areas", ""), sci.get("categories", "")) + + result = { + "journals": enriched, + "subject_repositories": subject_repos, + "repository_recommendation": oaworks.recommendation_from(green, **policy_context), + "next_actions": oaworks.next_actions_from(green, doi, khazna, **policy_context), + "global_notes": "", + "doi": doi, + "doi_url": doi_utils.doi_url(doi), + "shareyourpaper_url": oaworks.share_link(doi), + "khazna_status": khazna, + "source_mode": "doi", + "khazna": KHAZNA_DEPOSIT_CARD, + } + return {"result": result} + + # ══ PATH B — no DOI: authenticated Open Policy Finder journal record ══ + if not li.journal_name and not li.issn: + raise HTTPException(422, "Provide an article DOI/URL, or a journal name or ISSN.") + + issn = doi_utils.normalise_issn(li.issn) or li.issn + opf_record = await openpolicyfinder.lookup_policy( + li.journal_name, + issn, + li.publisher, + ) + lookup_status = opf_record.get("lookup_status", "upstream_error") + + if lookup_status == "ok": + green = openpolicyfinder.to_green_oa(opf_record, **policy_context) + source_mode = "journal_api" + display_name = opf_record.get("journal_title") or li.journal_name + display_issn = issn or ((opf_record.get("issns") or [""])[0]) + display_publisher = opf_record.get("publisher") or li.publisher + next_actions = oaworks.next_actions_from(green, "", **policy_context) + global_notes = ( + "This is a journal-level policy record from the authenticated Jisc " + "Open Policy Finder API. It may contain several routes with different " + "conditions. Use the DOI search above for an article-level permission " + "check and a Khazna status check." + ) + else: + reason_by_status = { + "unconfigured": "Open Policy Finder API access is not configured", + "authentication_error": "Open Policy Finder rejected the configured API key", + "rate_limited": "Open Policy Finder is temporarily rate limiting requests", + "not_found": "no matching journal record was returned by Open Policy Finder", + "identity_mismatch": "Open Policy Finder returned records that did not match the supplied journal identity", + "invalid_response": "Open Policy Finder returned an unexpected response", + "upstream_error": "Open Policy Finder is temporarily unavailable", + } + reason = reason_by_status.get(lookup_status, reason_by_status["upstream_error"]) + green = oaworks.not_confirmed(li.journal_name, issn, reason=reason) + green["source_type"] = "open_policy_finder_api" + green["policy_scope"] = "journal" + green["repository_action_note"] = ( + "Open the linked Open Policy Finder search result and check the current " + "journal record. No policy determination was made by ResearchBee." + ) + green["_verify_url"] = opf_record.get("record_url") or openpolicyfinder.public_search_url( + li.journal_name, issn + ) + green["evidence_note"] = opf_record.get("message") or ( + "No authenticated journal-level policy record was available for this request." + ) + source_mode = "journal" + display_name = li.journal_name + display_issn = issn + display_publisher = li.publisher + next_actions = oaworks.next_actions_from(green, "", **policy_context) + global_notes = green["evidence_note"] + + gap_code = { + "unconfigured": "opf_unconfigured", + "authentication_error": "opf_auth_error", + "rate_limited": "opf_rate_limited", + "not_found": "opf_not_found", + "identity_mismatch": "opf_identity_mismatch", + "invalid_response": "opf_invalid_response", + "upstream_error": "opf_upstream_error", + }.get(lookup_status, "opf_upstream_error") + usage.usage.record_gap("licence_deposit", gap_code) + request.state.usage_recorded = True + + journal = { + "name": display_name, + "issn": display_issn, + "publisher": display_publisher, + "green_oa": green, + } + enriched = await enrich_journals([journal]) + if source_mode == "journal_api" and enriched: + enriched[0].setdefault("verify_links", {})["sherpa_romeo"] = green.get( + "_verify_url", + openpolicyfinder.public_search_url(display_name, display_issn), + ) + + parsed = { + "journals": enriched, + "repository_recommendation": oaworks.recommendation_from(green, **policy_context), + "next_actions": next_actions, + "global_notes": global_notes, + "khazna_status": {"checked": False, "reason": "no DOI supplied"}, + "source_mode": source_mode, + "opf_lookup": { + "status": lookup_status, + "source": "Jisc Open Policy Finder API", + "scope": "journal", + "record_url": opf_record.get("record_url", ""), + "record_id": opf_record.get("record_id", ""), + "retrieved_at": opf_record.get("retrieved_at", ""), + }, + "khazna": KHAZNA_DEPOSIT_CARD, + } + return {"result": parsed} + + +async def _re3data_supplement(dataset: "DatasetInput") -> dict: + """Bounded re3data lookup that never raises into the route.""" + fallback = { + "status": "unavailable", "query": re3data.build_query(dataset.discipline), + "repositories": [], "source": "re3data.org", "source_url": "https://www.re3data.org", + "message": "re3data.org did not respond in time. The curated KU Library recommendations are unaffected.", + } + try: + return await asyncio.wait_for( + re3data.find_supplementary( + dataset.discipline, + constraints=dataset.model_dump(), + curated=REGISTRY.values(), + ), + timeout=float(os.getenv("RE3DATA_TIMEOUT_SECONDS", "12")), + ) + except asyncio.TimeoutError: + return fallback + except Exception as exc: + logger.warning("[re3data] supplement failed (%s)", type(exc).__name__) + return fallback + + +@app.post("/api/find-repository") +async def find_repository(req: RepoRequest, request: Request): + ip = get_client_ip(request) + if not check_rate_limit(ip, "global"): + raise HTTPException(429, "Too many requests. Please wait a moment and try again.") + client = get_client() + lang = lang_instruction(req.language) + user_msg = ( + f"{lang}Select suitable repositories for this dataset FROM THE REGISTRY BELOW. " + f"Return JSON only. Use the exact \"id\" values. Do not invent repositories " + f"and do not state factual repository attributes.\n\n" + f"AVAILABLE REGISTRY:\n{registry_for_prompt()}\n\n" + f"DATASET INPUT:\n{json.dumps(req.dataset.model_dump(), indent=2)}" + ) + # re3data runs alongside the model call, so it adds no latency in the + # normal case. It only receives the discipline field and can never fail + # the request; the curated registry flow below is unchanged. + re3data_task = asyncio.create_task(_re3data_supplement(req.dataset)) + try: + return await _curated_repository_result(req, request, client, user_msg, re3data_task) + finally: + # Every error path (model failure, invalid selection, cancellation) + # must not leave the background lookup running. + if not re3data_task.done(): + re3data_task.cancel() + await asyncio.gather(re3data_task, return_exceptions=True) + + +async def _curated_repository_result(req, request, client, user_msg, re3data_task) -> dict: + """The existing curated-registry flow, plus the awaited re3data list.""" + parsed = await call_openai(client, REPO_SYSTEM_PROMPT, user_msg, max_tokens=2000) + if "error" in parsed: + raise HTTPException(500, parsed["error"]) + + # The model selected ids and wrote fit_reasons; the SERVER supplies every + # factual field. Unknown ids are silently dropped. + constraints = req.dataset.model_dump() + if not isinstance(parsed.get("repositories", []), list): + raise HTTPException(503, "The AI service returned an invalid repository selection. Please try again.") + parsed["selection_exclusions"] = repositories.selection_exclusions( + parsed.get("repositories", []), constraints=constraints + ) + repos = hydrate_selection(parsed.get("repositories", []), constraints=constraints) + + # Khazna always leads, and is never a model choice. + repos = [r for r in repos if not r.get("is_khazna")] + if not repos: + # Only Khazna will be offered — the registry had nothing for this + # dataset. A recurring code here is a prompt to widen the registry. + usage.usage.record_gap("data_repository", "no_repository_match") + request.state.usage_recorded = True + repos.insert(0, repositories.khazna_entry(constraints=constraints)) + parsed["deposit_strategy"] = repositories.hydrate_deposit_strategy( + parsed.get("deposit_strategy"), repos, constraints=constraints + ) + for key in ("dataset_understanding", "fair_assessment"): + if key in parsed and not isinstance(parsed[key], dict): + parsed[key] = {} + for key in ("next_actions", "warnings"): + if key in parsed: + parsed[key] = _text_list(parsed[key], 8, 500) + + for r in repos: + if not r.get("is_khazna"): + clean_name = quote_plus(re.sub(r"[^\w\s]", "", r.get("name", "")).strip()) + r["verify_links"] = { + "re3data": f"https://www.re3data.org/search?query={clean_name}", + "fairsharing": f"https://fairsharing.org/search?q={clean_name}", + } + + parsed["repositories"] = repos + supplement = await re3data_task + curated_matches = sum(1 for r in repos if not r.get("is_khazna")) + # "Necessary" = the curated registry produced fewer than two file-deposit + # candidates. The frontend then shows the re3data list expanded. + supplement["recommended_view"] = "expanded" if curated_matches < 2 else "collapsed" + supplement["curated_candidate_count"] = curated_matches + parsed["re3data_supplement"] = supplement + parsed["selection_schema_version"] = "release-a-id-only-v1" + parsed["registry_note"] = ( + "Repositories are selected from a registry curated by KU Library. " + "Factual details are served from verified records, not generated." + ) + return {"result": parsed} + + +# ── Subject vocabulary injection ─────────────────────────────────────────── +# +# The LLM used to be asked for an "Exact SCImago category name" without being +# shown what the exact names are, so it generated plausible ones. "Crime and +# Society" reads like a SCImago category; SCImago has no category containing +# the word "crime". The invented name matched nothing, the search fell through +# to a substring test on the area, and a criminology query returned the +# Astrophysical Journal. +# +# Same pattern as the repository finder: the model chooses from a closed list +# and the server owns the facts. The validated vocabulary is cached locally. + +_SUBJECT_VOCAB_BLOCK: Optional[str] = None + + +def subject_prompt_with_vocabulary() -> str: + """SUBJECT_NORMALISE_PROMPT plus the real category and area names.""" + global _SUBJECT_VOCAB_BLOCK + if _SUBJECT_VOCAB_BLOCK is None: + v = subject_vocabulary() + cats = sorted(v["categories"]) + areas = sorted(v["areas"]) + _SUBJECT_VOCAB_BLOCK = ( + "\n\nVALID AREAS (" + str(len(areas)) + "):\n" + + "\n".join("- " + a.title() for a in areas) + + "\n\nVALID CATEGORIES (" + str(len(cats)) + "):\n" + + "\n".join("- " + c.title() for c in cats) + ) + return SUBJECT_NORMALISE_PROMPT + _SUBJECT_VOCAB_BLOCK + + +@app.post("/api/browse-subject") +@with_links +async def browse_subject(req: SubjectRequest, request: Request): + ip = get_client_ip(request) + if not check_rate_limit(ip, "global"): + raise HTTPException(429, "Too many requests. Please wait a moment and try again.") + + from scimago_index import _subject_cache, _scimago_cache + if not _subject_cache or not _scimago_cache: + load_scimago() + load_subject_index() + + subject_text = req.subject.strip() + typed_category = subject_text if is_real_category(subject_text) else "" + typed_area = subject_text if is_real_area(subject_text) else "" + spelling_match = None if (typed_category or typed_area) else closest_subject_term(subject_text) + # Reviewed aliases and whole-word containment ("physics" -> the area + # "Physics and Astronomy"). Deterministic, so it also works when the + # LLM classifier is unavailable, and it never returns an unlisted term. + nearest_match = ( + None if (typed_category or typed_area or spelling_match) + else nearest_subject_term(subject_text) + ) + llm_attempted = False + + # Exact vocabulary terms and conservative spelling repairs are resolved + # locally. The LLM is reserved for synonym and semantic nearest-category + # mapping, and may only return names from the closed vocabulary. + if typed_category or typed_area: + parsed = { + "confidence": "exact", + "normalised": subject_text, + "scimago_category": typed_category, + "scimago_area": typed_area, + "nearest_categories": [], + "mapping_type": "exact", + "use_nearest_category": False, + "user_message": "Matched directly to the verified ASJC/SCImago vocabulary.", + } + elif spelling_match: + corrected = str(spelling_match["name"]) + is_category = spelling_match["kind"] == "category" + parsed = { + "confidence": "near", + "normalised": corrected, + "scimago_category": corrected if is_category else "", + "scimago_area": corrected if not is_category else "", + "nearest_categories": suggest_categories(corrected, limit=4), + "mapping_type": "spelling_correction", + "use_nearest_category": True, + # The disclosure is built once below from the final selection; + # pre-filling it here produced the correction notice twice. + "user_message": "", + } + elif nearest_match: + nearest_name = str(nearest_match["name"]) + nearest_is_category = nearest_match["kind"] == "category" + parsed = { + "confidence": "near", + "normalised": nearest_name, + "scimago_category": nearest_name if nearest_is_category else "", + "scimago_area": nearest_name if not nearest_is_category else "", + "nearest_categories": list(nearest_match.get("alternatives") or []), + "mapping_type": "synonym" if nearest_match.get("match") == "synonym" else "nearest_category", + "use_nearest_category": True, + "user_message": "", + } + else: + llm_attempted = True + try: + parsed = await call_openai( + get_client(), + subject_prompt_with_vocabulary(), + f"{lang_instruction(req.language)}Subject input: {subject_text}", + max_tokens=520, + model=MODEL_LIGHT, + ) + except (RateLimitError, APIStatusError, HTTPException) as exc: + logger.warning("[BrowseSubject] classifier unavailable (%s)", type(exc).__name__) + parsed = { + "confidence": "not_found", + "normalised": subject_text, + "scimago_category": "", + "scimago_area": "", + "nearest_categories": [], + "mapping_type": "none", + "use_nearest_category": False, + "user_message": "The subject classifier was unavailable. Choose a verified category suggestion or try again.", + } + except Exception as exc: + logger.warning("[BrowseSubject] classifier failed (%s)", type(exc).__name__) + parsed = { + "confidence": "not_found", + "normalised": subject_text, + "scimago_category": "", + "scimago_area": "", + "nearest_categories": [], + "mapping_type": "none", + "use_nearest_category": False, + "user_message": "The subject could not be classified safely. Choose a verified category suggestion or try again.", + } + + confidence = parsed.get("confidence", "not_found") + if not isinstance(confidence, str) or confidence not in {"exact", "near", "not_found"}: + confidence = "not_found" + mapping_type = str(parsed.get("mapping_type") or "").strip().lower() + if mapping_type not in {"exact", "synonym", "spelling_correction", "nearest_category", "none"}: + mapping_type = "nearest_category" if confidence == "near" else "none" + use_nearest_category = parsed.get("use_nearest_category") is True + normalised = str(parsed.get("normalised") or subject_text).strip() + category = str(parsed.get("scimago_category") or "").strip() + area = str(parsed.get("scimago_area") or "").strip() + user_message = " ".join(str(parsed.get("user_message") or "").split()) + + nearest_categories: list[str] = [] + nearest_seen: set[str] = set() + for name in _text_list(parsed.get("nearest_categories"), 6, 200): + value = str(name).strip() + key = value.lower() + if key and key not in nearest_seen and is_real_category(value): + nearest_seen.add(key) + nearest_categories.append(value) + + bad_category = bool(category) and not is_real_category(category) + bad_area = bool(area) and not is_real_area(area) + if bad_category: + logger.warning("[BrowseSubject] discarded unlisted category %r", category) + category = "" + if bad_area: + logger.warning("[BrowseSubject] discarded unlisted area %r", area) + area = "" + + # Exact typed terms always win over a model's alternative valid term. + if typed_category: + category = typed_category + area = "" + confidence = "exact" + mapping_type = "exact" + if typed_area: + area = typed_area + category = "" + confidence = "exact" + mapping_type = "exact" + + # A model may conservatively return the chosen nearest category only in + # nearest_categories. Use it only when the model explicitly says the + # fallback is safe, or when it labelled the response "near". A true + # not_found with use_nearest_category=false remains a no-result response. + used_nearest_fallback = False + if not category and not area and nearest_categories and ( + use_nearest_category or confidence == "near" + ): + category = nearest_categories[0] + confidence = "near" + mapping_type = "nearest_category" + used_nearest_fallback = True + if not normalised or normalised == subject_text: + normalised = category + + # Last resort after the semantic classifier was unavailable or found no + # mapping: a partial whole-word match on a distinctive term (for example + # "quantum physics" -> "Physics and Astronomy"). Disclosed as nearest. + if llm_attempted and not category and not area and not typed_category and not typed_area: + fallback = nearest_subject_term(subject_text, allow_partial=True) + if fallback: + fallback_name = str(fallback["name"]) + if fallback["kind"] == "category": + category = fallback_name + else: + area = fallback_name + confidence = "near" + mapping_type = "nearest_category" + used_nearest_fallback = True + normalised = fallback_name + # The classifier's failure/no-match explanation no longer applies. + user_message = "" + for name in fallback.get("alternatives") or []: + key = str(name).strip().lower() + if key and key not in nearest_seen and is_real_category(key): + nearest_seen.add(key) + nearest_categories.append(key) + + selected_term = category or area + if selected_term: + display_term = selected_term.title() + if mapping_type == "spelling_correction": + disclosure = ( + f'Spelling corrected: "{subject_text}" → "{display_term}". ' + f'Results use this verified {"category" if category else "broad area"}.' + ) + elif mapping_type == "nearest_category" or used_nearest_fallback: + disclosure = ( + f'SCImago and Scopus have no exact subject category for "{subject_text}". ' + f'ResearchBee selected the nearest verified ' + f'{"category" if category else "subject area"} "{display_term}".' + ) + elif mapping_type == "synonym" and selected_term.lower() != subject_text.lower(): + disclosure = ( + f'Interpreted "{subject_text}" as the verified ' + f'{"category" if category else "area"} "{display_term}".' + ) + elif confidence == "near" and selected_term.lower() != subject_text.lower(): + disclosure = ( + f'Interpreted "{subject_text}" using the closest verified ' + f'{"category" if category else "area"} "{display_term}".' + ) + else: + disclosure = "" + if disclosure and disclosure.lower() not in user_message.lower(): + user_message = f"{disclosure} {user_message}".strip() + + def suggestions() -> list[dict]: + out: list[str] = [] + seen: set[str] = set() + for name in nearest_categories: + key = str(name).strip().lower() + if key and key not in seen and is_real_category(key): + seen.add(key) + out.append(key) + for name in suggest_categories(subject_text, limit=8): + key = str(name).strip().lower() + if key and key not in seen and is_real_category(key): + seen.add(key) + out.append(key) + counts = subject_vocabulary()["categories"] + return [{"category": name.title(), "n_journals": counts.get(name, 0)} for name in out[:6]] + + have_category = bool(category) + # Permit a corrected or LLM-resolved broad area when no category is + # available. Exact typed areas continue to take precedence. + have_area = bool(area) and not have_category + if confidence == "not_found" or (not have_category and not have_area): + usage.usage.record_gap( + "journal_subject", + "category_invented" if bad_category else "subject_unrecognised", + ) + request.state.usage_recorded = True + nearby = suggestions() + base = f"SCImago and Scopus have no exact subject category for '{subject_text}'." + hint = ( + " Closest verified categories: " + ", ".join(item["category"] for item in nearby) + "." + if nearby else "" + ) + return {"result": { + "schema_version": "journal_discovery_v2", + "mode": "subject", + "confidence": "not_found", + "normalised": normalised, + "user_message": (user_message + " " if user_message else "") + base + hint, + "subject_mapping": { + "original_input": subject_text, + "normalised": normalised, + "match_type": "none", + "selected_category": "", + "selected_area": "", + "used_nearest_category": False, + "spelling_corrected": False, + "alternatives": nearby, + }, + "suggestions": nearby, + "journals": [], + "extended_list": [], + "total": 0, + }} + + index = get_index() + if index is None: + raise HTTPException( + 503, + "The verified journal index is unavailable. ResearchBee will not generate unverified journal names as a fallback.", + ) + + preferences = _preferences_from_subject(req) + asjc_rows = resolve_asjc_rows( + index, + category=category if have_category else "", + area=area if have_area else "", + ) + query_text = _subject_query_text(req.subject, normalised, category, area) + pool = build_candidate_pool( + index, + title=query_text, + abstract="", + asjc_rows=asjc_rows, + preferences=preferences, + limit=50, + candidate_pool=600, + ) + record_map = {rec.journal.scopus_id: rec for rec in pool.recommendations} + asjc_codes = [str(row.get("code")) for row in asjc_rows if row.get("code")] + candidates = [ + _candidate_base_dict(rec, asjc_codes, pool.score_ratios.get(rec.journal.scopus_id, 0.0)) + for rec in pool.recommendations + ] + await _attach_article_evidence(candidates, query_text, limit=12) + ranked = _score_subject_candidates( + candidates, record_map, asjc_codes, preferences, exact_category=have_category + ) + ranked, policy_notes = await _apply_sharing_requirement(ranked, preferences) + + unverified = list(pool.unverified_criteria) + if policy_notes: + unverified = [ + note for note in unverified + if not note.startswith("Accepted-manuscript sharing will be checked") + ] + unverified.extend(policy_notes) + if have_area and not have_category: + if area.title() in user_message: + # The nearest/synonym disclosure already names the area. + area_note = "Results span this broad area; choose a narrower category below for a more specific list." + else: + area_note = ( + f"'{subject_text}' was resolved to the broad subject area " + f"'{area.title()}'. Results span that area; choose a narrower category for a more specific list." + ) + user_message = f"{user_message} {area_note}".strip() + usage.usage.record_gap("journal_subject", "area_fallback") + request.state.usage_recorded = True + + all_suggestions = suggestions() + alternatives = [ + item for item in all_suggestions + if item["category"].lower() != str(category).lower() + ][:4] + result = _finalise_discovery_result( + mode="subject", + preferred_titles=preferences.preferred_journals, + ranked=ranked, + applied_criteria=pool.applied_criteria, + unverified_criteria=unverified, + legacy_notes=pool.legacy_notes, + next_actions=[], + global_notes=user_message, + subject_understanding={ + "summary": f"Verified journal discovery for {normalised}.", + "discipline": normalised, + "category": category, + "area": area, + "confidence": confidence, + "mapping_type": mapping_type, + "original_input": subject_text, + "alternatives": alternatives, + }, + classification_note=( + "This is a broad area result, not a manuscript-specific fit assessment." + if have_area and not have_category else + "Subject relevance is based on the verified ASJC/SCImago category mapping." + ), + ) + result.update({ + "confidence": confidence, + "normalised": normalised, + "scimago_category": category, + "scimago_area": area, + "user_message": user_message, + "asjc_codes": asjc_codes, + "subject_mapping": { + "original_input": subject_text, + "normalised": normalised, + "match_type": mapping_type, + "selected_category": category, + "selected_area": area, + "used_nearest_category": mapping_type == "nearest_category" or used_nearest_fallback, + "spelling_corrected": mapping_type == "spelling_correction", + "alternatives": alternatives, + }, + "suggestions": alternatives if confidence == "near" else [], + }) + if not ranked: + result["filter_no_match"] = True + result["global_notes"] = ( + "The subject was recognised, but no active journal met every selected hard filter. " + "ResearchBee did not silently relax those filters." + ) + return {"result": result} + + +@app.post("/api/export-citations") +async def export_citations(req: ExportRequest): + fmt = req.format.lower() + journals = [item.model_dump() for item in req.journals] + + if fmt == "bibtex": + lines = [] + for i, j in enumerate(journals): + key = f"journal{i + 1}" + def bibtext(value): + return str(value or "").replace("\\", " ").replace("{", "(").replace("}", ")").replace("\n", " ").replace("\r", " ") + j = {name: bibtext(value) if name != "ranking" else value for name, value in j.items()} + lines.append( + f"@misc{{{key},\n" + f" title = {{{j.get('name', '')}}},\n" + f" howpublished = {{Journal shortlist record, not an article citation}},\n" + f" publisher = {{{j.get('publisher', '')}}},\n" + f" issn = {{{j.get('issn', '')}}},\n" + f" note = {{Quartile: {j.get('quartile') or (j.get('ranking') or {}).get('quartile', '—')}. Recommended by ResearchBee.}}\n}}" + ) + return {"content": "\n\n".join(lines), "filename": "journals.bib", "mimetype": "text/plain"} + + elif fmt == "ris": + lines = [] + for j in journals: + j = {key: " ".join(value.split()) if isinstance(value, str) else value for key, value in j.items()} + q = j.get("quartile") or (j.get("ranking") or {}).get("quartile", "") + lines.append( + f"TY - GEN\nTI - {j.get('name', '')}\n" + f"PB - {j.get('publisher', '')}\nSN - {j.get('issn', '')}\n" + f"N1 - Journal shortlist record, not an article citation. Quartile: {q}. Recommended by ResearchBee.\nER - " + ) + return {"content": "\n\n".join(lines), "filename": "journals.ris", "mimetype": "text/plain"} + + raise HTTPException(400, "Format must be bibtex or ris") + + +@app.post("/api/cover-letter") +async def generate_cover_letter(req: CoverLetterRequest): + client = get_client() + lang = "Respond in Arabic." if req.language == "arabic" else "Respond in English." + user_msg = ( + f"{lang}\n\nGenerate a cover letter for the following submission:\n\n" + f"Manuscript title: {req.manuscript_title}\nAbstract: {req.abstract}\n" + f"Article type: {req.article_type}\nTarget journal: {req.journal_name}\n" + f"Publisher: {req.publisher or 'Not specified'}\nDiscipline: {req.discipline or 'Not specified'}\n" + f"Author name: {req.author_name or 'The Author(s)'}\n\nReturn JSON only." + ) + parsed = await call_openai(client, COVER_LETTER_PROMPT, user_msg, max_tokens=800, model=MODEL_LIGHT) + if parsed.get("error"): + raise HTTPException(503, parsed["error"]) + if not isinstance(parsed.get("cover_letter"), str) or not parsed["cover_letter"].strip(): + raise HTTPException(503, "The AI service returned an invalid cover-letter draft. Please try again.") + parsed["cover_letter"] = parsed["cover_letter"][:12000] + parsed["subject_line"] = str(parsed.get("subject_line") or "Manuscript submission")[:500] + parsed["requires_author_review"] = True + parsed["review_note"] = "Verify every submission declaration and replace the bracketed author-confirmation placeholders before use." + return {"result": parsed} + + +# ── Chat endpoint — streaming SSE ───────────────────────────────────────── +@app.post("/api/chat") +async def chat(req: ChatRequest, request: Request): + """ + Three-layer resolution with streaming: + 1. Keyword pre-filter — instant routing, no LLM call + 2. Open Policy Finder — live policy lookup for specific journals + 3. LLM stream — tokens stream via SSE; route extracted from full reply at end + """ + ip = get_client_ip(request) + if not check_rate_limit(ip, "chat"): + raise HTTPException(429, "Chat limit reached. Please wait a minute before sending more messages.") + + if not req.messages: + async def empty_stream(): + greeting = "Hi! How can I help you with publishing or Open Access today?" + yield f'data: {json.dumps({"token": greeting, "done": False})}\n\n' + yield f'data: {json.dumps({"token": "", "done": True, "reply": greeting, "route": None, "prefill": {}})}\n\n' + return StreamingResponse(empty_stream(), media_type="text/event-stream", + headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}) + + last_msg = req.messages[-1].content.strip() + + # ── Layer 1: keyword pre-filter — instant, no streaming needed ──────── + fast_route = quick_route(last_msg) + if fast_route: + TAB_DESCRIPTIONS = { + "journals": "Journal Submission tool", + "license": "License Checking tool", + "data": "Data Repository tool", + } + reply = ( + f"The {TAB_DESCRIPTIONS[fast_route]} above is perfect for this! " + f"It'll give you accurate, structured results in seconds." + ) + if req.language == "arabic": + reply = f"أداة {TAB_DESCRIPTIONS[fast_route]} المتاحة أعلاه مناسبة تماماً لهذا! ستمنحك نتائج دقيقة ومنظمة في ثوانٍ." + # Return as SSE with route + async def fast_stream(): + payload = json.dumps({"token": reply, "done": False}) + yield f"data: {payload}\n\n" + done = json.dumps({"token": "", "done": True, "route": fast_route, "prefill": {}}) + yield f"data: {done}\n\n" + return StreamingResponse(fast_stream(), media_type="text/event-stream", + headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}) + + # ── Layer 2: Open Policy Finder ─────────────────────────────────────── + opf_data = "" + journal_name = detect_journal_policy_question(last_msg) + if journal_name: + opf_data = await fetch_opf_policy(journal_name) + + # ── Layer 3: LLM streaming ──────────────────────────────────────────── + client = get_client() + + try: + retrieved_knowledge = get_chat_knowledge(last_msg, context_query=" ".join(m.content for m in req.messages[-5:-1] if m.role == "user")[-2000:]) + except Exception as exc: # noqa: BLE001 - chat should degrade safely + logger.warning("[Chat] Knowledge retrieval unavailable (%s)", type(exc).__name__) + retrieved_knowledge = ( + "The curated knowledge index is temporarily unavailable. Give only stable general " + "guidance, do not invent current policies or links, and direct KU-specific questions " + "to the KU Library." + ) + + system = CHAT_SYSTEM_PROMPT_BASE.replace( + "{RETRIEVED_KNOWLEDGE}", retrieved_knowledge + ).replace( + "{OPF_DATA}", opf_data if opf_data else "No live Open Policy Finder data was retrieved for this query." + ) + + if req.language == "arabic": + system += "\n\nRespond entirely in Arabic." + + messages = [{"role": "system", "content": system}] + # Preserve the latest turn while bounding total model context independently + # of the browser's history representation. + history = [] + remaining = 24000 + for message in reversed(req.messages): + text = message.content[-min(remaining, 12000):] + if not text or remaining <= 0: + break + history.append({"role": message.role, "content": text}) + remaining -= len(text) + messages += list(reversed(history)) + + def extract_route(text: str): + """Brace-counting extraction of routing JSON from end of reply.""" + starts = [m.start() for m in re.finditer(r'\{', text)] + for start in reversed(starts): + candidate = text[start:] + depth = 0; end = None + for i, ch in enumerate(candidate): + if ch == '{': depth += 1 + elif ch == '}': + depth -= 1 + if depth == 0: end = i + 1; break + if end is None: continue + try: + parsed = json.loads(candidate[:end]) + if "route" in parsed: + return text[:start].strip(), parsed.get("route"), parsed.get("prefill", {}) + except Exception: + continue + return text.strip(), None, {} + + # Create streaming response outside generator to catch connection errors early + try: + openai_stream = await client.chat.completions.create( + model=MODEL, + messages=messages, + max_tokens=600, + temperature=0.3, + stream=True, + ) + except Exception as e: + logger.warning("[Chat] AI request failed (%s)", type(e).__name__) + raise HTTPException(503, "AI service is temporarily unavailable. Please try again.") from None + + request.state.usage_recorded = True + async def stream_response() -> AsyncGenerator[str, None]: + full_text = "" + try: + async for chunk in openai_stream: + token = chunk.choices[0].delta.content if chunk.choices else None + if token: + full_text += token + yield f'data: {json.dumps({"token": token, "done": False})}\n\n' + + reply, route, prefill = extract_route(full_text) + if not isinstance(route, (str, type(None))) or route not in ("journals", "license", "data", None): + route = None + + done_payload = json.dumps({ + "token": "", "done": True, + "reply": reply, + "route": route, + "prefill": prefill if isinstance(prefill, dict) else {}, + }) + usage.usage.record("chat", "ok") + yield f"data: {done_payload}\n\n" + + except asyncio.CancelledError: + usage.usage.record("chat", "miss", "cancelled") + raise + except Exception as e: + usage.usage.record("chat", "error", "stream_error") + logger.warning("[Chat] Stream failed (%s)", type(e).__name__) + err = json.dumps({"token": "", "error": "The answer was interrupted. Please try again.", "done": True, "route": None, "prefill": {}}) + yield f"data: {err}\n\n" + finally: + await openai_stream.close() + + return StreamingResponse( + stream_response(), + media_type="text/event-stream", + headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}, + ) + + +# ── Journal meta lookup — multi-suggestion with ISSN auto-fill ─────────── +@app.get("/api/lookup-journal-meta") +async def lookup_journal_meta(request: Request, name: str): + """ + Multi-suggestion journal lookup: + Layer 1a: SCImago — up to 4 ranked title matches (name + ISSN + publisher) + Layer 1b: OpenAlex — up to 4 results, merged + deduplicated with SCImago + Layer 2: LLM — only if nothing found; returns up to 3 name suggestions only + (NO ISSN from LLM — those come from re-running Layer 1 on acceptance) + """ + ip = get_client_ip(request) + if not check_rate_limit(ip, "global"): + raise HTTPException(429, "Too many requests.") + + name = name.strip() + if len(name) > 300: + raise HTTPException(422, "Journal name must contain at most 300 characters.") + if not name or len(name) < 3: + return {"suggestions": [], "llm_suggestions": []} + + name_lower = name.lower() + words = [w for w in name_lower.split() if len(w) > 2] + + def score_title(title: str) -> float: + """Score a title against the query — higher = better match.""" + t = title.lower() + if t == name_lower: + return 1.0 + # Word overlap score + overlap = sum(1 for w in words if w in t) + # Bonus for starts-with + starts = 0.2 if t.startswith(name_lower[:6]) else 0 + # Penalty for very different lengths + len_ratio = min(len(name_lower), len(t)) / max(len(name_lower), len(t)) + return (overlap / max(len(words), 1)) * 0.6 + starts + len_ratio * 0.2 + + # ── Layer 1a: SCImago — collect top 4 scored matches ───────────────── + rows = load_subject_index() + scored = [] + for r in rows: + s = score_title(r["title"]) + if s > 0.35: # threshold — avoid noise + scored.append((s, r)) + scored.sort(key=lambda x: -x[0]) + top_scimago = scored[:4] + + suggestions = [] + seen_titles = set() + + for score, r in top_scimago: + t = r["title"] + if t.lower() in seen_titles: + continue + seen_titles.add(t.lower()) + ni = norm_issn(r.get("issn", "")) + suggestions.append({ + "title": t, + "issn": ni or r.get("issn", ""), + "publisher": r.get("publisher", ""), + "source": "SCImago", + "confidence": "high" if score > 0.8 else "medium", + "score": round(score, 3), + }) + + # ── Layer 1b: OpenAlex — up to 4, merge with SCImago results ───────── + try: + oa_client = openalex._get_client() + r = await oa_client.get( + "https://api.openalex.org/sources", + params={"search": name, "per_page": 4, "filter": "type:journal"}, + timeout=6.0, + ) + if r.is_success: + for s in r.json().get("results", []): + title = s.get("display_name", "") + if not title or title.lower() in seen_titles: + continue + issns = s.get("issn", []) + issn_l = s.get("issn_l", "") + publisher = s.get("host_organization_name", "") + best_issn = issn_l or (issns[0] if issns else "") + ni = norm_issn(best_issn) + sc = score_title(title) + seen_titles.add(title.lower()) + suggestions.append({ + "title": title, + "issn": ni or best_issn, + "publisher": publisher, + "source": "OpenAlex", + "confidence": "high" if sc > 0.8 else "medium", + "score": round(sc, 3), + }) + except Exception as e: + logger.warning("[LookupJournal] OpenAlex unavailable (%s)", type(e).__name__) + + # Re-sort merged list by score, keep top 4 + suggestions.sort(key=lambda x: -x.get("score", 0)) + suggestions = suggestions[:4] + + # If we have good suggestions, return them + if suggestions: + return {"suggestions": suggestions, "llm_suggestions": []} + + # ── Layer 2: LLM — name suggestions only, NO ISSN ──────────────────── + llm_suggestions = [] + try: + client = get_client() + resp = await client.chat.completions.create( + model=MODEL, + response_format={"type": "json_object"}, + messages=[ + { + "role": "system", + "content": ( + "You are an academic journal name expert. " + "The user has typed a journal name that may be misspelled or abbreviated. " + "Return ONLY valid JSON: " + "{suggestions: [{name: string, confidence: high|medium|low}]} " + "Return up to 3 most likely correct journal names, ranked by confidence. " + "Only include journals you are confident exist. " + "Do NOT return ISSN, publisher, or any other fields. " + "Return JSON only." + ), + }, + {"role": "user", "content": f"Journal name typed by user: {name}"}, + ], + max_tokens=150, + temperature=0.1, + ) + raw = resp.choices[0].message.content or "{}" + parsed = json.loads(raw) + for s in parsed.get("suggestions", [])[:3]: + n = s.get("name", "").strip() + c = s.get("confidence", "low") + if n and n.lower() != name_lower and c in ("high", "medium"): + llm_suggestions.append({"name": n, "confidence": c}) + except Exception as e: + logger.warning("[LookupJournal] AI lookup unavailable (%s)", type(e).__name__) + + return {"suggestions": [], "llm_suggestions": llm_suggestions} + + +# ── Static frontend ──────────────────────────────────────────────────────── +@app.get("/") +async def root(): + if os.path.exists("static/index.html"): + return FileResponse("static/index.html") + return {"status": "ResearchBee API running", "docs": "/docs"} + + +if os.path.exists("static"): + if os.path.exists("static/css"): + app.mount("/css", StaticFiles(directory="static/css"), name="css") + if os.path.exists("static/js"): + app.mount("/js", StaticFiles(directory="static/js"), name="js") + + +class JournalLinksRequest(BaseModel): + scopus_id: str = Field("", max_length=30) + issn: str = Field("", max_length=20) + +@app.post("/api/journal-links") +async def find_author_guidelines(req: JournalLinksRequest, request: Request): + if not check_rate_limit(get_client_ip(request), "global"): + raise HTTPException(429, "Too many requests. Please try again later.") + return await discover(req.model_dump())