Blame for app.py

"""

ResearchBee - FastAPI backend v1.8.3 (nearest-subject matching, re3data supplement)



Changes in v1.8.1:

- Subject browsing applies conservative closed-vocabulary spelling correction before the LLM.

- Meaningful unlisted subjects may be mapped to the nearest verified category with explicit disclosure.

- Subject responses include structured mapping provenance and alternative verified categories.

- The backend version aligns with the unified comparison/OA front end.



Changes in v1.8.0:

- Browse by subject and manuscript matching now share one database-first journal pipeline.

- Journal identities, ISSNs, metrics, active status and access flags come from journals.db.

- Hard filters are enforced in code; unsupported top-percent and fee claims are not inferred.

- Results include explainable fit components, recent OpenAlex article evidence, comparison data,

  and on-demand Jisc Open Policy Finder checks.

- Accepted-manuscript sharing requirements trigger policy checks for leading candidates.



Changes in v1.7.0:

- Chat knowledge is stored as source-aware topic records and retrieved per question.

- Educational journal, metrics, publishing, OA, and RDM questions are answered in chat.

- Fast routing is limited to explicit tool requests so it no longer hides useful guidance.

- The chat prompt no longer claims unrestricted web browsing.



Changes in v1.6.0:

- Journal-level licence checks use the authenticated Jisc Open Policy Finder API.

- The DOI-first OA.Works + Khazna path remains unchanged and primary.

- The former public-page renderer has been removed.

- Open Policy Finder policy facts are displayed as source data and are never generated by an LLM.



Changes from v1.3 (verified-sources release):

- /api/check-license is now DOI-first and GROUNDED. Policy facts come from the

  OA.Works Permissions API (the dataset behind cOAlition S's Journal Checker

  Tool), never from the LLM. The LLM writes advisory prose only.

- Khazna deposit status via a nightly OAI-PMH harvest (three-valued: checked /

  in_khazna / needs_deposit — "could not check" is never reported as "absent").

- Every policy answer carries a provenance label. Any LLM-derived

  policy_status is forcibly downgraded to "Not confirmed".

- Repository finder constrained to a curated registry; the LLM selects and

  explains, the server supplies the facts.



Changes from v1.2:

- Rate limiting middleware (sliding window, 20 req/min global, 10/min chat per IP)

- /api/chat now streams via SSE (StreamingResponse)

- /api/openalex-works — related literature per journal (OpenAlex)

- /api/altmetric — journal attention score badge (Altmetric free API)

"""
import os
import json
import re
import asyncio
import logging
import secrets
from trusted_links import with_links, discover
import time
from contextlib import asynccontextmanager
from typing import Optional, List, AsyncGenerator, Any, Sequence, Literal

# Process boot time, captured at import so it is set before any heavy
# module-level work and before lifespan runs. Exposed on /api/health so
# the uptime monitor can distinguish "still running" from "died and came
# back between polls" — a restarted container reports RUNNING and 200 at
# both polls, but its boot_ts moves.
BOOT_TS = time.time()

from fastapi import FastAPI, HTTPException, Request
from fastapi.middleware.cors import CORSMiddleware
from fastapi.staticfiles import StaticFiles
from fastapi.responses import FileResponse, StreamingResponse
from pydantic import BaseModel, Field, model_validator

from openai import AsyncOpenAI, RateLimitError, APIStatusError
import httpx
from urllib.parse import quote_plus

# journal_lookup wraps scimago_index and serves lookups from journals.db
# (Scopus source list x SCImago 2025). It forwards every other name straight
# through to scimago_index, so the subject-vocabulary functions below behave
# exactly as before. If journals.db is unreadable it falls back to the CSV,
# so this import cannot take the Space down.
from journal_lookup import (
    lookup_any, load_scimago, search_by_subject, load_subject_index,
    subject_vocabulary, is_real_category, is_real_area, suggest_categories,
    closest_subject_term, nearest_subject_term, get_index,
)
import openalex
import openpolicyfinder
from prompts import (
    JOURNAL_SYSTEM_PROMPT, JOURNAL_PROFILE_PROMPT, JOURNAL_RERANK_PROMPT,
    REPO_SYSTEM_PROMPT, SUBJECT_NORMALISE_PROMPT, COVER_LETTER_PROMPT,
    CHAT_SYSTEM_PROMPT_BASE,
)
from chat_knowledge import get_chat_knowledge, knowledge_status
from utils import norm_issn
from journal_discovery import (
    DiscoveryPreferences, build_candidate_pool, resolve_asjc_rows,
    matching_subjects, quartile_for, access_model, ranking_component,
    practical_component, policy_requirement_state, clamp_int, normalise_title,
)

# ── Verified-source modules (no LLM inside any of these) ──────────────────
import oaworks
import doi_utils
import repositories
import re3data
from khazna_loader import khazna_index
from repositories import REGISTRY, registry_for_prompt, hydrate_selection
from prompts import ADVISORY_NOTE_PROMPT

# ── Usage counting + admin dashboard ──────────────────────────────────
# usage.py counts how often each tool is used. It stores COUNTS ONLY —
# no question text, no abstracts, no DOIs — see the module docstring.
import usage
from admin import router as admin_router
from runtime_security import RuntimeSecurityMiddleware, check_rate_limit, get_client_ip
from policy_evaluation import evaluate_policy

logger = logging.getLogger(__name__)

# ── Constants ──────────────────────────────────────────────────────────────
# Models are server-controlled; browser input cannot select an arbitrary model.
MODEL = os.getenv("RESEARCHBEE_MODEL", "gpt-4.1-mini")
MODEL_LIGHT = os.getenv("RESEARCHBEE_MODEL_LIGHT", "gpt-4.1-nano")
RELEASE_VERSION = "1.8.3"

# ── KU publisher-agreement candidates ─────────────────────────────────────
# Publisher matching is a discovery signal only. It does not establish title-
# level APC eligibility, author eligibility, quota availability, article-type
# coverage, or an exact top-percentile requirement.
# Aliases that are ordinary English and would over-match as substrings.
# These only count when they lead the publisher string.
AMBIGUOUS_ALIASES = {
    "academic press", "saunders", "masson", "adis", "informa",
    "nature research", "nature publishing",
}

KU_APC_PUBLISHER_GROUPS = [
    # ACS
    {"american chemical society"},
    # Elsevier — all regional entities
    {"elsevier", "elsevier b.v.", "elsevier bv", "elsevier ltd", "elsevier inc.",
     "elsevier inc", "elsevier ireland ltd", "elsevier gmbh", "elsevier masson",
     "elsevier singapore", "elsevier australia", "elsevier beijing",
     "elsevier editora", "elsevier espana",
     # Imprints whose SCImago string never contains "Elsevier".
     # NB: normalisation strips dots, so "W.B. Saunders" becomes "wb saunders"
     # — that variant must be listed explicitly or ~108 titles are missed.
     "academic press", "cell press", "wb saunders", "w b saunders",
     "churchill livingstone", "mosby", "pergamon", "saunders",
     "masson", "urban and fischer", "hanley and belfus"},
    # IEEE — all society variants
    {"ieee", "ieee advancing technology", "ieee communications society",
     "ieee computational intelligence", "ieee computer society",
     "ieee electron devices", "ieee microwave"},
    # Springer Nature — all Springer variants
    {"springer", "springer nature", "springernature", "nature portfolio",
     "nature publishing", "springer international publishing",
     "springer international publishing ag", "springer berlin", "springer boston",
     "springer china", "springer netherlands", "springer new york",
     "springer london", "springer japan", "springer verlag",
     "springer science and business media", "springer publishing",
     "springer science and business media llc", "springer nature america",
     "biomed central", "palgrave", "palgrave macmillan", "adis",
     "birkhauser", "nature research"},
    # Taylor & Francis — all T&F variants (note: & in SCImago CSV)
    {"taylor and francis", "taylor & francis", "taylor & francis",
     "routledge", "routledge, taylor", "taylor and francis ltd",
     "taylor and francis inc", "taylor and francis a.s",
     "taylor and francis asia pacific", "informa", "informa uk",
     "informa uk limited", "psychology press", "crc press",
     "dove medical press", "taylor and francis group"},
    # Wiley — all Wiley variants
    {"wiley", "wiley-blackwell", "wiley-blackwell publishing",
     "john wiley and sons", "john wiley & sons", "john wiley & sons",
     "wiley-liss", "wiley-vch", "wiley - vch",
     "blackwell", "blackwell publishing", "blackwell science",
     "john wiley and sons inc", "john wiley and sons ltd"},
]

def is_ku_agreement_publisher(publisher: str) -> bool:
    """Whether the publisher string matches a KU agreement publisher group.



    This is only a publisher-level candidate signal. It is never treated as

    title-level APC eligibility, quota availability, or a top-percentile check.

    """
    pub = (publisher or "").lower().strip()
    if not pub:
        return False

    def norm(value: str) -> str:
        return (value.lower()
                .replace("&", "&").replace("&", "and")
                .replace(".", "").replace(",", "")
                .replace("-", " ").strip())

    pub_norm = norm(pub)
    pub_tokens = pub_norm.split()
    for group in KU_APC_PUBLISHER_GROUPS:
        for alias in group:
            alias_norm = norm(alias)
            alias_tokens = alias_norm.split()
            if pub_norm == alias_norm:
                return True
            if alias_norm in AMBIGUOUS_ALIASES:
                if pub_norm.startswith(alias_norm + " "):
                    return True
                continue
            n = len(alias_tokens)
            for i in range(len(pub_tokens) - n + 1):
                if pub_tokens[i:i + n] == alias_tokens:
                    return True
    return False


def is_ku_apc_covered(publisher: str, quartile: str = "") -> bool:
    """Backward-compatible candidate flag; not a coverage determination."""
    return quartile == "Q1" and is_ku_agreement_publisher(publisher)


# Request identity and bounded quotas are shared with runtime_security.

KHAZNA_RELOAD_HEADER = "X-ResearchBee-Reload-Token"


def require_khazna_reload_auth(request: Request) -> None:
    """Require the Space secret before allowing a Khazna index reload."""
    expected = os.environ.get("KHAZNA_RELOAD_TOKEN", "").strip()
    if not expected:
        raise HTTPException(
            503,
            "Khazna reload authentication is not configured.",
        )

    supplied = request.headers.get(KHAZNA_RELOAD_HEADER, "").strip()
    if not supplied:
        auth = request.headers.get("Authorization", "").strip()
        if auth.lower().startswith("bearer "):
            supplied = auth[7:].strip()

    if not supplied:
        raise HTTPException(
            401,
            "Khazna reload token required.",
            headers={"WWW-Authenticate": "Bearer"},
        )
    if not secrets.compare_digest(supplied, expected):
        raise HTTPException(403, "Invalid Khazna reload token.")



# ── Khazna article workflow contact card ─────────────────────────────────
KHAZNA_DEPOSIT_CARD = {
    "title": "🏛️ Deposit to Khazna — KU Institutional Repository",
    "url": "https://khazna.ku.ac.ae",
    "contact": "khazna@ku.ac.ae",
    "library_url": "https://library.ku.ac.ae/lib",
    "message": "Register your publication with Khazna. Deposit a manuscript file only when the checked version, repository location, licence and embargo conditions permit it; contact KU Library when permission is unclear.",
    "metadata_tip": "Even if depositing data in a domain-specific repository, always register metadata in Khazna so your work appears in KU's research portfolio.",
}

FAST_ROUTE_PATTERNS = {
    "journals": [
        r"\b(?:find|recommend|suggest|match|identify|show me|give me)\b.{0,100}\bjournal(?:s)?\b",
        r"\bwhere (?:should|can) i publish\b",
        r"\banaly[sz]e my (?:paper|manuscript|abstract)\b",
        r"\bbrowse journals?\b",
        r"\bopen (?:the )?journal (?:finder|tool|tab)\b",
    ],
    "license": [
        r"\b(?:open|use|show|take me to)\b.{0,50}\b(?:license|sharing|self[- ]?archiv) (?:checker|tool|tab)\b",
        r"\bopen policy checker\b",
    ],
    "data": [
        r"\b(?:find|recommend|suggest|match|identify|show me|give me)\b.{0,100}\b(?:data )?repositor(?:y|ies)\b",
        r"\bwhere (?:should|can) i deposit (?:my )?(?:data|dataset)\b",
        r"\bwhich repository (?:should|can) i use for (?:my )?(?:data|dataset)\b",
        r"\bopen (?:the )?(?:data )?repository (?:finder|tool|tab)\b",
    ],
}

_INFORMATIONAL_OPENERS = (
    "what is", "what are", "what does", "how does", "how do", "how can",
    "why is", "why are", "explain", "tell me about", "difference between",
    "compare", "which metric", "when should", "tips for", "criteria for",
)


def quick_route(message: str) -> Optional[str]:
    """Return a tab only for an explicit structured-tool request.



    The previous broad keyword router sent questions such as "What is FAIR

    data?" or "How do I choose the best journal?" straight to a tab. That made

    an expanded knowledge base unreachable. Informational questions now go to

    the assistant, while clear action requests still receive an instant route.

    """
    msg = " ".join(message.lower().split())
    if not msg:
        return None
    if msg.startswith(_INFORMATIONAL_OPENERS):
        return None
    for tab, patterns in FAST_ROUTE_PATTERNS.items():
        if any(re.search(pattern, msg, flags=re.IGNORECASE) for pattern in patterns):
            return tab
    return None


# ── Open Policy Finder (Jisc) authenticated API ──────────────────────────
async def fetch_opf_policy(journal_name: str, issn: Optional[str] = None) -> str:
    """Return a grounded journal-level summary for the chat assistant."""
    record = await openpolicyfinder.lookup_policy(journal_name, issn or "")
    return openpolicyfinder.format_for_chat(record)


def detect_journal_policy_question(message: str) -> Optional[str]:
    """

    Detect if the user is asking about a specific journal's self-archiving policy.

    Returns the journal name if detected, else None.

    Uses simple heuristics — good enough for 90% of cases.

    """
    msg = message.lower()
    policy_triggers = [
        "self-archiv", "self archiv", "can i deposit", "can i post",
        "embargo", "postprint", "accepted manuscript", "preprint policy",
        "archiving rights", "green oa", "repository policy",
        "version of record", "aam", "open access rights",
    ]
    if not any(t in msg for t in policy_triggers):
        return None

    # Try to extract journal name — look for quoted names or "in X journal" patterns
    quoted = re.findall(r'["\']([^"\']{3,80})["\']', message)
    if quoted:
        return quoted[0]

    patterns = [
        r'\bin\s+([A-Z][A-Za-z &\-]{3,60}(?:Journal|Review|Letters|Reports|Science|Nature|Communications|Research)?)',
        r'\bfor\s+([A-Z][A-Za-z &\-]{3,60}(?:Journal|Review|Letters|Reports|Science|Nature|Communications|Research)?)',
        r'\b([A-Z][A-Za-z &\-]{3,60}(?:Journal|Review|Letters|Reports|Science|Nature|Communications|Research))\b',
    ]
    for pat in patterns:
        m = re.search(pat, message)
        if m:
            candidate = m.group(1).strip()
            if len(candidate) > 3:
                return candidate
    return None


# ── Singleton OpenAI client ────────────────────────────────────────────────
_openai_client: Optional[AsyncOpenAI] = None


def get_client() -> AsyncOpenAI:
    global _openai_client
    key = os.environ.get("OPENAI_API_KEY")
    if not key:
        raise HTTPException(500, "OPENAI_API_KEY not configured")
    if _openai_client is None:
        _openai_client = AsyncOpenAI(api_key=key, timeout=45.0, max_retries=0)
    return _openai_client


# ── Lifespan ───────────────────────────────────────────────────────────────
async def refresh_khazna() -> None:
    """A native coroutine is scheduled on the event loop by APScheduler."""
    await asyncio.to_thread(khazna_index.reload)


@asynccontextmanager
async def lifespan(app: FastAPI):
    load_scimago()
    load_subject_index()
    openalex.init_client()
    re3data.init_client()
    await openpolicyfinder.init_client()
    usage.usage.start()
    refresh_task = asyncio.create_task(refresh_khazna())
    scheduler = None
    try:
        from apscheduler.schedulers.asyncio import AsyncIOScheduler
        from apscheduler.triggers.cron import CronTrigger
        scheduler = AsyncIOScheduler(timezone="Asia/Dubai")
        scheduler.add_job(
            refresh_khazna,
            CronTrigger(hour=23, minute=30, timezone="Asia/Dubai"),
            id="khazna_refresh", replace_existing=True, max_instances=1,
        )
        scheduler.start()
        logger.info("Khazna refresh scheduled for 23:30 Asia/Dubai")
    except Exception as exc:
        logger.warning("Khazna scheduler unavailable (%s)", type(exc).__name__)
    app.state.khazna_scheduler = scheduler
    try:
        yield
    finally:
        if scheduler:
            scheduler.shutdown(wait=False)
        if not refresh_task.done():
            refresh_task.cancel()
        await asyncio.gather(refresh_task, return_exceptions=True)
        await usage.usage.stop()
        await openpolicyfinder.close_client()
        await openalex.close_client()
        await re3data.close_client()
        global _openai_client
        if _openai_client is not None:
            await _openai_client.close()
            _openai_client = None


# ── App ────────────────────────────────────────────────────────────────────
app = FastAPI(title="ResearchBee API", version=RELEASE_VERSION, lifespan=lifespan)

app.add_middleware(RuntimeSecurityMiddleware)
app.add_middleware(
    CORSMiddleware,
    allow_origins=[
        "https://ku-library.github.io",
        # The ResearchBee fragment is embedded in LibGuides and calls this
        # API from the guide's own origin. Without these entries every
        # request from the guide fails in the browser and the tools appear
        # to do nothing when submitted.
        "https://kustar.libguides.com",
        "https://libguides.ku.ac.ae",
        "http://localhost",
        "http://127.0.0.1",
        "http://localhost:3000",
        "http://127.0.0.1:3000",
    ],
    expose_headers=["Retry-After"],
    allow_methods=["GET", "POST", "OPTIONS"],
    allow_headers=["Content-Type", "Authorization", "X-ResearchBee-Reload-Token"],
)

# Counting observes completed responses; lifespan owns the usage task lifecycle.
usage.install(app)

# Admin dashboard at /admin. Set RESEARCHBEE_ADMIN_PASSWORD in the Space
# secrets; without it the dashboard cannot be opened at all.
app.include_router(admin_router, prefix="/admin")


# ── Helpers ────────────────────────────────────────────────────────────────
def lang_instruction(language: str) -> str:
    return "Please respond entirely in Arabic." if language == "arabic" else ""


def build_verify_links(name: str, issn: Optional[str], sourceid: str = "") -> dict:
    q = re.sub(r"[^\w\s]", "", name).strip()
    q_encoded = q.replace(" ", "%20")
    ni = norm_issn(issn)
    return {
        "scimago":        (f"https://www.scimagojr.com/journalsearch.php?q={ni}&tip=iss"
                           if ni else f"https://www.scimagojr.com/journalsearch.php?q={q_encoded}&tip=jou"),
        "sherpa_romeo":   (f"https://openpolicyfinder.jisc.ac.uk/search?search={q_encoded}"
                           "&per_page=10&publication_page=1&publisher_page=1&funder_page=1"),
        "doaj":           (f"https://doaj.org/toc/{ni.replace('-', '')}"
                           if ni else f"https://doaj.org/search/journals?source=%7B%22query%22%3A%7B%22query_string%22%3A%7B%22query%22%3A%22{q_encoded}%22%7D%7D%7D"),
        "openalex":       (f"https://openalex.org/sources/issn:{ni}"
                           if ni else f"https://openalex.org/sources?filter=display_name.search:{q_encoded}"),
        "scopus_sources": "https://www.scopus.com/sources",
        "issn_display":   ni or "",
        "name_display":   name.strip(),
    }


def _index_extras(scimago: Optional[dict]) -> dict:
    """

    journals.db-only fields, guarded.



    Additive metadata must never be able to fail a request. If anything in

    here raises, the caller still gets a complete ranking block from the

    fields that predate the journals.db work. The warning names the journal so

    a recurring failure is traceable rather than silent.

    """
    if not scimago:
        return {}
    try:
        return {
            "quartiles_by_field": scimago.get("quartiles_by_field") or [],
            "sjr":                scimago.get("sjr"),
            "indexation_state":   scimago.get("state"),
            "indexation_message": scimago.get("state_message"),
            "warn":               bool(scimago.get("warn")),
            "currently_indexed":  scimago.get("currently_indexed", True),
            "successor_title":    scimago.get("successor_title"),
            "oa_diamond":         bool(scimago.get("oa_diamond")),
            "from_scopus":        bool(scimago.get("from_scopus")),
            "from_scimago":       bool(scimago.get("from_scimago", True)),
        }
    except Exception as e:                                    # noqa: BLE001
        logger.warning("[JournalIndex] extras unavailable (%s)", type(e).__name__)
        return {}


def _quartile_interpretation(scimago: Optional[dict]) -> str:
    """

    Per-field quartile sentence, guarded, with the original wording as the

    fallback. 46% of ranked journals rank differently across their own subject

    areas, so the spread is worth showing — but never at the cost of the call.

    """
    try:
        by_field = (scimago or {}).get("quartiles_by_field") or []
        if len(by_field) > 1:
            spread = ", ".join(f"{f['quartile']} in {f['field']}" for f in by_field)
            return f"Quartile varies by subject area: {spread}."
    except Exception as e:                                    # noqa: BLE001
        logger.warning("[JournalIndex] quartile spread unavailable (%s)", type(e).__name__)
    return (f"Quartile verified from journal metadata "
            f"(2025 dataset, matched by ISSN {(scimago or {}).get('matched_issn', '')}).")


def enrich_ranking(journal: dict, scimago: Optional[dict], oa_pct: Optional[dict]) -> dict:
    if scimago:
        q = scimago.get("quartile", "")
        out = {
            "requested": True, "source": "Verified from journal data", "year": "2025",
            "category": scimago.get("categories", ""), "metric_value": None,
            "percentile": f"Rank #{scimago.get('rank', '—')} globally" if scimago.get("rank") else None,
            "quartile": q if q and q != "-" else None,
            "interpretation": _quartile_interpretation(scimago),
            "verification_status": "Confirmed", "h_index": scimago.get("h_index"),
        }
        # journals.db extras are merged in, never inlined. If the index or its
        # shape misbehaves the response is still complete and correct — the
        # caller simply does not get the extra keys.
        out.update(_index_extras(scimago))
        return out
    elif oa_pct:
        return {
            "requested": True, "source": "OpenAlex-derived percentile", "year": "2025",
            "category": oa_pct.get("concept"), "metric_value": None,
            "percentile": f"{oa_pct['percentile']}th percentile within {oa_pct['peer_count']} {oa_pct['concept']} journals",
            "quartile": oa_pct.get("quartile"),
            "interpretation": "OpenAlex-derived percentile — NOT a Scopus/SCImago quartile.",
            "verification_status": "OpenAlex-derived (not a Scopus/SCImago metric)",
        }
    else:
        return {
            "requested": True, "source": "Not available", "year": None,
            "category": None, "metric_value": None, "percentile": None, "quartile": None,
            "interpretation": "No journal metrics match found. Use verify links to check Scopus Sources directly.",
            "verification_status": "Not confirmed — verify manually via Scopus Sources link",
        }


def _scimago_title_fallback(title: str) -> Optional[dict]:
    """

    Fallback: fuzzy title match in SCImago when ISSN lookup fails.

    Used when LLM returns a wrong/hallucinated ISSN.

    """
    if not title:
        return None
    rows = load_subject_index()
    title_lower = title.lower().strip()
    # Exact title match first
    for r in rows:
        if r["title"].lower() == title_lower:
            ni = norm_issn(r.get("issn", ""))
            if ni:
                return lookup_any([ni])
    # Partial match — title contains all significant words
    words = [w for w in title_lower.split() if len(w) > 3]
    if len(words) >= 2:
        for r in rows:
            rt = r["title"].lower()
            if all(w in rt for w in words):
                ni = norm_issn(r.get("issn", ""))
                if ni:
                    result = lookup_any([ni])
                    if result:
                        return result
    return None


async def enrich_journals(journals: list) -> list:
    enriched_oa = await openalex.enrich_all(journals)
    result = []
    for j in enriched_oa:
        issn = j.get("issn")
        oa = j.get("openalex") or {}
        all_issns = [issn, oa.get("issn_l")] + (oa.get("issns") or [])
        scimago = lookup_any(all_issns)

        # ── Title fallback: if ISSN lookup fails, try matching by journal name ──
        if not scimago:
            scimago = _scimago_title_fallback(j.get("name", ""))
            if scimago:
                logger.info(f"[Enrich] Title fallback matched: {j.get('name')}")

        oa_pct = j.get("oa_percentile")
        final_issn = (scimago or {}).get("matched_issn") or oa.get("issn_l") or issn
        j["issn"] = final_issn
        j["ranking"] = enrich_ranking(j, scimago, oa_pct)
        sourceid = (scimago or {}).get("sourceid", "")
        j["verify_links"] = build_verify_links(j.get("name", ""), final_issn, sourceid)
        j.pop("oa_percentile", None)
        result.append(j)
    return result


def enrich_extended(extended: list) -> list:
    result = []
    for j in extended:
        issn = j.get("issn")
        scimago = lookup_any([issn]) if issn else None
        q        = (scimago or {}).get("quartile", "")
        h_index  = (scimago or {}).get("h_index", "")
        publisher= (scimago or {}).get("publisher", "") or j.get("publisher", "")
        oa       = (scimago or {}).get("open_access", "No")
        country  = (scimago or {}).get("country", "")
        sourceid = (scimago or {}).get("sourceid", "")
        j["quartile"]    = q if q and q != "-" else None
        j["h_index"]     = int(h_index) if str(h_index).isdigit() else None
        j["publisher"]   = publisher
        j["open_access"] = oa
        j["country"]     = country

        # warn is True ONLY for the 967 titles Scopus discontinued for cause.
        # It is deliberately False for the 6,855 that were merely renamed:
        # telling a researcher that a journal which changed its name is a
        # quality risk is worse than saying nothing at all.
        #
        # Merged through the guarded helper for the same reason as above: a
        # quick-match row must not be able to fail the whole request.
        j.update(_index_extras(scimago))

        j["verify_links"]  = build_verify_links(j.get("name", ""), issn, sourceid)
        j["ku_apc_covered"] = is_ku_apc_covered(publisher, q)
        result.append(j)
    return result

_POLICY_ACTION_RE = re.compile(
    r"(?:self[- ]?archiv|embargo|deposit|repository|licen[cs]e|preprint|postprint|"
    r"accepted manuscript|version of record|green oa|\baam\b)",
    re.IGNORECASE,
)


def enforce_journal_policy_guard(parsed: dict) -> dict:
    """

    Make the journal-finder workflow incapable of returning LLM policy facts.



    The model may rank and explain journals, but every policy-bearing field is

    replaced with a canonical, three-valued "unknown" response. This guard is

    deliberately server-side so a future prompt regression cannot reintroduce

    apparent permissions, embargoes, or licences.

    """
    if not isinstance(parsed, dict):
        return {}

    for journal in parsed.get("journals") or []:
        if not isinstance(journal, dict):
            continue
        green = oaworks.not_confirmed(
            journal.get("name", ""),
            journal.get("issn", ""),
            reason="Journal Finder evaluates fit only; no article-level permission lookup was performed",
        )
        green["repository_action_note"] = (
            "Use ResearchBee License Checking with the article DOI before depositing any manuscript version."
        )
        green["evidence_note"] = (
            "No article-level permission lookup is performed by Journal Finder. "
            "Policy values from the language model are discarded by the server."
        )
        green["risk_flag"] = "Deposit rights not checked — verify with the article DOI."
        journal["green_oa"] = green
        journal["oa_compliance_note"] = (
            "Journal Finder does not determine self-archiving rights. Use the DOI-based "
            "License Checking tool before depositing any version."
        )

    unknown = oaworks.not_confirmed(
        reason="Journal Finder does not perform an article-level permission lookup"
    )
    parsed["repository_recommendation"] = oaworks.recommendation_from(unknown)

    safe_actions = []
    for action in parsed.get("next_actions") or []:
        action = " ".join(str(action or "").split())
        if action and not _POLICY_ACTION_RE.search(action):
            safe_actions.append(action[:600])
    safe_actions.extend([
        "Verify the journal scope, author guidelines, current fees, and indexation before submission.",
        "Use ResearchBee License Checking with the article DOI before depositing any manuscript version.",
    ])
    # Stable de-duplication.
    parsed["next_actions"] = list(dict.fromkeys(safe_actions))
    parsed["global_notes"] = (
        "Journal Finder evaluates publication fit and verified journal metrics only. "
        "Self-archiving permissions, embargoes, and licences require a DOI-based check."
    )
    parsed["policy_source_mode"] = "not_checked_in_journal_finder"
    return parsed


def build_subject_journal_result(journal: dict) -> dict:
    """Build one subject-browse row, including all DB-backed field quartiles."""
    issn = journal.get("issn")
    scimago = lookup_any([issn]) if issn else None
    if not scimago:
        scimago = _scimago_title_fallback(journal.get("title", ""))

    source = scimago or journal
    final_issn = source.get("matched_issn") or issn or ""
    quartile = source.get("quartile") or journal.get("quartile", "")
    h_index = source.get("h_index", journal.get("h_index"))
    try:
        h_index = int(h_index) if h_index not in (None, "") else None
    except (TypeError, ValueError):
        h_index = None

    result = {
        "name": journal.get("title", ""),
        "issn": final_issn,
        "quartile": quartile if quartile and quartile != "-" else None,
        "h_index": h_index,
        "publisher": source.get("publisher", "") or journal.get("publisher", ""),
        "open_access": source.get("open_access", journal.get("open_access", "No")),
        "country": source.get("country", journal.get("country", "")),
        "categories": source.get("categories", journal.get("categories", "")),
        "verify_links": build_verify_links(
            journal.get("title", ""),
            final_issn,
            source.get("sourceid", journal.get("sourceid", "")),
        ),
    }
    result.update(_index_extras(scimago))
    result["ku_apc_covered"] = is_ku_apc_covered(result["publisher"], quartile)
    return result


# ── OpenAI calls: bounded retries, validated top-level JSON, redacted logs ──
async def call_openai(client: AsyncOpenAI, system: str, user: str, max_tokens: int = 4096, model: str = None) -> dict:
    chosen_model = model or MODEL
    for attempt in range(3):
        try:
            response = await client.chat.completions.create(
                model=chosen_model,
                response_format={"type": "json_object"},
                messages=[{"role": "system", "content": system}, {"role": "user", "content": user}],
                max_tokens=max_tokens,
                temperature=0.3,
            )
            content = response.choices[0].message.content or ""
            if getattr(response.choices[0], "finish_reason", None) == "length":
                logger.warning("Model response truncated: model=%s chars=%d", chosen_model, len(content))
                return {"error": "The answer was cut off before it could be completed. Please try again."}
            try:
                parsed = json.loads(content)
            except (json.JSONDecodeError, TypeError):
                logger.warning("Invalid model JSON: model=%s chars=%d", chosen_model, len(content))
                return {"error": "The AI service returned an unreadable answer. Please try again."}
            if not isinstance(parsed, dict) or parsed.get("error"):
                return {"error": "The AI service returned an unexpected answer. Please try again."}
            return parsed
        except RateLimitError as exc:
            # Inspect only the machine code, never expose/log provider response text.
            code = str(getattr(exc, "code", "") or "")
            if code in {"insufficient_quota", "billing_hard_limit_reached"}:
                raise HTTPException(503, "AI capacity is unavailable. Please contact the library.") from None
            if attempt == 2:
                raise HTTPException(429, "AI service is busy. Please try again shortly.", headers={"Retry-After": "30"}) from None
        except APIStatusError as exc:
            status = getattr(exc, "status_code", 500)
            if status < 500 or attempt == 2:
                logger.warning("AI upstream HTTP status=%s", status)
                raise HTTPException(503, "AI service is temporarily unavailable. Please try again.") from None
        except Exception as exc:
            logger.warning("AI request failed (%s)", type(exc).__name__)
            raise HTTPException(503, "AI service is temporarily unavailable. Please try again.") from None
        await asyncio.sleep(2 ** attempt)
    raise HTTPException(503, "AI service is temporarily unavailable.")


def _text_list(value: Any, limit: int = 10, width: int = 300) -> list[str]:
    """Validate model-authored list shape before downstream use."""
    if not isinstance(value, list):
        return []
    return [" ".join(v.split())[:width] for v in value if isinstance(v, str) and v.strip()][:limit]


# ── Request models ─────────────────────────────────────────────────────────
_T300 = Field("", max_length=300)
_T500 = Field("", max_length=500)


class ManuscriptInput(BaseModel):
    title:               str = Field(..., max_length=300)
    abstract:            str = Field(..., max_length=5000)
    keywords:            str = _T500
    discipline:          str = _T300
    article_type:        str = Field("Original research article", max_length=100)
    methods:             str = _T500
    audience:            str = _T300
    funder:              str = _T300
    institution:         str = _T300
    country:             str = _T300
    apc_budget:          str = _T300
    # v1.8 names. Access model and manuscript-sharing rights are intentionally
    # separate. The legacy fields remain accepted for older LibGuides copies.
    access_preference:   str = Field("Any journal", max_length=80)
    sharing_requirement: bool = False
    intended_repository: str = Field("Institutional repository", max_length=100)
    ranking_preference:  str = Field("No filter", max_length=50)
    preferred_journals:  str = _T500
    avoid_journals:      str = _T500
    oa_preference:       str = Field("", max_length=50)
    ranking_source:      str = Field("", max_length=50)
    speed_preference:    str = Field("", max_length=50)
    repository_target:   str = Field("", max_length=100)


class JournalRequest(BaseModel):
    manuscript: ManuscriptInput
    model:      str = Field("gpt-4o-mini", max_length=30)
    language:   str = Field("english", max_length=10)


class LicenseInput(BaseModel):
    # journal_name is no longer required: a DOI alone is a complete request,
    # and the endpoint raises 422 if neither is supplied.
    journal_name:         str = Field("", max_length=300)
    doi:                  str = Field("", max_length=400)   # DOI or article URL
    issn:                 str = Field("", max_length=20)
    publisher:            str = _T300
    manuscript_version:   str = Field("All versions", max_length=50)
    funder:               str = _T300
    intended_repository:  str = Field("Institutional repository", max_length=100)
    intended_licence:     str = Field("CC BY", max_length=50)
    notes:                str = _T500


class LicenseRequest(BaseModel):
    license_input: LicenseInput
    model:         str = Field("gpt-4o-mini", max_length=30)
    language:      str = Field("english", max_length=10)


class DatasetInput(BaseModel):
    title:                  str = Field(..., max_length=300)
    description:            str = Field(..., max_length=2000)
    discipline:             str = Field(..., max_length=300)
    data_types:             str = Field(..., max_length=300)
    file_formats:           str = _T300
    approx_size:            str = _T300
    sensitivity:            str = Field("Open", max_length=50)
    contains_personal_data: str = Field("No", max_length=50)
    licence_intent:         str = Field("CC BY", max_length=50)
    funder:                 str = _T300
    institution:            str = _T300
    country:                str = _T300
    needs_doi:              str = Field("Yes", max_length=10)
    needs_versioning:       str = Field("Yes", max_length=10)
    embargo_required:       Literal["Yes", "No", "Unsure"] = "No"
    embargo_months:         Optional[int] = Field(None, ge=0, le=120)
    preferred_repository:   str = _T300
    related_publication:    str = _T300
    notes:                  str = _T500


class RepoRequest(BaseModel):
    dataset:  DatasetInput
    model:    str = Field("gpt-4o-mini", max_length=30)
    language: str = Field("english", max_length=10)


class SubjectRequest(BaseModel):
    subject:              str = Field(..., min_length=1, max_length=200)
    ranking_preference:   str = Field("No filter", max_length=50)
    access_preference:    str = Field("Any journal", max_length=80)
    sharing_requirement:  bool = False
    intended_repository:  str = Field("Institutional repository", max_length=100)
    apc_budget:           str = _T300
    funder:               str = _T300
    preferred_journals:   str = _T500
    avoid_journals:       str = _T500
    model:                str = Field("gpt-4o-mini", max_length=30)
    language:             str = Field("english", max_length=10)


class JournalPolicyItem(BaseModel):
    key:       str = Field("", max_length=200)
    name:      str = Field(..., min_length=1, max_length=300)
    issn:      str = Field("", max_length=20)
    publisher: str = _T300


class JournalPolicyBatchRequest(BaseModel):
    journals:             List[JournalPolicyItem] = Field(..., min_length=1, max_length=5)
    manuscript_version:   str = Field("Accepted manuscript (AAM / postprint)", max_length=80)
    intended_repository:  str = Field("Institutional repository", max_length=100)
    funder:               str = _T300
    language:             str = Field("english", max_length=10)


class CoverLetterRequest(BaseModel):
    manuscript_title: str = Field(..., max_length=300)
    abstract:         str = Field(..., max_length=5000)
    journal_name:     str = Field(..., max_length=300)
    publisher:        str = Field("", max_length=300)
    article_type:     str = Field("Original research article", max_length=100)
    author_name:      str = Field("", max_length=200)
    discipline:       str = Field("", max_length=300)
    language:         str = Field("english", max_length=10)
    model:            str = Field("gpt-4o-mini", max_length=30)


class ExportRanking(BaseModel):
    quartile: Optional[str] = Field(None, max_length=10)


class ExportJournal(BaseModel):
    name: str = Field(..., min_length=1, max_length=300)
    publisher: str = Field("", max_length=500)
    issn: Optional[str] = Field("", max_length=50)
    quartile: Optional[str] = Field(None, max_length=10)
    ranking: Optional[ExportRanking] = None


class ExportRequest(BaseModel):
    journals: List[ExportJournal] = Field(..., min_length=1, max_length=30)
    format: Literal["bibtex", "ris"] = "bibtex"


class ChatMessage(BaseModel):
    role:    str = Field(..., pattern="^(user|assistant)$")
    content: str = Field(..., max_length=12000)

    @model_validator(mode="after")
    def validate_role_limit(self):
        if self.role == "user" and len(self.content) > 2000:
            raise ValueError("User messages must contain at most 2000 characters.")
        return self


class ChatRequest(BaseModel):
    messages: List[ChatMessage] = Field(..., max_length=20)
    language: str = Field("english", max_length=10)

    @model_validator(mode="after")
    def validate_history(self):
        if self.messages and self.messages[-1].role != "user":
            raise ValueError("The final chat message must be from the user.")
        return self


# ── Routes ─────────────────────────────────────────────────────────────────
def _journal_index_status() -> dict:
    """Whether journals.db opened, for /api/health. Never raises."""
    try:
        import journal_lookup
        ok = journal_lookup.index_available()
        if not ok:
            return {"loaded": False, "fallback": "scimago CSV"}
        meta = journal_lookup._get_index().meta()
        return {
            "loaded": True,
            "journals": int(meta.get("n_journals", 0)),
            "active": int(meta.get("n_active", 0)),
            "discontinued": int(meta.get("n_discontinued", 0)),
            "built": meta.get("built_at", "")[:10],
        }
    except Exception as e:                                    # noqa: BLE001
        return {"loaded": False, "error": "Journal index could not be opened"}


@app.get("/api/health")
async def health():
    """

    Service status plus data-source provenance.



    The `sources` block distinguishes local readiness, configuration and

    observed upstream status. It does not actively probe external services.

    """
    now = time.time()
    index_status = _journal_index_status()
    try:
        scimago_count = len(load_subject_index())
    except Exception:
        scimago_count = 0
    ready = bool(index_status.get("loaded")) and scimago_count > 0
    return {
        "status": "ok" if ready else "degraded",
        "ready": ready,
        "liveness": "ok",
        "service": "ResearchBee",
        "version": RELEASE_VERSION,
        # Restart detection for the uptime monitor. boot_ts is the stable
        # identity of this process; when it changes between two polls the
        # container was replaced, whatever the HTTP status said.
        "boot_ts": round(BOOT_TS, 3),
        "uptime_s": round(now - BOOT_TS, 1),
        "knowledge_base": knowledge_status(),
        "sources": {
            "oaworks": {
                "endpoint": oaworks.OA_WORKS_BASE + oaworks.PERMISSIONS_PATH,
                "auth": "none required",
                "ku_ror_set": bool(oaworks.KU_ROR),
            },
            "khazna": khazna_index.status,
            "repository_registry": {"entries": len(REGISTRY)},
            "re3data": re3data.status(),
            "open_policy_finder": openpolicyfinder.status(),
            "scimago": {"loaded": scimago_count > 0, "records": scimago_count, "metric_year": 2025},
            "openalex": openalex.status(),
            "openai": {"configured": bool(os.getenv("OPENAI_API_KEY")), "availability": "not_probed"},
            # False here means journals.db did not open and every lookup is
            # falling back to the CSV — the Space still works, but none of the
            # Scopus data is reaching users. Worth alerting on.
            "journal_index": index_status,
        },
    }


@app.post("/api/admin/reload-khazna")
async def reload_khazna(request: Request):
    """

    Force an immediate reload of the Khazna index.



    Called by the harvest workflow after a successful publish, so the Space

    picks up new data within seconds instead of waiting for the 23:30 refresh.

    Idempotent and harmless: it only re-reads a public dataset.

    """
    require_khazna_reload_auth(request)
    ip = get_client_ip(request)
    if not check_rate_limit(ip, "global"):
        raise HTTPException(429, "Too many requests.")
    ok = await asyncio.to_thread(khazna_index.reload)
    return {"reloaded": ok, "khazna": khazna_index.status}


# ── OpenAlex related works per journal ────────────────────────────────────
@app.get("/api/openalex-works")
async def openalex_works(request: Request, source_id: str, per_page: int = 5):
    """

    Fetch recent highly-cited works from an OpenAlex source (journal).

    source_id: OpenAlex source ID e.g. S137773608

    """
    ip = get_client_ip(request)
    if not check_rate_limit(ip, "global"):
        raise HTTPException(429, "Too many requests. Please wait a moment.")

    clean_id = str(source_id or "").strip().rsplit("/", 1)[-1]
    if clean_id.startswith("S"):
        clean_id = clean_id[1:]
    if not re.fullmatch(r"[0-9]{1,20}", clean_id):
        raise HTTPException(400, "Invalid OpenAlex source_id")

    try:
        client = openalex._get_client()
        r = await client.get(
            "https://api.openalex.org/works",
            params={
                "filter":   f"primary_location.source.id:S{clean_id},is_retracted:false",
                "sort":     "cited_by_count:desc",
                "per_page": min(max(per_page, 1), 8),
                "select":   "id,title,publication_year,doi,cited_by_count,authorships",
            },
            timeout=6.0,
        )
        if not r.is_success:
            raise HTTPException(503, "OpenAlex is temporarily unavailable. Please try again.")

        works = []
        for w in r.json().get("results", []):
            authors = [
                a.get("author", {}).get("display_name", "")
                for a in (w.get("authorships") or [])[:3]
            ]
            works.append({
                "title":        w.get("title", ""),
                "year":         w.get("publication_year"),
                "doi":          w.get("doi", ""),
                "cited_by":     w.get("cited_by_count", 0),
                "authors":      authors,
                "openalex_url": w.get("id", ""),
            })
        return {"works": works, "status": "available" if works else "no_results"}
    except HTTPException:
        raise
    except Exception as e:
        logger.warning("OpenAlex works unavailable (%s)", type(e).__name__)
        raise HTTPException(503, "OpenAlex is temporarily unavailable. Please try again.") from None


@app.get("/api/openalex-cited-by")
async def openalex_cited_by(request: Request, work_id: str, per_page: int = 5):
    """Return the most-cited papers that cite one validated OpenAlex work id."""
    ip = get_client_ip(request)
    if not check_rate_limit(ip, "global"):
        raise HTTPException(429, "Too many requests. Please wait a moment.")

    clean_id = str(work_id or "").strip().replace("https://openalex.org/", "")
    if not re.fullmatch(r"W\d+", clean_id):
        raise HTTPException(400, "Invalid OpenAlex work_id")

    try:
        client = openalex._get_client()
        response = await client.get(
            "https://api.openalex.org/works",
            params={
                "filter": f"cites:{clean_id},is_retracted:false",
                "sort": "cited_by_count:desc",
                "per_page": min(max(per_page, 1), 8),
                "select": "id,title,publication_year,doi,cited_by_count,authorships",
            },
            timeout=6.0,
        )
        if not response.is_success:
            raise HTTPException(503, "OpenAlex is temporarily unavailable. Please try again.")

        works = []
        for work in response.json().get("results", []):
            authors = [
                authorship.get("author", {}).get("display_name", "")
                for authorship in (work.get("authorships") or [])[:3]
            ]
            works.append({
                "title": work.get("title", ""),
                "year": work.get("publication_year"),
                "doi": work.get("doi", ""),
                "cited_by": work.get("cited_by_count", 0),
                "authors": [author for author in authors if author],
                "openalex_url": work.get("id", ""),
            })
        return {"works": works, "status": "available" if works else "no_results"}
    except HTTPException:
        raise
    except Exception as exc:
        logger.warning("OpenAlex cited-by unavailable (%s)", type(exc).__name__)
        raise HTTPException(503, "OpenAlex is temporarily unavailable. Please try again.") from None


# ── Trending papers by concept/topic ─────────────────────────────────────
@app.get("/api/trending-papers")
async def trending_papers(request: Request, concept: str = "", concept_id: str = "", per_page: int = 5):
    """

    Fetch most-cited papers in last 3 years for a given concept/topic via OpenAlex.

    concept_id: OpenAlex concept ID (e.g. C41008148) — preferred

    concept:    display name fallback (e.g. "Machine Learning")

    """
    ip = get_client_ip(request)
    if not check_rate_limit(ip, "global"):
        raise HTTPException(429, "Too many requests.")

    if not concept and not concept_id:
        return {"works": [], "status": "no_results"}

    if len(concept) > 200 or (concept_id and not re.fullmatch(r"C?[0-9]{1,20}", concept_id)):
        raise HTTPException(400, "Invalid subject identifier")
    try:
        from datetime import datetime
        year_cutoff = datetime.now().year - 3
        client = openalex._get_client()

        # Build filter — prefer concept_id for accuracy
        if concept_id:
            clean_id = concept_id.strip().lstrip("C")
            filter_str = f"concepts.id:C{clean_id},publication_year:>{year_cutoff},is_retracted:false"
        else:
            # Use keyword search via default_search
            filter_str = f"publication_year:>{year_cutoff},is_retracted:false"

        params = {
            "filter":   filter_str,
            "sort":     "cited_by_count:desc",
            "per_page": min(max(per_page, 1), 8),
            "select":   "id,title,publication_year,doi,cited_by_count,authorships,primary_location",
        }

        # If only concept name, use search param instead of filter
        if not concept_id and concept:
            params["search"] = concept
            params["filter"] = f"publication_year:>{year_cutoff},is_retracted:false"

        r = await client.get(
            "https://api.openalex.org/works",
            params=params,
            timeout=8.0,
        )
        if not r.is_success:
            raise HTTPException(503, "OpenAlex is temporarily unavailable. Please try again.")

        works = []
        for w in r.json().get("results", []):
            authors = [
                a.get("author", {}).get("display_name", "")
                for a in (w.get("authorships") or [])[:3]
            ]
            journal = (w.get("primary_location") or {}).get("source", {}) or {}
            works.append({
                "title":        w.get("title", ""),
                "year":         w.get("publication_year"),
                "doi":          w.get("doi", ""),
                "cited_by":     w.get("cited_by_count", 0),
                "authors":      [a for a in authors if a],
                "journal":      journal.get("display_name", ""),
                "openalex_url": w.get("id", ""),
            })
        return {"works": works, "status": "available" if works else "no_results"}
    except HTTPException:
        raise
    except Exception as e:
        logger.warning("OpenAlex recent papers unavailable (%s)", type(e).__name__)
        raise HTTPException(503, "OpenAlex is temporarily unavailable. Please try again.") from None


# ── Altmetric journal attention ───────────────────────────────────────────
@app.get("/api/altmetric")
async def altmetric_score(request: Request, issn: str):
    """Legacy endpoint: no verified journal-level aggregate is available."""
    return {"score": None, "url": None, "issn": norm_issn(issn), "status": "not_supported",
            "message": "Article-level attention cannot be reported as a journal-level score."}


# ── Unified verified journal discovery (v1.8) ─────────────────────────────
_JOURNAL_PROFILE_VOCAB_BLOCK: Optional[str] = None


def journal_profile_prompt_with_vocabulary() -> str:
    global _JOURNAL_PROFILE_VOCAB_BLOCK
    if _JOURNAL_PROFILE_VOCAB_BLOCK is None:
        vocabulary = subject_vocabulary()
        categories = sorted(vocabulary["categories"])
        areas = sorted(vocabulary["areas"])
        _JOURNAL_PROFILE_VOCAB_BLOCK = (
            "\n\nVALID AREAS (" + str(len(areas)) + "):\n"
            + "\n".join("- " + value.title() for value in areas)
            + "\n\nVALID CATEGORIES (" + str(len(categories)) + "):\n"
            + "\n".join("- " + value.title() for value in categories)
        )
    return JOURNAL_PROFILE_PROMPT + _JOURNAL_PROFILE_VOCAB_BLOCK


def _legacy_access_and_sharing(manuscript: ManuscriptInput) -> tuple[str, bool, str]:
    access = " ".join((manuscript.access_preference or "").split())
    sharing = bool(manuscript.sharing_requirement)
    repository = " ".join((manuscript.intended_repository or "").split())
    legacy = " ".join((manuscript.oa_preference or "").split()).lower()
    if not access or access.lower() == "any journal":
        if "diamond" in legacy or "no apc" in legacy:
            access = "Diamond / no APC"
        elif "gold" in legacy:
            access = "Open-access journals"
        else:
            access = "Any journal"
    if "green" in legacy:
        sharing = True
    if (not repository or repository == "Institutional repository") and manuscript.repository_target:
        repository = manuscript.repository_target
    return access, sharing, repository or "Institutional repository"


def _preferences_from_manuscript(manuscript: ManuscriptInput) -> DiscoveryPreferences:
    access, sharing, repository = _legacy_access_and_sharing(manuscript)
    return DiscoveryPreferences.from_values(
        ranking_preference=manuscript.ranking_preference,
        access_preference=access,
        sharing_requirement=sharing,
        intended_repository=repository,
        apc_budget=manuscript.apc_budget,
        funder=manuscript.funder,
        preferred_journals=manuscript.preferred_journals,
        avoid_journals=manuscript.avoid_journals,
    )


def _preferences_from_subject(req: SubjectRequest) -> DiscoveryPreferences:
    return DiscoveryPreferences.from_values(
        ranking_preference=req.ranking_preference,
        access_preference=req.access_preference,
        sharing_requirement=req.sharing_requirement,
        intended_repository=req.intended_repository,
        apc_budget=req.apc_budget,
        funder=req.funder,
        preferred_journals=req.preferred_journals,
        avoid_journals=req.avoid_journals,
    )


def _fallback_manuscript_profile(manuscript: ManuscriptInput, note: str = "") -> dict:
    keywords = [k.strip() for k in re.split(r"[,;]", manuscript.keywords or "") if k.strip()]
    return {
        "manuscript_understanding": {
            "summary": manuscript.title,
            "discipline": manuscript.discipline,
            "article_type": manuscript.article_type,
            "methods": manuscript.methods,
            "audience": manuscript.audience,
            "keywords": keywords[:10],
            "inferred_criteria": [],
            "assumptions": [
                "The manuscript profile could not be fully inferred; supplied fields were used directly."
            ],
        },
        "primary_category": manuscript.discipline if is_real_category(manuscript.discipline) else "",
        "additional_categories": [],
        "area": manuscript.discipline if is_real_area(manuscript.discipline) else "",
        "confidence": "near" if manuscript.discipline else "not_found",
        "classification_note": note or "Subject classification used the author-supplied fields only.",
    }


async def _profile_manuscript(manuscript: ManuscriptInput, language: str) -> dict:
    payload = manuscript.model_dump()
    # Legacy form fields must not steer the classifier as if they were evidence
    # about the manuscript's subject.
    for key in ("oa_preference", "ranking_source", "speed_preference", "repository_target"):
        payload.pop(key, None)
    try:
        parsed = await call_openai(
            get_client(),
            journal_profile_prompt_with_vocabulary(),
            f"{lang_instruction(language)}Classify this manuscript.\n\n{json.dumps(payload, ensure_ascii=False, indent=2)}",
            max_tokens=1400,
            model=MODEL_LIGHT,
        )
        if parsed.get("error"):
            return _fallback_manuscript_profile(manuscript, parsed["error"])
    except Exception as exc:
        logger.warning("[JournalProfile] using supplied fields (%s)", type(exc).__name__)
        return _fallback_manuscript_profile(manuscript, "AI classification was unavailable; supplied fields were used.")

    understanding = parsed.get("manuscript_understanding") or {}
    if not isinstance(understanding, dict):
        return _fallback_manuscript_profile(manuscript, "AI profile format was invalid; supplied fields were used.")
    understanding = {
        "summary": " ".join(str(understanding.get("summary") or manuscript.title).split())[:1200],
        "discipline": " ".join(str(understanding.get("discipline") or manuscript.discipline).split())[:300],
        "article_type": " ".join(str(understanding.get("article_type") or manuscript.article_type).split())[:100],
        "methods": " ".join(str(understanding.get("methods") or manuscript.methods).split())[:500],
        "audience": " ".join(str(understanding.get("audience") or manuscript.audience).split())[:300],
        "keywords": _text_list(understanding.get("keywords"), 10, 80),
        "inferred_criteria": _text_list(understanding.get("inferred_criteria"), 8, 300),
        "assumptions": _text_list(understanding.get("assumptions"), 8, 300),
    }
    primary = str(parsed.get("primary_category") or "").strip()
    additional = _text_list(parsed.get("additional_categories"), 3, 200)
    area = str(parsed.get("area") or "").strip()
    # Closed-list validation. Invalid names are discarded rather than fuzzily
    # corrected into a different field.
    valid_categories = subject_vocabulary()["categories"]
    valid_areas = subject_vocabulary()["areas"]
    primary = primary if primary.lower() in valid_categories else ""
    additional = [v for v in additional if v.lower() in valid_categories and v.lower() != primary.lower()][:2]
    area = area if area.lower() in valid_areas else ""
    return {
        "manuscript_understanding": understanding,
        "primary_category": primary,
        "additional_categories": additional,
        "area": area,
        "confidence": parsed.get("confidence") if isinstance(parsed.get("confidence"), str) and parsed.get("confidence") in {"exact", "near", "not_found"} else "near",
        "classification_note": " ".join(str(parsed.get("classification_note") or "").split())[:500],
    }


def _candidate_query_text(manuscript: ManuscriptInput, profile: dict) -> str:
    understanding = profile.get("manuscript_understanding") or {}
    parts = [
        manuscript.title,
        manuscript.abstract,
        manuscript.keywords,
        manuscript.discipline,
        understanding.get("discipline", ""),
        " ".join(understanding.get("keywords") or []),
        understanding.get("methods", ""),
    ]
    return " ".join(str(part or "") for part in parts)


def _subject_query_text(subject: str, normalised: str, category: str, area: str) -> str:
    return " ".join(value for value in (subject, normalised, category, area) if value)


def _candidate_base_dict(rec_obj, asjc_codes: Sequence[str], score_ratio: float) -> dict:
    record = rec_obj.journal
    matched_q, matched_subject_rows = quartile_for(record, asjc_codes)
    matched_subjects = [
        {
            "code": row.get("asjc", ""),
            "field": row.get("name", ""),
            "parent": row.get("parent", ""),
            "quartile": row.get("quartile"),
        }
        for row in matched_subject_rows
    ]
    all_fields = [
        {"code": row.get("asjc", ""), "field": row.get("name", ""), "quartile": row.get("quartile")}
        for row in record.subjects if row.get("quartile")
    ]
    issn = record.issn_e or record.issn_print or ""
    publisher = record.publisher_group or record.publisher or ""
    access = access_model(record)
    agreement_publisher = is_ku_agreement_publisher(publisher)
    ku_candidate = bool(agreement_publisher and matched_q == "Q1")
    return {
        "id": record.scopus_id,
        "scopus_id": record.scopus_id,
        "name": record.title,
        "issn": issn,
        "publisher": publisher,
        "publisher_raw": record.publisher or "",
        "identity_verification": {
            "status": "Confirmed",
            "method": "Canonical Scopus source record",
            "scopus_id": record.scopus_id,
            "issn_print": record.issn_print or "",
            "issn_electronic": record.issn_e or "",
        },
        "ranking": {
            "source": "Scopus source list + SCImago 2025",
            "year": "2025",
            "quartile": matched_q,
            "category": "; ".join(row["field"] for row in matched_subjects if row.get("field"))[:500],
            "h_index": record.h_index,
            "sjr": record.sjr,
            "rank": record.sjr_rank,
            "verification_status": "Confirmed",
            "verification_method": "Canonical Scopus source ID and bundled SCImago match",
            "interpretation": (
                f"{matched_q} in a matched subject field; compare within the relevant category."
                if matched_q else "Currently indexed, but a matched-field quartile was not available."
            ),
        },
        # Top-level mirrors keep the shared LibGuides renderers and exports
        # consistent across manuscript and subject discovery modes.
        "quartile": matched_q,
        "h_index": record.h_index,
        "sjr": record.sjr,
        "quartiles_by_field": all_fields,
        "matched_subjects": matched_subjects,
        "state": record.state,
        "state_message": record.state_message,
        "indexation_state": record.state,
        "indexation_message": record.state_message,
        "currently_indexed": record.currently_indexed,
        "warn": record.warn,
        "successor_title": record.successor_title,
        "from_scopus": True,
        "source_type": record.source_type,
        "coverage_start": record.coverage_start,
        "coverage_end": record.coverage_end,
        "coverage_has_gap": record.coverage_has_gap,
        "open_access": "Yes" if access["code"] in {"open", "diamond"} else "No/unknown",
        "oa_diamond": record.oa_diamond,
        "access_model": access,
        "ku_agreement_publisher": agreement_publisher,
        "ku_apc_candidate": ku_candidate,
        "ku_apc_status": (
            "Potential KU publisher-agreement candidate; exact title, author, article type, quota and top-percentile eligibility are not confirmed."
            if ku_candidate else
            "Publisher-level KU agreement signal only; title-level APC eligibility is not confirmed."
            if agreement_publisher else
            "No KU publisher-agreement signal found in the configured publisher list."
        ),
        # Backward-compatible flag, now explicitly a candidate signal only.
        "ku_apc_covered": ku_candidate,
        "candidate_method": rec_obj.method,
        "candidate_reasons": list(rec_obj.reasons),
        "base_score": round(float(rec_obj.score), 4),
        "base_score_ratio": round(float(score_ratio), 4),
        "similar_articles": [],
        "article_similarity_score": 0,
        "article_evidence_status": "not_checked",
        "sharing_policy": {
            "status": "not_checked",
            "requirement_status": "not_requested",
            "message": "Journal-level sharing policy has not been checked."
        },
        "verify_links": build_verify_links(record.title, issn, record.scimago_id or record.scopus_id),
    }


async def _attach_article_evidence(candidates: list[dict], query: str, limit: int = 12) -> None:
    if not candidates:
        return
    try:
        evidence = await openalex.evidence_for_journals(candidates, query, limit=limit, per_page=3)
    except Exception as exc:
        logger.warning("[JournalEvidence] batch unavailable (%s)", type(exc).__name__)
        return
    for journal in candidates:
        got = evidence.get(str(journal.get("id")))
        if not got:
            continue
        journal["similar_articles"] = got.get("works") or []
        journal["article_similarity_score"] = clamp_int(got.get("similarity_score"), 0, 100)
        journal["article_evidence_status"] = got.get("status") or "unavailable"
        journal["openalex"] = got.get("source")
        if journal.get("openalex") and journal["openalex"].get("display_name"):
            journal["identity_verification"]["openalex_title"] = journal["openalex"].get("display_name")


def _retain_preferred(journals: list[dict], preferred_titles: Sequence[str], limit: int = 15) -> list[dict]:
    selected = list(journals[:limit])
    wanted = {normalise_title(title) for title in preferred_titles}
    if not wanted:
        return selected
    selected_ids = {str(j.get("id")) for j in selected}
    for journal in journals:
        if normalise_title(journal.get("name", "")) not in wanted or str(journal.get("id")) in selected_ids:
            continue
        replace_at = next((i for i in range(len(selected)-1, -1, -1) if normalise_title(selected[i].get("name", "")) not in wanted), None)
        if len(selected) < limit:
            selected.append(journal)
        elif replace_at is not None:
            selected_ids.discard(str(selected[replace_at].get("id")))
            selected[replace_at] = journal
        else:
            break
        selected_ids.add(str(journal.get("id")))
    order = {str(j.get("id")): i for i, j in enumerate(journals)}
    return sorted(selected, key=lambda j: order.get(str(j.get("id")), len(journals)))


def _compact_candidate_for_rerank(journal: dict) -> dict:
    return {
        "id": journal.get("id"),
        "title": journal.get("name"),
        "publisher": journal.get("publisher"),
        "matched_subjects": journal.get("matched_subjects", [])[:6],
        "quartile": (journal.get("ranking") or {}).get("quartile"),
        "h_index": (journal.get("ranking") or {}).get("h_index"),
        "access_model": (journal.get("access_model") or {}).get("label"),
        "ku_agreement_candidate": journal.get("ku_apc_candidate", False),
        "base_score_ratio": journal.get("base_score_ratio", 0),
        "article_similarity_score": journal.get("article_similarity_score", 0),
        "recent_article_titles": [
            {"title": item.get("title"), "year": item.get("year")}
            for item in (journal.get("similar_articles") or [])[:3]
        ],
    }


async def _rerank_manuscript_candidates(

    manuscript: ManuscriptInput,

    profile: dict,

    candidates: list[dict],

    record_map: dict[str, Any],

    asjc_codes: Sequence[str],

    preferences: DiscoveryPreferences,

    language: str,

) -> tuple[list[dict], list[str], str]:
    if not candidates:
        return [], [], "No verified journal candidates met the applied filters."
    candidate_payload = [_compact_candidate_for_rerank(j) for j in _retain_preferred(candidates, preferences.preferred_journals, 35)]
    user_payload = {
        "manuscript": {
            "title": manuscript.title,
            "abstract": manuscript.abstract,
            "article_type": manuscript.article_type,
            "methods": manuscript.methods,
            "audience": manuscript.audience,
        },
        "profile": profile,
        "verified_candidates": candidate_payload,
    }
    parsed: dict = {}
    try:
        parsed = await call_openai(
            get_client(),
            JOURNAL_RERANK_PROMPT,
            f"{lang_instruction(language)}Rerank these verified candidates.\n\n{json.dumps(user_payload, ensure_ascii=False)}",
            max_tokens=7000,
            model=MODEL,
        )
    except Exception as exc:
        logger.warning("[JournalRerank] subject-based fallback (%s)", type(exc).__name__)
        parsed = {"selected": [], "next_actions": [], "global_notes": "AI reranking was unavailable. These are subject-based suggestions; methods and audience fit were not evaluated."}

    by_id = {str(j["id"]): j for j in candidates}
    selected_rows = []
    seen: set[str] = set()
    model_rows = parsed.get("selected") if isinstance(parsed.get("selected"), list) else []
    for item in model_rows:
        if not isinstance(item, dict):
            continue
        sid = str(item.get("id") or "")
        if sid in by_id and sid not in seen:
            seen.add(sid)
            selected_rows.append((by_id[sid], item))
        if len(selected_rows) >= 15:
            break
    # Fewer is better than padding, but always retain at least five verified
    # candidates when the deterministic engine found them.
    for journal in candidates:
        sid = str(journal["id"])
        if len(selected_rows) >= min(15, len(candidates)):
            break
        if sid not in seen:
            selected_rows.append((journal, {})); seen.add(sid)

    # Include eligible preferred records before scoring and final truncation.
    selected_by_id = {str(j["id"]): item for j, item in selected_rows}
    selected_order = [j for j, _ in selected_rows] + [j for j in candidates if str(j["id"]) not in selected_by_id]
    selected_rows = [(j, selected_by_id.get(str(j["id"]), {})) for j in _retain_preferred(selected_order, preferences.preferred_journals, 15)]
    output: list[dict] = []
    for journal, item in selected_rows:
        record = record_map[str(journal["id"])].journal
        topic = clamp_int(item.get("topic_scope_score"), 0, 40,
                          default=20 if journal.get("matched_subjects") else 0)
        # Recent article-title overlap is a modest, transparent supporting
        # signal. It can strengthen a scope score but never dominate it.
        evidence_bonus = min(5, round(clamp_int(journal.get("article_similarity_score"), 0, 100) / 20))
        topic = min(40, topic + evidence_bonus)
        article = clamp_int(item.get("article_type_methods_score"), 0, 20,
                            default=0)
        audience = clamp_int(item.get("audience_score"), 0, 15,
                             default=0)
        rank = ranking_component(record, asjc_codes, 15)
        practical = practical_component(
            record, preferences, 10, ku_agreement_candidate=journal.get("ku_apc_candidate", False)
        )
        journal["fit_components"] = [
            {"key": "topic_scope", "label": "Topic and scope evidence", "score": topic, "max": 40},
            {"key": "article_methods", "label": "Article type and methods", "score": article, "max": 20},
            {"key": "audience", "label": "Audience alignment", "score": audience, "max": 15},
            {"key": "ranking", "label": "Indexing and subject ranking", "score": rank, "max": 15},
            {"key": "practical", "label": "Practical preferences", "score": practical, "max": 10},
        ]
        journal["scoring_mode"] = "model_assisted" if item else "subject_fallback"
        journal["scoring_note"] = (
            "Heuristic comparison score; not an acceptance probability."
            if item else
            "Subject-based fallback: methods and audience fit were not evaluated. Zero for these components means not evaluated."
        )
        journal["fit_score"] = topic + article + audience + rank + practical
        journal["fit_reason"] = " ".join(str(item.get("fit_reason") or "").split()) or (
            "Selected from the verified subject-filtered journal index; review the matched fields and recent articles before submission."
        )
        journal["audience_match"] = " ".join(str(item.get("audience_match") or "").split()) or (
            "The journal's indexed subject fields overlap the inferred manuscript discipline."
        )
        journal["submission_strategy"] = " ".join(str(item.get("submission_strategy") or "").split()) or (
            "Review the current aims and scope, recent articles, author guidelines, fees, and article-type requirements."
        )
        journal["open_access_planning"] = " ".join(str(item.get("open_access_planning") or "").split()) or (
            "Use the journal access flag for initial planning, then check current fees and the Open Policy Finder sharing options."
        )
        journal["article_evidence_summary"] = " ".join(str(item.get("article_evidence_summary") or "").split()) or (
            "Recent article-title evidence was unavailable."
            if not journal.get("similar_articles") else
            "Recent OpenAlex article titles provide supporting, not conclusive, scope evidence."
        )
        output.append(journal)

    output.sort(key=lambda j: (-j.get("fit_score", 0), j.get("name", "")))
    next_actions = _text_list(parsed.get("next_actions"), 4, 500)
    global_notes = " ".join(str(parsed.get("global_notes") or "").split())[:800]
    return output, next_actions, global_notes


def _score_subject_candidates(

    candidates: list[dict],

    record_map: dict[str, Any],

    asjc_codes: Sequence[str],

    preferences: DiscoveryPreferences,

    *,

    exact_category: bool,

) -> list[dict]:
    output = []
    for journal in candidates:
        record = record_map[str(journal["id"])].journal
        ratio = float(journal.get("base_score_ratio") or 0)
        subject_score = round((42 if exact_category else 32) + (18 if exact_category else 20) * ratio)
        evidence_bonus = min(5, round(clamp_int(journal.get("article_similarity_score"), 0, 100) / 20))
        subject_score = min(60, subject_score + evidence_bonus)
        rank = ranking_component(record, asjc_codes, 25)
        practical = practical_component(
            record, preferences, 15, ku_agreement_candidate=journal.get("ku_apc_candidate", False)
        )
        journal["fit_components"] = [
            {"key": "subject", "label": "Subject relevance", "score": subject_score, "max": 60},
            {"key": "ranking", "label": "Indexing and subject ranking", "score": rank, "max": 25},
            {"key": "practical", "label": "Practical preferences", "score": practical, "max": 15},
        ]
        journal["fit_score"] = subject_score + rank + practical
        fields = [row.get("field") for row in journal.get("matched_subjects") or [] if row.get("field")]
        field_text = ", ".join(fields[:3]) or "the selected subject area"
        journal["fit_reason"] = f"Indexed in {field_text}; ranked using verified subject and journal metadata."
        journal["audience_match"] = f"Relevant to researchers and readers working in {field_text}."
        journal["submission_strategy"] = "Use this as a discovery lead; review current aims and scope, recent articles, article types, and author instructions."
        journal["open_access_planning"] = "The access-model flag is journal-level. Check current fees and run the Open Policy Finder policy check before planning manuscript sharing."
        journal["article_evidence_summary"] = (
            "Recent OpenAlex article titles support the subject match."
            if journal.get("similar_articles") else
            "Recent article-title evidence was unavailable; rely on the verified subject classification and inspect the journal site."
        )
        output.append(journal)
    output.sort(key=lambda j: (-j.get("fit_score", 0), j.get("name", "")))
    return output


async def _lookup_policy_for_journal(journal: dict, *,

    manuscript_version: str = "Accepted manuscript (AAM / postprint)",

    intended_repository: str = "Institutional repository", intended_licence: str = "", funder: str = "",

) -> dict:
    try:
        record = await openpolicyfinder.lookup_policy(
            journal.get("name", ""), journal.get("issn", ""), journal.get("publisher", "")
        )
        if record.get("lookup_status") == "ok":
            green = openpolicyfinder.to_green_oa(record, manuscript_version=manuscript_version,
                intended_repository=intended_repository, intended_licence=intended_licence, funder=funder)
            return {
                "status": "confirmed",
                "requirement_status": policy_requirement_state(green, manuscript_version=manuscript_version,
                    intended_repository=intended_repository, intended_licence=intended_licence, funder=funder),
                "policy_evaluation": evaluate_policy(green, manuscript_version=manuscript_version,
                    intended_repository=intended_repository, intended_licence=intended_licence, funder=funder),
                "green_oa": green,
                "record": {
                    "title": record.get("journal_title", ""),
                    "publisher": record.get("publisher", ""),
                    "record_url": record.get("record_url", ""),
                    "match_quality": record.get("match_quality", ""),
                    "retrieved_at": record.get("retrieved_at", ""),
                },
                "message": "Journal-level options retrieved from Jisc Open Policy Finder.",
            }
        return {
            "status": "unavailable",
            "requirement_status": "unknown",
            "lookup_status": record.get("lookup_status", "upstream_error"),
            "message": record.get("user_message") or "No matching Open Policy Finder record was retrieved.",
            "verify_url": openpolicyfinder.public_search_url(journal.get("name", ""), journal.get("issn", "")),
        }
    except Exception as exc:
        logger.warning("[JournalPolicy] lookup unavailable (%s)", type(exc).__name__)
        return {
            "status": "unavailable", "requirement_status": "unknown",
            "message": "The journal-level policy could not be retrieved.",
            "verify_url": openpolicyfinder.public_search_url(journal.get("name", ""), journal.get("issn", "")),
        }


async def _lookup_policies(journals: Sequence[dict], limit: int = 5, *,

    manuscript_version: str = "Accepted manuscript (AAM / postprint)",

    intended_repository: str = "Institutional repository", intended_licence: str = "", funder: str = "",

) -> dict[str, dict]:
    semaphore = asyncio.Semaphore(4)

    async def one(journal: dict) -> tuple[str, dict]:
        async with semaphore:
            return str(journal.get("id") or journal.get("issn") or journal.get("name")), await _lookup_policy_for_journal(journal, manuscript_version=manuscript_version,
                intended_repository=intended_repository, intended_licence=intended_licence, funder=funder)

    selected = list(journals)[: max(0, limit)]
    pairs = await asyncio.gather(*(one(j) for j in selected)) if selected else []
    return dict(pairs)


async def _apply_sharing_requirement(

    journals: list[dict],

    preferences: DiscoveryPreferences,

) -> tuple[list[dict], list[str]]:
    if not preferences.sharing_requirement or not journals:
        for journal in journals:
            journal["sharing_policy"]["requirement_status"] = "not_requested"
        return journals, []

    policies = await _lookup_policies(journals, limit=min(15, len(journals)),
        intended_repository=preferences.intended_repository, funder=preferences.funder)
    for journal in journals:
        key = str(journal.get("id"))
        if key in policies:
            journal["sharing_policy"] = policies[key]
        else:
            journal["sharing_policy"]["requirement_status"] = "unknown"

    acceptable = [j for j in journals if (j.get("sharing_policy") or {}).get("requirement_status") in {"meets", "conditional"}]
    unknown = [j for j in journals if (j.get("sharing_policy") or {}).get("requirement_status") == "unknown"]
    denied = [j for j in journals if (j.get("sharing_policy") or {}).get("requirement_status") == "does_not_meet"]
    notes = []
    if denied:
        notes.append(f"Removed {len(denied)} candidate(s) whose retrieved accepted-manuscript option did not meet the sharing requirement.")
    if unknown:
        notes.append(f"{len(unknown)} candidate(s) could not be confirmed and are retained after confirmed/conditional matches; missing data is not treated as permission or prohibition.")
    if not acceptable:
        notes.append("No candidate had a clearly applicable accepted-manuscript option; results are shown for manual review rather than claiming the requirement was satisfied.")
    ordered = acceptable + unknown
    return ordered, notes


def _finalise_discovery_result(

    *,

    mode: str,

    ranked: list[dict],

    applied_criteria: list[str],

    unverified_criteria: list[str],

    legacy_notes: list[str],

    next_actions: list[str],

    global_notes: str,

    understanding: Optional[dict] = None,

    subject_understanding: Optional[dict] = None,

    classification_note: str = "",

    preferred_titles: Sequence[str] = (),

) -> dict:
    # Five evidence-rich cards plus up to ten additional verified leads.
    shortlist = _retain_preferred(ranked, preferred_titles, 15)
    detailed = shortlist[:5]
    extended = shortlist[5:15]
    actions = [a for a in next_actions if a]
    actions.extend([
        "Compare two to five shortlisted journals using the decision table.",
        "Check the current journal website for aims and scope, article types, fees and author instructions.",
        "Run the Open Policy Finder check for shortlisted journals; use a DOI-based article check after publication.",
    ])
    actions = list(dict.fromkeys(actions))[:6]
    note_parts = [value for value in (global_notes, classification_note) if value]
    return {
        "schema_version": "journal_discovery_v2",
        "mode": mode,
        "manuscript_understanding": understanding,
        "subject_understanding": subject_understanding,
        "applied_criteria": applied_criteria,
        "unverified_criteria": unverified_criteria + legacy_notes,
        "journals": detailed,
        "extended_list": extended,
        "total": len(detailed) + len(extended),
        "candidate_count_before_shortlist": len(ranked),
        "next_actions": actions,
        "global_notes": " ".join(note_parts) or (
            "Recommendations are decision support, not guarantees of acceptance or policy eligibility."
        ),
        "policy_source_mode": "on_demand_open_policy_finder",
        "comparison_enabled": True,
    }


@app.post("/api/analyze-journal")
@with_links
async def analyze_journal(req: JournalRequest, request: Request):
    ip = get_client_ip(request)
    if not check_rate_limit(ip, "global"):
        raise HTTPException(429, "Too many requests. Please wait a moment and try again.")

    index = get_index()
    if index is None:
        raise HTTPException(
            503,
            "The verified journal index is unavailable. ResearchBee will not generate unverified journal names as a fallback.",
        )

    manuscript = req.manuscript
    preferences = _preferences_from_manuscript(manuscript)
    profile = await _profile_manuscript(manuscript, req.language)
    asjc_rows = resolve_asjc_rows(
        index,
        category=profile.get("primary_category", ""),
        area=profile.get("area", ""),
        additional_categories=profile.get("additional_categories") or [],
    )
    query_text = _candidate_query_text(manuscript, profile)
    pool = build_candidate_pool(
        index,
        title=manuscript.title,
        abstract=query_text,
        asjc_rows=asjc_rows,
        preferences=preferences,
        limit=50,
        candidate_pool=600,
    )

    record_map = {rec.journal.scopus_id: rec for rec in pool.recommendations}
    asjc_codes = [str(row.get("code")) for row in asjc_rows if row.get("code")]
    candidates = [
        _candidate_base_dict(rec, asjc_codes, pool.score_ratios.get(rec.journal.scopus_id, 0.0))
        for rec in pool.recommendations
    ]
    evidence_query = " ".join(str(row.get("label") or row.get("name") or "") for row in asjc_rows)
    if not evidence_query:
        evidence_query = " ".join(v for v in [profile.get("primary_category", ""), profile.get("area", "")] if v)
    await _attach_article_evidence(candidates, evidence_query, limit=12)
    ranked, next_actions, global_notes = await _rerank_manuscript_candidates(
        manuscript, profile, candidates, record_map, asjc_codes, preferences, req.language
    )
    ranked, policy_notes = await _apply_sharing_requirement(ranked, preferences)

    unverified = list(pool.unverified_criteria)
    if policy_notes:
        # Replace the provisional note added by build_candidate_pool with the
        # actual outcome of the Open Policy Finder pass.
        unverified = [
            note for note in unverified
            if not note.startswith("Accepted-manuscript sharing will be checked")
        ]
        unverified.extend(policy_notes)
    if not asjc_rows:
        unverified.append(
            "No exact closed-vocabulary subject category was resolved, so candidate generation used verified full-index text matching and should be reviewed carefully."
        )

    result = _finalise_discovery_result(
        mode="manuscript",
        preferred_titles=preferences.preferred_journals,
        ranked=ranked,
        applied_criteria=pool.applied_criteria,
        unverified_criteria=unverified,
        legacy_notes=pool.legacy_notes,
        next_actions=next_actions,
        global_notes=global_notes,
        understanding=profile.get("manuscript_understanding"),
        classification_note=profile.get("classification_note", ""),
    )
    result["classification"] = {
        "confidence": profile.get("confidence"),
        "primary_category": profile.get("primary_category"),
        "additional_categories": profile.get("additional_categories") or [],
        "area": profile.get("area"),
        "asjc_codes": asjc_codes,
    }
    return {"result": result}


# ── Khazna deposit status ─────────────────────────────────────────────────
def khazna_status_for(doi: str) -> dict:
    """

    THREE-VALUED on purpose.



    "could not check" and "not in Khazna" are different answers. Conflating them

    would tell a researcher to deposit something already deposited, or imply an

    absence we never actually verified.

    """
    if not doi:
        return {"checked": False, "reason": "no DOI supplied"}
    source = khazna_index.status
    snapshot = {
        "checked_scope": "loaded_snapshot",
        "snapshot_loaded_at": source.get("loaded_at"),
        "snapshot_freshness": source.get("snapshot_freshness", "unknown"),
        "freshness_note": source.get("freshness_note", "Source harvest date is unknown; confirm current status in Khazna."),
        "retaining_previous_snapshot": source.get("retaining_previous_snapshot", False),
    }
    if not khazna_index.available:
        return {**snapshot, "checked": False, "reason": "Khazna index unavailable"}
    rec = khazna_index.get(doi)
    if rec is None:
        return {**snapshot, "checked": True, "in_khazna": False,
                "message": "This DOI was not found in the loaded Khazna snapshot. Confirm in the live portal before creating another record."}
    state = rec.get("deposit_state", "undetermined")
    return {
        **snapshot,
        "checked": True,
        "in_khazna": True,
        "deposit_state": state,
        "embargo_end": rec.get("embargo_end"),
        "portal_url": rec.get("portal_url", ""),
        "needs_deposit": state in ("metadata_only", "closed"),
        "message": {
            "open":          "The loaded Khazna record reports an open full text. Check the portal for its current status.",
            "embargoed":     "In the KU repository, Khazna, with an embargoed file.",
            "restricted":    "In the KU repository, Khazna, but the attached file is restricted.",
            "closed":        "In the KU repository, Khazna, as a closed record — an open version may be permitted.",
            # Khazna is currently a metadata-only CRIS: records exist for KU
            # output but no files are attached. This is therefore the normal
            # case, and the message is framed as an opportunity rather than a
            # problem with the author's record.
            "metadata_only": ("Your publication is already recorded in the KU repository, "
                              "Khazna, but no full text is attached. Adding the permitted "
                              "version makes it openly readable and discoverable."),
            "undetermined":  "In the KU repository, Khazna; access status could not be determined.",
        }.get(state, "In the KU repository, Khazna."),
    }


async def _advisory_note(green: dict, li: "LicenseInput", lang: str) -> str:
    """The LLM writes prose from verified facts only. It never supplies a value."""
    facts = {
        "permission_status":  green.get("permission_status", "unknown"),
        "embargo_status":     green.get("embargo_status", "unknown"),
        "can_archive":        green.get("_can_archive"),
        "versions_permitted": green.get("_versions"),
        "best_version":       green.get("_best_version"),
        "locations":          green.get("_locations"),
        "embargo_months":     green.get("_embargo_months"),
        "embargo_stated":     green.get("embargo_status") != "unknown",
        "embargo_end":        green.get("embargo_end"),
        "licence":            green.get("_licence"),
        "journal_oa_type":    green.get("journal_oa_type"),
        "institutional_repository_permitted": green.get("_institutional_ok"),
        "author_intent": {
            "repository": li.intended_repository,
            "licence":    li.intended_licence,
            "version":    li.manuscript_version,
            "funder":     li.funder,
        },
    }
    try:
        out = await call_openai(
            get_client(), ADVISORY_NOTE_PROMPT,
            lang + json.dumps(facts, ensure_ascii=False),
            max_tokens=280, model=MODEL_LIGHT,
        )
        return out.get("note", "") if isinstance(out, dict) else ""
    except Exception as e:
        logger.warning("Advisory note unavailable (%s)", type(e).__name__)
        return ""


@app.post("/api/check-journal-policies")
async def check_journal_policies(req: JournalPolicyBatchRequest, request: Request):
    """On-demand journal-level policy checks for either discovery mode.



    Up to five shortlist items are accepted per call. Every item is resolved

    again against ``journals.db`` before Jisc is queried, so browser-supplied

    titles, ISSNs, and publishers cannot replace the canonical journal identity.

    Facts come only from the authenticated Jisc Open Policy Finder API; a miss

    remains unknown.

    """
    ip = get_client_ip(request)
    if not check_rate_limit(ip, "global"):
        raise HTTPException(429, "Too many requests. Please wait a moment and try again.")

    index = get_index()
    if index is None:
        raise HTTPException(
            503,
            "The verified journal index is unavailable, so journal identity cannot be checked safely.",
        )

    prepared: list[dict] = []
    verified_journals: list[dict] = []
    for item in req.journals:
        response_key = item.key or item.issn or item.name
        record = None
        try:
            if item.key:
                record = index.lookup_exact(scopus_id=item.key)
            if record is None and item.issn:
                record = index.lookup_exact(issn=item.issn)
            if record is None and item.name:
                record = index.lookup_exact(title=item.name)
        except Exception as exc:
            logger.warning("[JournalPolicy] identity lookup unavailable (%s)", type(exc).__name__)

        if (
            record is None
            or not record.currently_indexed
            or record.source_type != "Journal"
        ):
            prepared.append({
                "key": response_key,
                "journal": None,
                "result": {
                    "key": response_key,
                    "name": item.name,
                    "issn": item.issn,
                    "publisher": item.publisher,
                    "identity_verification": {
                        "status": "Not confirmed",
                        "method": "No active canonical journal record matched the submitted selection",
                    },
                    "policy": {
                        "status": "unavailable",
                        "requirement_status": "unknown",
                        "message": "Policy was not queried because the journal identity could not be re-verified.",
                    },
                    "context": {
                        "manuscript_version": req.manuscript_version,
                        "intended_repository": req.intended_repository,
                        "funder": req.funder,
                    },
                },
            })
            continue

        canonical = {
            "id": response_key,
            "name": record.title,
            "issn": record.issn_e or record.issn_print or "",
            "publisher": record.publisher_group or record.publisher or "",
            "scopus_id": record.scopus_id,
        }
        prepared.append({"key": response_key, "journal": canonical, "result": None})
        verified_journals.append(canonical)

    policies = await _lookup_policies(verified_journals, limit=5, manuscript_version=req.manuscript_version,
        intended_repository=req.intended_repository, funder=req.funder)
    results = []
    for entry in prepared:
        if entry["result"] is not None:
            results.append(entry["result"])
            continue
        journal = entry["journal"]
        key = str(entry["key"])
        policy = policies.get(key) or {
            "status": "unavailable",
            "requirement_status": "unknown",
            "message": "The journal-level policy could not be retrieved.",
        }
        results.append({
            "key": key,
            "name": journal["name"],
            "issn": journal["issn"],
            "publisher": journal["publisher"],
            "identity_verification": {
                "status": "Confirmed",
                "method": "Canonical journals.db record re-resolved before policy lookup",
                "scopus_id": journal["scopus_id"],
            },
            "policy": policy,
            "context": {
                "manuscript_version": req.manuscript_version,
                "intended_repository": req.intended_repository,
                "funder": req.funder,
            },
        })
    return {"results": results, "source_mode": "journal_api"}


@app.post("/api/check-license")
@with_links
async def check_license(req: LicenseRequest, request: Request):
    ip = get_client_ip(request)
    if not check_rate_limit(ip, "global"):
        raise HTTPException(429, "Too many requests. Please wait a moment and try again.")

    li = req.license_input
    policy_context = {"manuscript_version": li.manuscript_version,
                      "intended_repository": li.intended_repository,
                      "intended_licence": li.intended_licence, "funder": li.funder}
    lang = lang_instruction(req.language)
    doi = doi_utils.extract_doi(li.doi) or doi_utils.extract_doi(li.notes)
    if li.doi.strip() and not doi:
        raise HTTPException(422, "Enter a valid DOI or a URL containing a DOI.")

    # ══ PATH A — DOI supplied: verified, article-level, no LLM policy facts ══
    if doi:
        permissions, khazna = await asyncio.gather(
            oaworks.fetch_permissions(doi, ror=oaworks.KU_ROR),
            asyncio.to_thread(khazna_status_for, doi),
        )
        green = oaworks.parse_permissions(permissions, **policy_context)

        if green is None:
            # The DOI resolved but OA.Works holds no policy for it, so the
            # researcher leaves without the verified answer they came for.
            usage.usage.record_gap("licence_deposit", "oaworks_404")
            request.state.usage_recorded = True
            green = oaworks.not_confirmed(
                li.journal_name, li.issn,
                reason="no permission record for this DOI in OA.Works")
        else:
            green["repository_action_note"] = await _advisory_note(green, li, lang)

        journal = {
            "name":      li.journal_name or green.get("copyright_name", ""),
            "issn":      li.issn or ((green.get("issuer_issns") or [""])[0]),
            "publisher": li.publisher or green.get("policy_issuer", ""),
            "green_oa":  green,
        }
        enriched = await enrich_journals([journal])

        # Subject repositories, but ONLY when OA.Works says one is permitted.
        # Suggesting arXiv for a journal that forbids subject-repository
        # deposit would be exactly the kind of confident-but-wrong advice this
        # build exists to prevent.
        subject_repos = []
        if any("subject repository" in loc.lower()
               for loc in green.get("_locations", [])):
            sci = lookup_any([enriched[0].get("issn"), li.issn]) or {}
            subject_repos = repositories.suggest_subject_repositories(
                sci.get("areas", ""), sci.get("categories", ""))

        result = {
            "journals": enriched,
            "subject_repositories": subject_repos,
            "repository_recommendation": oaworks.recommendation_from(green, **policy_context),
            "next_actions": oaworks.next_actions_from(green, doi, khazna, **policy_context),
            "global_notes": "",
            "doi": doi,
            "doi_url": doi_utils.doi_url(doi),
            "shareyourpaper_url": oaworks.share_link(doi),
            "khazna_status": khazna,
            "source_mode": "doi",
            "khazna": KHAZNA_DEPOSIT_CARD,
        }
        return {"result": result}

    # ══ PATH B — no DOI: authenticated Open Policy Finder journal record ══
    if not li.journal_name and not li.issn:
        raise HTTPException(422, "Provide an article DOI/URL, or a journal name or ISSN.")

    issn = doi_utils.normalise_issn(li.issn) or li.issn
    opf_record = await openpolicyfinder.lookup_policy(
        li.journal_name,
        issn,
        li.publisher,
    )
    lookup_status = opf_record.get("lookup_status", "upstream_error")

    if lookup_status == "ok":
        green = openpolicyfinder.to_green_oa(opf_record, **policy_context)
        source_mode = "journal_api"
        display_name = opf_record.get("journal_title") or li.journal_name
        display_issn = issn or ((opf_record.get("issns") or [""])[0])
        display_publisher = opf_record.get("publisher") or li.publisher
        next_actions = oaworks.next_actions_from(green, "", **policy_context)
        global_notes = (
            "This is a journal-level policy record from the authenticated Jisc "
            "Open Policy Finder API. It may contain several routes with different "
            "conditions. Use the DOI search above for an article-level permission "
            "check and a Khazna status check."
        )
    else:
        reason_by_status = {
            "unconfigured": "Open Policy Finder API access is not configured",
            "authentication_error": "Open Policy Finder rejected the configured API key",
            "rate_limited": "Open Policy Finder is temporarily rate limiting requests",
            "not_found": "no matching journal record was returned by Open Policy Finder",
            "identity_mismatch": "Open Policy Finder returned records that did not match the supplied journal identity",
            "invalid_response": "Open Policy Finder returned an unexpected response",
            "upstream_error": "Open Policy Finder is temporarily unavailable",
        }
        reason = reason_by_status.get(lookup_status, reason_by_status["upstream_error"])
        green = oaworks.not_confirmed(li.journal_name, issn, reason=reason)
        green["source_type"] = "open_policy_finder_api"
        green["policy_scope"] = "journal"
        green["repository_action_note"] = (
            "Open the linked Open Policy Finder search result and check the current "
            "journal record. No policy determination was made by ResearchBee."
        )
        green["_verify_url"] = opf_record.get("record_url") or openpolicyfinder.public_search_url(
            li.journal_name, issn
        )
        green["evidence_note"] = opf_record.get("message") or (
            "No authenticated journal-level policy record was available for this request."
        )
        source_mode = "journal"
        display_name = li.journal_name
        display_issn = issn
        display_publisher = li.publisher
        next_actions = oaworks.next_actions_from(green, "", **policy_context)
        global_notes = green["evidence_note"]

        gap_code = {
            "unconfigured": "opf_unconfigured",
            "authentication_error": "opf_auth_error",
            "rate_limited": "opf_rate_limited",
            "not_found": "opf_not_found",
            "identity_mismatch": "opf_identity_mismatch",
            "invalid_response": "opf_invalid_response",
            "upstream_error": "opf_upstream_error",
        }.get(lookup_status, "opf_upstream_error")
        usage.usage.record_gap("licence_deposit", gap_code)
        request.state.usage_recorded = True

    journal = {
        "name": display_name,
        "issn": display_issn,
        "publisher": display_publisher,
        "green_oa": green,
    }
    enriched = await enrich_journals([journal])
    if source_mode == "journal_api" and enriched:
        enriched[0].setdefault("verify_links", {})["sherpa_romeo"] = green.get(
            "_verify_url",
            openpolicyfinder.public_search_url(display_name, display_issn),
        )

    parsed = {
        "journals": enriched,
        "repository_recommendation": oaworks.recommendation_from(green, **policy_context),
        "next_actions": next_actions,
        "global_notes": global_notes,
        "khazna_status": {"checked": False, "reason": "no DOI supplied"},
        "source_mode": source_mode,
        "opf_lookup": {
            "status": lookup_status,
            "source": "Jisc Open Policy Finder API",
            "scope": "journal",
            "record_url": opf_record.get("record_url", ""),
            "record_id": opf_record.get("record_id", ""),
            "retrieved_at": opf_record.get("retrieved_at", ""),
        },
        "khazna": KHAZNA_DEPOSIT_CARD,
    }
    return {"result": parsed}


async def _re3data_supplement(dataset: "DatasetInput") -> dict:
    """Bounded re3data lookup that never raises into the route."""
    fallback = {
        "status": "unavailable", "query": re3data.build_query(dataset.discipline),
        "repositories": [], "source": "re3data.org", "source_url": "https://www.re3data.org",
        "message": "re3data.org did not respond in time. The curated KU Library recommendations are unaffected.",
    }
    try:
        return await asyncio.wait_for(
            re3data.find_supplementary(
                dataset.discipline,
                constraints=dataset.model_dump(),
                curated=REGISTRY.values(),
            ),
            timeout=float(os.getenv("RE3DATA_TIMEOUT_SECONDS", "12")),
        )
    except asyncio.TimeoutError:
        return fallback
    except Exception as exc:
        logger.warning("[re3data] supplement failed (%s)", type(exc).__name__)
        return fallback


@app.post("/api/find-repository")
async def find_repository(req: RepoRequest, request: Request):
    ip = get_client_ip(request)
    if not check_rate_limit(ip, "global"):
        raise HTTPException(429, "Too many requests. Please wait a moment and try again.")
    client = get_client()
    lang = lang_instruction(req.language)
    user_msg = (
        f"{lang}Select suitable repositories for this dataset FROM THE REGISTRY BELOW. "
        f"Return JSON only. Use the exact \"id\" values. Do not invent repositories "
        f"and do not state factual repository attributes.\n\n"
        f"AVAILABLE REGISTRY:\n{registry_for_prompt()}\n\n"
        f"DATASET INPUT:\n{json.dumps(req.dataset.model_dump(), indent=2)}"
    )
    # re3data runs alongside the model call, so it adds no latency in the
    # normal case. It only receives the discipline field and can never fail
    # the request; the curated registry flow below is unchanged.
    re3data_task = asyncio.create_task(_re3data_supplement(req.dataset))
    try:
        return await _curated_repository_result(req, request, client, user_msg, re3data_task)
    finally:
        # Every error path (model failure, invalid selection, cancellation)
        # must not leave the background lookup running.
        if not re3data_task.done():
            re3data_task.cancel()
            await asyncio.gather(re3data_task, return_exceptions=True)


async def _curated_repository_result(req, request, client, user_msg, re3data_task) -> dict:
    """The existing curated-registry flow, plus the awaited re3data list."""
    parsed = await call_openai(client, REPO_SYSTEM_PROMPT, user_msg, max_tokens=2000)
    if "error" in parsed:
        raise HTTPException(500, parsed["error"])

    # The model selected ids and wrote fit_reasons; the SERVER supplies every
    # factual field. Unknown ids are silently dropped.
    constraints = req.dataset.model_dump()
    if not isinstance(parsed.get("repositories", []), list):
        raise HTTPException(503, "The AI service returned an invalid repository selection. Please try again.")
    parsed["selection_exclusions"] = repositories.selection_exclusions(
        parsed.get("repositories", []), constraints=constraints
    )
    repos = hydrate_selection(parsed.get("repositories", []), constraints=constraints)

    # Khazna always leads, and is never a model choice.
    repos = [r for r in repos if not r.get("is_khazna")]
    if not repos:
        # Only Khazna will be offered — the registry had nothing for this
        # dataset. A recurring code here is a prompt to widen the registry.
        usage.usage.record_gap("data_repository", "no_repository_match")
        request.state.usage_recorded = True
    repos.insert(0, repositories.khazna_entry(constraints=constraints))
    parsed["deposit_strategy"] = repositories.hydrate_deposit_strategy(
        parsed.get("deposit_strategy"), repos, constraints=constraints
    )
    for key in ("dataset_understanding", "fair_assessment"):
        if key in parsed and not isinstance(parsed[key], dict):
            parsed[key] = {}
    for key in ("next_actions", "warnings"):
        if key in parsed:
            parsed[key] = _text_list(parsed[key], 8, 500)

    for r in repos:
        if not r.get("is_khazna"):
            clean_name = quote_plus(re.sub(r"[^\w\s]", "", r.get("name", "")).strip())
            r["verify_links"] = {
                "re3data":     f"https://www.re3data.org/search?query={clean_name}",
                "fairsharing": f"https://fairsharing.org/search?q={clean_name}",
            }

    parsed["repositories"] = repos
    supplement = await re3data_task
    curated_matches = sum(1 for r in repos if not r.get("is_khazna"))
    # "Necessary" = the curated registry produced fewer than two file-deposit
    # candidates. The frontend then shows the re3data list expanded.
    supplement["recommended_view"] = "expanded" if curated_matches < 2 else "collapsed"
    supplement["curated_candidate_count"] = curated_matches
    parsed["re3data_supplement"] = supplement
    parsed["selection_schema_version"] = "release-a-id-only-v1"
    parsed["registry_note"] = (
        "Repositories are selected from a registry curated by KU Library. "
        "Factual details are served from verified records, not generated."
    )
    return {"result": parsed}


# ── Subject vocabulary injection ───────────────────────────────────────────
#
# The LLM used to be asked for an "Exact SCImago category name" without being
# shown what the exact names are, so it generated plausible ones. "Crime and
# Society" reads like a SCImago category; SCImago has no category containing
# the word "crime". The invented name matched nothing, the search fell through
# to a substring test on the area, and a criminology query returned the
# Astrophysical Journal.
#
# Same pattern as the repository finder: the model chooses from a closed list
# and the server owns the facts. The validated vocabulary is cached locally.

_SUBJECT_VOCAB_BLOCK: Optional[str] = None


def subject_prompt_with_vocabulary() -> str:
    """SUBJECT_NORMALISE_PROMPT plus the real category and area names."""
    global _SUBJECT_VOCAB_BLOCK
    if _SUBJECT_VOCAB_BLOCK is None:
        v = subject_vocabulary()
        cats = sorted(v["categories"])
        areas = sorted(v["areas"])
        _SUBJECT_VOCAB_BLOCK = (
            "\n\nVALID AREAS (" + str(len(areas)) + "):\n"
            + "\n".join("- " + a.title() for a in areas)
            + "\n\nVALID CATEGORIES (" + str(len(cats)) + "):\n"
            + "\n".join("- " + c.title() for c in cats)
        )
    return SUBJECT_NORMALISE_PROMPT + _SUBJECT_VOCAB_BLOCK


@app.post("/api/browse-subject")
@with_links
async def browse_subject(req: SubjectRequest, request: Request):
    ip = get_client_ip(request)
    if not check_rate_limit(ip, "global"):
        raise HTTPException(429, "Too many requests. Please wait a moment and try again.")

    from scimago_index import _subject_cache, _scimago_cache
    if not _subject_cache or not _scimago_cache:
        load_scimago()
        load_subject_index()

    subject_text = req.subject.strip()
    typed_category = subject_text if is_real_category(subject_text) else ""
    typed_area = subject_text if is_real_area(subject_text) else ""
    spelling_match = None if (typed_category or typed_area) else closest_subject_term(subject_text)
    # Reviewed aliases and whole-word containment ("physics" -> the area
    # "Physics and Astronomy"). Deterministic, so it also works when the
    # LLM classifier is unavailable, and it never returns an unlisted term.
    nearest_match = (
        None if (typed_category or typed_area or spelling_match)
        else nearest_subject_term(subject_text)
    )
    llm_attempted = False

    # Exact vocabulary terms and conservative spelling repairs are resolved
    # locally. The LLM is reserved for synonym and semantic nearest-category
    # mapping, and may only return names from the closed vocabulary.
    if typed_category or typed_area:
        parsed = {
            "confidence": "exact",
            "normalised": subject_text,
            "scimago_category": typed_category,
            "scimago_area": typed_area,
            "nearest_categories": [],
            "mapping_type": "exact",
            "use_nearest_category": False,
            "user_message": "Matched directly to the verified ASJC/SCImago vocabulary.",
        }
    elif spelling_match:
        corrected = str(spelling_match["name"])
        is_category = spelling_match["kind"] == "category"
        parsed = {
            "confidence": "near",
            "normalised": corrected,
            "scimago_category": corrected if is_category else "",
            "scimago_area": corrected if not is_category else "",
            "nearest_categories": suggest_categories(corrected, limit=4),
            "mapping_type": "spelling_correction",
            "use_nearest_category": True,
            # The disclosure is built once below from the final selection;
            # pre-filling it here produced the correction notice twice.
            "user_message": "",
        }
    elif nearest_match:
        nearest_name = str(nearest_match["name"])
        nearest_is_category = nearest_match["kind"] == "category"
        parsed = {
            "confidence": "near",
            "normalised": nearest_name,
            "scimago_category": nearest_name if nearest_is_category else "",
            "scimago_area": nearest_name if not nearest_is_category else "",
            "nearest_categories": list(nearest_match.get("alternatives") or []),
            "mapping_type": "synonym" if nearest_match.get("match") == "synonym" else "nearest_category",
            "use_nearest_category": True,
            "user_message": "",
        }
    else:
        llm_attempted = True
        try:
            parsed = await call_openai(
                get_client(),
                subject_prompt_with_vocabulary(),
                f"{lang_instruction(req.language)}Subject input: {subject_text}",
                max_tokens=520,
                model=MODEL_LIGHT,
            )
        except (RateLimitError, APIStatusError, HTTPException) as exc:
            logger.warning("[BrowseSubject] classifier unavailable (%s)", type(exc).__name__)
            parsed = {
                "confidence": "not_found",
                "normalised": subject_text,
                "scimago_category": "",
                "scimago_area": "",
                "nearest_categories": [],
                "mapping_type": "none",
                "use_nearest_category": False,
                "user_message": "The subject classifier was unavailable. Choose a verified category suggestion or try again.",
            }
        except Exception as exc:
            logger.warning("[BrowseSubject] classifier failed (%s)", type(exc).__name__)
            parsed = {
                "confidence": "not_found",
                "normalised": subject_text,
                "scimago_category": "",
                "scimago_area": "",
                "nearest_categories": [],
                "mapping_type": "none",
                "use_nearest_category": False,
                "user_message": "The subject could not be classified safely. Choose a verified category suggestion or try again.",
            }

    confidence = parsed.get("confidence", "not_found")
    if not isinstance(confidence, str) or confidence not in {"exact", "near", "not_found"}:
        confidence = "not_found"
    mapping_type = str(parsed.get("mapping_type") or "").strip().lower()
    if mapping_type not in {"exact", "synonym", "spelling_correction", "nearest_category", "none"}:
        mapping_type = "nearest_category" if confidence == "near" else "none"
    use_nearest_category = parsed.get("use_nearest_category") is True
    normalised = str(parsed.get("normalised") or subject_text).strip()
    category = str(parsed.get("scimago_category") or "").strip()
    area = str(parsed.get("scimago_area") or "").strip()
    user_message = " ".join(str(parsed.get("user_message") or "").split())

    nearest_categories: list[str] = []
    nearest_seen: set[str] = set()
    for name in _text_list(parsed.get("nearest_categories"), 6, 200):
        value = str(name).strip()
        key = value.lower()
        if key and key not in nearest_seen and is_real_category(value):
            nearest_seen.add(key)
            nearest_categories.append(value)

    bad_category = bool(category) and not is_real_category(category)
    bad_area = bool(area) and not is_real_area(area)
    if bad_category:
        logger.warning("[BrowseSubject] discarded unlisted category %r", category)
        category = ""
    if bad_area:
        logger.warning("[BrowseSubject] discarded unlisted area %r", area)
        area = ""

    # Exact typed terms always win over a model's alternative valid term.
    if typed_category:
        category = typed_category
        area = ""
        confidence = "exact"
        mapping_type = "exact"
    if typed_area:
        area = typed_area
        category = ""
        confidence = "exact"
        mapping_type = "exact"

    # A model may conservatively return the chosen nearest category only in
    # nearest_categories. Use it only when the model explicitly says the
    # fallback is safe, or when it labelled the response "near". A true
    # not_found with use_nearest_category=false remains a no-result response.
    used_nearest_fallback = False
    if not category and not area and nearest_categories and (
        use_nearest_category or confidence == "near"
    ):
        category = nearest_categories[0]
        confidence = "near"
        mapping_type = "nearest_category"
        used_nearest_fallback = True
        if not normalised or normalised == subject_text:
            normalised = category

    # Last resort after the semantic classifier was unavailable or found no
    # mapping: a partial whole-word match on a distinctive term (for example
    # "quantum physics" -> "Physics and Astronomy"). Disclosed as nearest.
    if llm_attempted and not category and not area and not typed_category and not typed_area:
        fallback = nearest_subject_term(subject_text, allow_partial=True)
        if fallback:
            fallback_name = str(fallback["name"])
            if fallback["kind"] == "category":
                category = fallback_name
            else:
                area = fallback_name
            confidence = "near"
            mapping_type = "nearest_category"
            used_nearest_fallback = True
            normalised = fallback_name
            # The classifier's failure/no-match explanation no longer applies.
            user_message = ""
            for name in fallback.get("alternatives") or []:
                key = str(name).strip().lower()
                if key and key not in nearest_seen and is_real_category(key):
                    nearest_seen.add(key)
                    nearest_categories.append(key)

    selected_term = category or area
    if selected_term:
        display_term = selected_term.title()
        if mapping_type == "spelling_correction":
            disclosure = (
                f'Spelling corrected: "{subject_text}" → "{display_term}". '
                f'Results use this verified {"category" if category else "broad area"}.'
            )
        elif mapping_type == "nearest_category" or used_nearest_fallback:
            disclosure = (
                f'SCImago and Scopus have no exact subject category for "{subject_text}". '
                f'ResearchBee selected the nearest verified '
                f'{"category" if category else "subject area"} "{display_term}".'
            )
        elif mapping_type == "synonym" and selected_term.lower() != subject_text.lower():
            disclosure = (
                f'Interpreted "{subject_text}" as the verified '
                f'{"category" if category else "area"} "{display_term}".'
            )
        elif confidence == "near" and selected_term.lower() != subject_text.lower():
            disclosure = (
                f'Interpreted "{subject_text}" using the closest verified '
                f'{"category" if category else "area"} "{display_term}".'
            )
        else:
            disclosure = ""
        if disclosure and disclosure.lower() not in user_message.lower():
            user_message = f"{disclosure} {user_message}".strip()

    def suggestions() -> list[dict]:
        out: list[str] = []
        seen: set[str] = set()
        for name in nearest_categories:
            key = str(name).strip().lower()
            if key and key not in seen and is_real_category(key):
                seen.add(key)
                out.append(key)
        for name in suggest_categories(subject_text, limit=8):
            key = str(name).strip().lower()
            if key and key not in seen and is_real_category(key):
                seen.add(key)
                out.append(key)
        counts = subject_vocabulary()["categories"]
        return [{"category": name.title(), "n_journals": counts.get(name, 0)} for name in out[:6]]

    have_category = bool(category)
    # Permit a corrected or LLM-resolved broad area when no category is
    # available. Exact typed areas continue to take precedence.
    have_area = bool(area) and not have_category
    if confidence == "not_found" or (not have_category and not have_area):
        usage.usage.record_gap(
            "journal_subject",
            "category_invented" if bad_category else "subject_unrecognised",
        )
        request.state.usage_recorded = True
        nearby = suggestions()
        base = f"SCImago and Scopus have no exact subject category for '{subject_text}'."
        hint = (
            " Closest verified categories: " + ", ".join(item["category"] for item in nearby) + "."
            if nearby else ""
        )
        return {"result": {
            "schema_version": "journal_discovery_v2",
            "mode": "subject",
            "confidence": "not_found",
            "normalised": normalised,
            "user_message": (user_message + " " if user_message else "") + base + hint,
            "subject_mapping": {
                "original_input": subject_text,
                "normalised": normalised,
                "match_type": "none",
                "selected_category": "",
                "selected_area": "",
                "used_nearest_category": False,
                "spelling_corrected": False,
                "alternatives": nearby,
            },
            "suggestions": nearby,
            "journals": [],
            "extended_list": [],
            "total": 0,
        }}

    index = get_index()
    if index is None:
        raise HTTPException(
            503,
            "The verified journal index is unavailable. ResearchBee will not generate unverified journal names as a fallback.",
        )

    preferences = _preferences_from_subject(req)
    asjc_rows = resolve_asjc_rows(
        index,
        category=category if have_category else "",
        area=area if have_area else "",
    )
    query_text = _subject_query_text(req.subject, normalised, category, area)
    pool = build_candidate_pool(
        index,
        title=query_text,
        abstract="",
        asjc_rows=asjc_rows,
        preferences=preferences,
        limit=50,
        candidate_pool=600,
    )
    record_map = {rec.journal.scopus_id: rec for rec in pool.recommendations}
    asjc_codes = [str(row.get("code")) for row in asjc_rows if row.get("code")]
    candidates = [
        _candidate_base_dict(rec, asjc_codes, pool.score_ratios.get(rec.journal.scopus_id, 0.0))
        for rec in pool.recommendations
    ]
    await _attach_article_evidence(candidates, query_text, limit=12)
    ranked = _score_subject_candidates(
        candidates, record_map, asjc_codes, preferences, exact_category=have_category
    )
    ranked, policy_notes = await _apply_sharing_requirement(ranked, preferences)

    unverified = list(pool.unverified_criteria)
    if policy_notes:
        unverified = [
            note for note in unverified
            if not note.startswith("Accepted-manuscript sharing will be checked")
        ]
        unverified.extend(policy_notes)
    if have_area and not have_category:
        if area.title() in user_message:
            # The nearest/synonym disclosure already names the area.
            area_note = "Results span this broad area; choose a narrower category below for a more specific list."
        else:
            area_note = (
                f"'{subject_text}' was resolved to the broad subject area "
                f"'{area.title()}'. Results span that area; choose a narrower category for a more specific list."
            )
        user_message = f"{user_message} {area_note}".strip()
        usage.usage.record_gap("journal_subject", "area_fallback")
        request.state.usage_recorded = True

    all_suggestions = suggestions()
    alternatives = [
        item for item in all_suggestions
        if item["category"].lower() != str(category).lower()
    ][:4]
    result = _finalise_discovery_result(
        mode="subject",
        preferred_titles=preferences.preferred_journals,
        ranked=ranked,
        applied_criteria=pool.applied_criteria,
        unverified_criteria=unverified,
        legacy_notes=pool.legacy_notes,
        next_actions=[],
        global_notes=user_message,
        subject_understanding={
            "summary": f"Verified journal discovery for {normalised}.",
            "discipline": normalised,
            "category": category,
            "area": area,
            "confidence": confidence,
            "mapping_type": mapping_type,
            "original_input": subject_text,
            "alternatives": alternatives,
        },
        classification_note=(
            "This is a broad area result, not a manuscript-specific fit assessment."
            if have_area and not have_category else
            "Subject relevance is based on the verified ASJC/SCImago category mapping."
        ),
    )
    result.update({
        "confidence": confidence,
        "normalised": normalised,
        "scimago_category": category,
        "scimago_area": area,
        "user_message": user_message,
        "asjc_codes": asjc_codes,
        "subject_mapping": {
            "original_input": subject_text,
            "normalised": normalised,
            "match_type": mapping_type,
            "selected_category": category,
            "selected_area": area,
            "used_nearest_category": mapping_type == "nearest_category" or used_nearest_fallback,
            "spelling_corrected": mapping_type == "spelling_correction",
            "alternatives": alternatives,
        },
        "suggestions": alternatives if confidence == "near" else [],
    })
    if not ranked:
        result["filter_no_match"] = True
        result["global_notes"] = (
            "The subject was recognised, but no active journal met every selected hard filter. "
            "ResearchBee did not silently relax those filters."
        )
    return {"result": result}


@app.post("/api/export-citations")
async def export_citations(req: ExportRequest):
    fmt = req.format.lower()
    journals = [item.model_dump() for item in req.journals]

    if fmt == "bibtex":
        lines = []
        for i, j in enumerate(journals):
            key = f"journal{i + 1}"
            def bibtext(value):
                return str(value or "").replace("\\", " ").replace("{", "(").replace("}", ")").replace("\n", " ").replace("\r", " ")
            j = {name: bibtext(value) if name != "ranking" else value for name, value in j.items()}
            lines.append(
                f"@misc{{{key},\n"
                f"  title     = {{{j.get('name', '')}}},\n"
                f"  howpublished = {{Journal shortlist record, not an article citation}},\n"
                f"  publisher = {{{j.get('publisher', '')}}},\n"
                f"  issn      = {{{j.get('issn', '')}}},\n"
                f"  note      = {{Quartile: {j.get('quartile') or (j.get('ranking') or {}).get('quartile', '—')}. Recommended by ResearchBee.}}\n}}"
            )
        return {"content": "\n\n".join(lines), "filename": "journals.bib", "mimetype": "text/plain"}

    elif fmt == "ris":
        lines = []
        for j in journals:
            j = {key: " ".join(value.split()) if isinstance(value, str) else value for key, value in j.items()}
            q = j.get("quartile") or (j.get("ranking") or {}).get("quartile", "")
            lines.append(
                f"TY  - GEN\nTI  - {j.get('name', '')}\n"
                f"PB  - {j.get('publisher', '')}\nSN  - {j.get('issn', '')}\n"
                f"N1  - Journal shortlist record, not an article citation. Quartile: {q}. Recommended by ResearchBee.\nER  - "
            )
        return {"content": "\n\n".join(lines), "filename": "journals.ris", "mimetype": "text/plain"}

    raise HTTPException(400, "Format must be bibtex or ris")


@app.post("/api/cover-letter")
async def generate_cover_letter(req: CoverLetterRequest):
    client = get_client()
    lang = "Respond in Arabic." if req.language == "arabic" else "Respond in English."
    user_msg = (
        f"{lang}\n\nGenerate a cover letter for the following submission:\n\n"
        f"Manuscript title: {req.manuscript_title}\nAbstract: {req.abstract}\n"
        f"Article type: {req.article_type}\nTarget journal: {req.journal_name}\n"
        f"Publisher: {req.publisher or 'Not specified'}\nDiscipline: {req.discipline or 'Not specified'}\n"
        f"Author name: {req.author_name or 'The Author(s)'}\n\nReturn JSON only."
    )
    parsed = await call_openai(client, COVER_LETTER_PROMPT, user_msg, max_tokens=800, model=MODEL_LIGHT)
    if parsed.get("error"):
        raise HTTPException(503, parsed["error"])
    if not isinstance(parsed.get("cover_letter"), str) or not parsed["cover_letter"].strip():
        raise HTTPException(503, "The AI service returned an invalid cover-letter draft. Please try again.")
    parsed["cover_letter"] = parsed["cover_letter"][:12000]
    parsed["subject_line"] = str(parsed.get("subject_line") or "Manuscript submission")[:500]
    parsed["requires_author_review"] = True
    parsed["review_note"] = "Verify every submission declaration and replace the bracketed author-confirmation placeholders before use."
    return {"result": parsed}


# ── Chat endpoint — streaming SSE ─────────────────────────────────────────
@app.post("/api/chat")
async def chat(req: ChatRequest, request: Request):
    """

    Three-layer resolution with streaming:

    1. Keyword pre-filter — instant routing, no LLM call

    2. Open Policy Finder — live policy lookup for specific journals

    3. LLM stream — tokens stream via SSE; route extracted from full reply at end

    """
    ip = get_client_ip(request)
    if not check_rate_limit(ip, "chat"):
        raise HTTPException(429, "Chat limit reached. Please wait a minute before sending more messages.")

    if not req.messages:
        async def empty_stream():
            greeting = "Hi! How can I help you with publishing or Open Access today?"
            yield f'data: {json.dumps({"token": greeting, "done": False})}\n\n'
            yield f'data: {json.dumps({"token": "", "done": True, "reply": greeting, "route": None, "prefill": {}})}\n\n'
        return StreamingResponse(empty_stream(), media_type="text/event-stream",
                                 headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"})

    last_msg = req.messages[-1].content.strip()

    # ── Layer 1: keyword pre-filter — instant, no streaming needed ────────
    fast_route = quick_route(last_msg)
    if fast_route:
        TAB_DESCRIPTIONS = {
            "journals": "Journal Submission tool",
            "license":  "License Checking tool",
            "data":     "Data Repository tool",
        }
        reply = (
            f"The {TAB_DESCRIPTIONS[fast_route]} above is perfect for this! "
            f"It'll give you accurate, structured results in seconds."
        )
        if req.language == "arabic":
            reply = f"أداة {TAB_DESCRIPTIONS[fast_route]} المتاحة أعلاه مناسبة تماماً لهذا! ستمنحك نتائج دقيقة ومنظمة في ثوانٍ."
        # Return as SSE with route
        async def fast_stream():
            payload = json.dumps({"token": reply, "done": False})
            yield f"data: {payload}\n\n"
            done = json.dumps({"token": "", "done": True, "route": fast_route, "prefill": {}})
            yield f"data: {done}\n\n"
        return StreamingResponse(fast_stream(), media_type="text/event-stream",
                                 headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"})

    # ── Layer 2: Open Policy Finder ───────────────────────────────────────
    opf_data = ""
    journal_name = detect_journal_policy_question(last_msg)
    if journal_name:
        opf_data = await fetch_opf_policy(journal_name)

    # ── Layer 3: LLM streaming ────────────────────────────────────────────
    client = get_client()

    try:
        retrieved_knowledge = get_chat_knowledge(last_msg, context_query=" ".join(m.content for m in req.messages[-5:-1] if m.role == "user")[-2000:])
    except Exception as exc:  # noqa: BLE001 - chat should degrade safely
        logger.warning("[Chat] Knowledge retrieval unavailable (%s)", type(exc).__name__)
        retrieved_knowledge = (
            "The curated knowledge index is temporarily unavailable. Give only stable general "
            "guidance, do not invent current policies or links, and direct KU-specific questions "
            "to the KU Library."
        )

    system = CHAT_SYSTEM_PROMPT_BASE.replace(
        "{RETRIEVED_KNOWLEDGE}", retrieved_knowledge
    ).replace(
        "{OPF_DATA}", opf_data if opf_data else "No live Open Policy Finder data was retrieved for this query."
    )

    if req.language == "arabic":
        system += "\n\nRespond entirely in Arabic."

    messages = [{"role": "system", "content": system}]
    # Preserve the latest turn while bounding total model context independently
    # of the browser's history representation.
    history = []
    remaining = 24000
    for message in reversed(req.messages):
        text = message.content[-min(remaining, 12000):]
        if not text or remaining <= 0:
            break
        history.append({"role": message.role, "content": text})
        remaining -= len(text)
    messages += list(reversed(history))

    def extract_route(text: str):
        """Brace-counting extraction of routing JSON from end of reply."""
        starts = [m.start() for m in re.finditer(r'\{', text)]
        for start in reversed(starts):
            candidate = text[start:]
            depth = 0; end = None
            for i, ch in enumerate(candidate):
                if ch == '{': depth += 1
                elif ch == '}':
                    depth -= 1
                    if depth == 0: end = i + 1; break
            if end is None: continue
            try:
                parsed = json.loads(candidate[:end])
                if "route" in parsed:
                    return text[:start].strip(), parsed.get("route"), parsed.get("prefill", {})
            except Exception:
                continue
        return text.strip(), None, {}

    # Create streaming response outside generator to catch connection errors early
    try:
        openai_stream = await client.chat.completions.create(
            model=MODEL,
            messages=messages,
            max_tokens=600,
            temperature=0.3,
            stream=True,
        )
    except Exception as e:
        logger.warning("[Chat] AI request failed (%s)", type(e).__name__)
        raise HTTPException(503, "AI service is temporarily unavailable. Please try again.") from None

    request.state.usage_recorded = True
    async def stream_response() -> AsyncGenerator[str, None]:
        full_text = ""
        try:
            async for chunk in openai_stream:
                token = chunk.choices[0].delta.content if chunk.choices else None
                if token:
                    full_text += token
                    yield f'data: {json.dumps({"token": token, "done": False})}\n\n'

            reply, route, prefill = extract_route(full_text)
            if not isinstance(route, (str, type(None))) or route not in ("journals", "license", "data", None):
                route = None

            done_payload = json.dumps({
                "token": "", "done": True,
                "reply": reply,
                "route": route,
                "prefill": prefill if isinstance(prefill, dict) else {},
            })
            usage.usage.record("chat", "ok")
            yield f"data: {done_payload}\n\n"

        except asyncio.CancelledError:
            usage.usage.record("chat", "miss", "cancelled")
            raise
        except Exception as e:
            usage.usage.record("chat", "error", "stream_error")
            logger.warning("[Chat] Stream failed (%s)", type(e).__name__)
            err = json.dumps({"token": "", "error": "The answer was interrupted. Please try again.", "done": True, "route": None, "prefill": {}})
            yield f"data: {err}\n\n"
        finally:
            await openai_stream.close()

    return StreamingResponse(
        stream_response(),
        media_type="text/event-stream",
        headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
    )


# ── Journal meta lookup — multi-suggestion with ISSN auto-fill ───────────
@app.get("/api/lookup-journal-meta")
async def lookup_journal_meta(request: Request, name: str):
    """

    Multi-suggestion journal lookup:

    Layer 1a: SCImago — up to 4 ranked title matches (name + ISSN + publisher)

    Layer 1b: OpenAlex — up to 4 results, merged + deduplicated with SCImago

    Layer 2:  LLM — only if nothing found; returns up to 3 name suggestions only

              (NO ISSN from LLM — those come from re-running Layer 1 on acceptance)

    """
    ip = get_client_ip(request)
    if not check_rate_limit(ip, "global"):
        raise HTTPException(429, "Too many requests.")

    name = name.strip()
    if len(name) > 300:
        raise HTTPException(422, "Journal name must contain at most 300 characters.")
    if not name or len(name) < 3:
        return {"suggestions": [], "llm_suggestions": []}

    name_lower = name.lower()
    words      = [w for w in name_lower.split() if len(w) > 2]

    def score_title(title: str) -> float:
        """Score a title against the query — higher = better match."""
        t = title.lower()
        if t == name_lower:
            return 1.0
        # Word overlap score
        overlap = sum(1 for w in words if w in t)
        # Bonus for starts-with
        starts  = 0.2 if t.startswith(name_lower[:6]) else 0
        # Penalty for very different lengths
        len_ratio = min(len(name_lower), len(t)) / max(len(name_lower), len(t))
        return (overlap / max(len(words), 1)) * 0.6 + starts + len_ratio * 0.2

    # ── Layer 1a: SCImago — collect top 4 scored matches ─────────────────
    rows = load_subject_index()
    scored = []
    for r in rows:
        s = score_title(r["title"])
        if s > 0.35:   # threshold — avoid noise
            scored.append((s, r))
    scored.sort(key=lambda x: -x[0])
    top_scimago = scored[:4]

    suggestions = []
    seen_titles = set()

    for score, r in top_scimago:
        t = r["title"]
        if t.lower() in seen_titles:
            continue
        seen_titles.add(t.lower())
        ni = norm_issn(r.get("issn", ""))
        suggestions.append({
            "title":      t,
            "issn":       ni or r.get("issn", ""),
            "publisher":  r.get("publisher", ""),
            "source":     "SCImago",
            "confidence": "high" if score > 0.8 else "medium",
            "score":      round(score, 3),
        })

    # ── Layer 1b: OpenAlex — up to 4, merge with SCImago results ─────────
    try:
        oa_client = openalex._get_client()
        r = await oa_client.get(
            "https://api.openalex.org/sources",
            params={"search": name, "per_page": 4, "filter": "type:journal"},
            timeout=6.0,
        )
        if r.is_success:
            for s in r.json().get("results", []):
                title     = s.get("display_name", "")
                if not title or title.lower() in seen_titles:
                    continue
                issns     = s.get("issn", [])
                issn_l    = s.get("issn_l", "")
                publisher = s.get("host_organization_name", "")
                best_issn = issn_l or (issns[0] if issns else "")
                ni        = norm_issn(best_issn)
                sc        = score_title(title)
                seen_titles.add(title.lower())
                suggestions.append({
                    "title":      title,
                    "issn":       ni or best_issn,
                    "publisher":  publisher,
                    "source":     "OpenAlex",
                    "confidence": "high" if sc > 0.8 else "medium",
                    "score":      round(sc, 3),
                })
    except Exception as e:
        logger.warning("[LookupJournal] OpenAlex unavailable (%s)", type(e).__name__)

    # Re-sort merged list by score, keep top 4
    suggestions.sort(key=lambda x: -x.get("score", 0))
    suggestions = suggestions[:4]

    # If we have good suggestions, return them
    if suggestions:
        return {"suggestions": suggestions, "llm_suggestions": []}

    # ── Layer 2: LLM — name suggestions only, NO ISSN ────────────────────
    llm_suggestions = []
    try:
        client = get_client()
        resp = await client.chat.completions.create(
            model=MODEL,
            response_format={"type": "json_object"},
            messages=[
                {
                    "role": "system",
                    "content": (
                        "You are an academic journal name expert. "
                        "The user has typed a journal name that may be misspelled or abbreviated. "
                        "Return ONLY valid JSON: "
                        "{suggestions: [{name: string, confidence: high|medium|low}]} "
                        "Return up to 3 most likely correct journal names, ranked by confidence. "
                        "Only include journals you are confident exist. "
                        "Do NOT return ISSN, publisher, or any other fields. "
                        "Return JSON only."
                    ),
                },
                {"role": "user", "content": f"Journal name typed by user: {name}"},
            ],
            max_tokens=150,
            temperature=0.1,
        )
        raw    = resp.choices[0].message.content or "{}"
        parsed = json.loads(raw)
        for s in parsed.get("suggestions", [])[:3]:
            n = s.get("name", "").strip()
            c = s.get("confidence", "low")
            if n and n.lower() != name_lower and c in ("high", "medium"):
                llm_suggestions.append({"name": n, "confidence": c})
    except Exception as e:
        logger.warning("[LookupJournal] AI lookup unavailable (%s)", type(e).__name__)

    return {"suggestions": [], "llm_suggestions": llm_suggestions}


# ── Static frontend ────────────────────────────────────────────────────────
@app.get("/")
async def root():
    if os.path.exists("static/index.html"):
        return FileResponse("static/index.html")
    return {"status": "ResearchBee API running", "docs": "/docs"}


if os.path.exists("static"):
    if os.path.exists("static/css"):
        app.mount("/css", StaticFiles(directory="static/css"), name="css")
    if os.path.exists("static/js"):
        app.mount("/js", StaticFiles(directory="static/js"), name="js")


class JournalLinksRequest(BaseModel):
    scopus_id: str = Field("", max_length=30)
    issn: str = Field("", max_length=20)

@app.post("/api/journal-links")
async def find_author_guidelines(req: JournalLinksRequest, request: Request):
    if not check_rate_limit(get_client_ip(request), "global"):
        raise HTTPException(429, "Too many requests. Please try again later.")
    return await discover(req.model_dump())