Upload 31 files

.env.example

@@ -0,0 +1,17 @@
+# Required
+OPENAI_API_KEY=
+KHAZNA_RELOAD_TOKEN=
+
+# Optional Khazna dataset configuration
+KHAZNA_DATASET=nikeshn/researchbee-data
+HF_TOKEN=
+
+# Optional admin / analytics configuration already supported by the project
+RESEARCHBEE_ADMIN_PASSWORD=
+CLOUDFLARE_WORKER_URL=
+CLOUDFLARE_WORKER_TOKEN=
+USAGE_PUSH_ENABLED=false
+
+# Optional model configuration
+ASJC_LLM_MODEL=gpt-4.1-nano
+ANTHROPIC_API_KEY=

.gitattributes

@@ -1,37 +1,37 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
-scimagojr_2025.csv filter=lfs diff=lfs merge=lfs -text
-journals.db filter=lfs diff=lfs merge=lfs -text
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+scimagojr_2025.csv filter=lfs diff=lfs merge=lfs -text
+journals.db filter=lfs diff=lfs merge=lfs -text

Dockerfile

@@ -1,7 +1,7 @@
-FROM python:3.11-slim
-WORKDIR /app
-COPY requirements.txt .
-RUN pip install --no-cache-dir -r requirements.txt
-COPY . .
-EXPOSE 7860
-CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]
+FROM python:3.11-slim
+WORKDIR /app
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+COPY . .
+EXPOSE 7860
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]

README-DEPLOY.md

@@ -1,130 +1,159 @@
-# ResearchBee v1.4 — verified-sources release
+# ResearchBee v1.5.0 — Release A deployment guide
 
-## What changed, in one line
+Release A is a correctness and safety gate. Journal and repository facts are now
+constrained by server-side guards, permission and embargo decisions use explicit
+three-valued states, and the Khazna reload endpoint is authenticated.
 
-Policy facts now come from databases, not from the language model. Where no
-source could be reached, the system says so instead of filling the gap.
+## 1. Files changed in this release
 
----
+| File | Change |
+|---|---|
+| `app.py` | Journal-policy hard guard, authenticated Khazna reload, citation-chain proxy endpoint, repository-strategy hydration, and subject results with all field quartiles |
+| `prompts.py` | ID-only repository schema and removal of journal-policy generation from Journal Finder |
+| `repositories.py` | Strict ID hydration; only fit reason/confidence survive from the model |
+| `oaworks.py` | Permission and embargo tri-state logic |
+| `ResearchBee_LibGuides.html` | Safe cover-letter/related-work events, fixed citation-chain panels, and subject-field quartile display |
+| `tests/` | Regression tests for every Release A requirement |
+| `requirements-dev.txt` | Test-only dependencies |
+| `.env.example` | Secret and optional environment-variable template |
 
-## 1. HuggingFace Space — `researchbee-hf/`
+The bundled journal database, SCImago CSV, and other backend modules are retained
+from the supplied Hugging Face project.
 
-| File | Status |
-|---|---|
-| `app.py` | **modified** — DOI-first `/api/check-license`, registry-backed `/api/find-repository`, fixed publisher matcher, Khazna startup + 23:30 refresh, new `/api/health` |
-| `prompts.py` | **modified** — "ResearchPilot" → "ResearchBee" everywhere; `LICENSE_SYSTEM_PROMPT` can no longer state policy facts; new `ADVISORY_NOTE_PROMPT`; `REPO_SYSTEM_PROMPT` constrained to registry selection |
-| `oaworks.py` | **new** — OA.Works Permissions client, schema-confirmed against T&F / Elsevier / Springer |
-| `doi_utils.py` | **new** — DOI extraction from URLs and text; ISSN mod-11 validation |
-| `repositories.py` | **new** — 25-entry curated repository registry |
-| `khazna_loader.py` | **new** — loads the harvested Khazna index from an HF Dataset |
-| `requirements.txt` | **modified** — added `huggingface_hub`, `apscheduler` |
+## 2. Required Hugging Face secrets
 
-Upload all seven. `ku_knowledge.py`, `openalex.py`, `scimago_index.py`,
-`utils.py`, `Dockerfile`, and the SCImago CSV are unchanged — leave them.
+Add these under **Space Settings → Variables and secrets**:
 
-### Before you deploy
+```text
+OPENAI_API_KEY=<your OpenAI API key>
+KHAZNA_RELOAD_TOKEN=<a long random secret shared only with the harvest caller>
+```
 
-1. **Set `KU_ROR` in `oaworks.py`.** Find it at https://ror.org (search
-   "Khalifa University"). This matters more than it looks: Elsevier publishes a
-   12-month embargo for UK institutions and 24 months for everyone else, and
-   the ROR lets OA.Works pick the right one.
-2. Optionally set `KHAZNA_DATASET` and `HF_TOKEN` in Space secrets.
-3. Check `GET /api/health` after deploy — it reports every source's status.
+Generate the reload token with a password manager or, for example:
+
+```bash
+python -c "import secrets; print(secrets.token_urlsafe(48))"
+```
 
-Khazna lookups return `checked: false` until the harvest has run. That is
-correct behaviour, not a failure.
+Optional Khazna dataset settings:
 
----
+```text
+KHAZNA_DATASET=nikeshn/researchbee-data
+HF_TOKEN=<token with access to the dataset when it is private>
+```
 
-## 2. GitHub Pages — `researchbee-github/`
+Do not place real secrets in `.env.example`, the LibGuides page, GitHub Pages,
+or browser JavaScript.
 
-| File | Status |
-|---|---|
-| `index.html` | **modified** — DOI/URL field added as the primary input |
-| `js/license.js` | **modified** — sends the DOI, renders provenance, Khazna status, deposit statement |
-| `js/render_license.js` | **new** — the new render helpers |
-| `css/style.css` | **modified** — styles appended at the end |
-| `khazna_oai.py` | **new** — the OAI-PMH harvester (runs in Actions, not in the Space) |
-| `.github/workflows/harvest-khazna.yml` | **new** — nightly 23:00 Dubai harvest |
-| `test_publishers.py` | **new** — validation harness |
+## 3. Authenticated Khazna reload
 
-Other JS files are untouched.
+`POST /api/admin/reload-khazna` now fails closed:
 
----
+- `503` when `KHAZNA_RELOAD_TOKEN` is not configured on the Space
+- `401` when the caller supplies no token
+- `403` when the supplied token is wrong
+- success only after constant-time token comparison
 
-## 3. Khazna harvest — first run
+The harvest workflow should call the endpoint with either header form below.
+The custom header is recommended:
 
 ```bash
-# See the deposit gap before automating anything
-python khazna_oai.py gap
+curl --fail-with-body --request POST \
+  "https://YOUR-SPACE.hf.space/api/admin/reload-khazna" \
+  --header "X-ResearchBee-Reload-Token: ${KHAZNA_RELOAD_TOKEN}"
+```
+
+Bearer form is also supported:
 
-# Verify the parser against real records
-python khazna_oai.py inspect --set publications:withFiles --format mods -n 2 --raw
-python khazna_oai.py inspect --set openaire --format oai_dc -n 2 --raw
+```bash
+curl --fail-with-body --request POST \
+  "https://YOUR-SPACE.hf.space/api/admin/reload-khazna" \
+  --header "Authorization: Bearer ${KHAZNA_RELOAD_TOKEN}"
 ```
 
-Then:
-1. Create an HF **Dataset** repo (default: `nikeshn/researchbee-data`)
-2. Add `HF_TOKEN` as a GitHub repository secret
-3. Actions → "Harvest Khazna" → **Run workflow** (first full run)
-4. Nightly thereafter: Sunday full, Mon–Sat incremental
+Store the same token as a protected secret in the system that runs the harvest.
+Never call this endpoint from the LibGuides frontend.
+
+## 4. LibGuides deployment
 
-Sunday is full because Khazna's OAI reports `deletedRecord: no` — incrementals
-can never learn that a record was withdrawn.
+Use `ResearchBee_LibGuides.html` as the replacement LibGuides code block. Its
+API base currently points to:
 
----
+```text
+https://nikeshn-researchbee.hf.space
+```
 
-## 4. How to tell it is working
+Change `RB_API_BASE` and the existing `_HF` constants only when deploying to a
+different Space.
 
-**With a DOI** — green "Verified · OA.Works Permissions" badge, per-version
-rights, the publisher's required deposit statement with a copy button, a source
-panel with evidence links, and Khazna status.
+Release A frontend behaviour:
 
-**Without a DOI** — amber "Unverified · journal-level estimate", all version
-blocks "Unclear", a prompt to enter a DOI. The backend forcibly downgrades any
-`policy_status: "Confirmed"` the model returns, so this cannot be bypassed.
+- Cover-letter payloads remain in JavaScript memory and are not embedded in
+  inline event attributes.
+- Related-work titles, authors, links, and citation-chain actions are created
+  through DOM APIs and event listeners.
+- Citation-chain requests pass through `/api/openalex-cited-by`.
+- Subject search expands each journal's complete `quartiles_by_field` list.
+- Journal Finder explicitly says that deposit rights require the DOI-based
+  License Checking workflow.
 
-**Khazna unavailable** — "Khazna status not checked", explicitly distinguished
-from "Not yet in Khazna".
+## 5. Verification before publishing
 
-Test DOIs from this build:
+Install production and test dependencies, then run:
 
+```bash
+python -m pip install -r requirements-dev.txt
+pytest -q
+python -m compileall -q .
 ```
-10.1080/19322909.2023.2221477   T&F      — no embargo stated
-10.1016/j.jclepro.2023.136775   Elsevier — 24 months (not 12; see below)
-10.1007/s11192-023-04812-4      Springer — preprint immediate, AAM 12 months
+
+The delivered Release A build passes 18 regression tests. The suite covers:
+
+1. Repository ID schema and rejection of model-authored repository facts
+2. Allowed/denied/unknown permission states
+3. Zero/stated/unknown embargo states
+4. Journal Finder policy downgrading
+5. Citation-chain panel preservation
+6. Removal of unsafe cover-letter and related-work inline handlers
+7. Khazna reload authentication
+8. Subject-search field quartiles
+
+After deployment, check:
+
+```text
+GET /api/health
 ```
 
----
+Then exercise the three UI workflows with test inputs. External source responses
+may vary; an unavailable source must be rendered as unavailable or unknown, not
+as a permissive default.
 
-## 5. Things worth knowing
+## 6. Tri-state interpretation
 
-**The Elsevier embargo trap.** OA.Works returns two permissions with identical
-`score: 1100` — 12 months (UK) and 24 months (everyone else). Ranking by score
-alone silently picks 12. `_permission_rank()` in `oaworks.py` breaks the tie by
-the *longer* embargo. Do not "optimise" that away.
+ResearchBee now distinguishes:
 
-**Absent is not zero.** T&F records carry no embargo field. The system reports
-"Not stated in the permission record" rather than "no embargo", because T&F's
-published policy is normally 12 months and a wrong permissive answer costs the
-library a takedown notice.
+```text
+permission_status: allowed | denied | unknown
+embargo_status:    zero | stated | unknown
+```
 
-**Publisher matcher.** Now covers imprints whose SCImago string never says the
-parent publisher — W.B. Saunders, Academic Press, Cell Press, BioMed Central,
-Palgrave, Blackwell, Informa, Dove Medical. Q1 APC badge coverage went from
-45.8% to 51.1% (+497 journals). `AMBIGUOUS_ALIASES` prevents "Maximum Academic
-Press" from being mistaken for Elsevier.
+An absent embargo field is `unknown`, never zero. Immediate open deposit is
+`Yes` only when permission is allowed and the trusted record explicitly states
+zero embargo. When no verified permission record exists, ResearchBee says that
+it could not determine the right rather than claiming the right is denied.
 
-**Repository registry.** The model selects entries by `id` and writes the
-`fit_reason`; every factual field is overwritten from `repositories.py`.
-Unknown ids are dropped. Entries you have not personally verified are left
-blank rather than guessed — please fill them in and bump `verified`.
+## 7. Repository response contract
 
----
+The language model may return only:
 
-## 6. Still open
+```json
+{
+  "id": "exact_registry_id",
+  "fit_reason": "dataset-specific explanation",
+  "fit_confidence": "High|Medium|Low"
+}
+```
 
-- `KU_ROR` is empty
-- Repository registry has blank fields awaiting your verification
-- Wiley, IEEE and ACS not yet validated against OA.Works (`test_publishers.py`)
-- Worth telling OA.Works about the UK/non-UK tie: `help@openaccessbutton.org`
+The server rejects unknown IDs and supplies the repository name, URL, cost,
+identifier, versioning, embargo support, source URL, and verification date from
+`repositories.py`.

README.md

@@ -1,5 +1,5 @@
 ---
-title: Research Navigator
+title: ResearchBee
 emoji: 🐝
 colorFrom: indigo
 colorTo: purple
@@ -7,96 +7,110 @@ sdk: docker
 app_port: 7860
 pinned: false
 license: mit
-short_description: Journal finder, OA rights checker, data repository guide
+short_description: Journal finder, OA rights checker, and data repository guide
 ---
 
-# 🐝 Research Navigator
+# 🐝 ResearchBee v1.5.0
 
-**Publishing & Repository Assistant for Khalifa University Researchers**
+**Publishing, repository, and Open Access assistant for Khalifa University researchers**
 
-> *Find the right journal. Check your rights. Deposit to the right repository.*
+ResearchBee helps researchers identify suitable journals, check article-level
+self-archiving rights, and select an appropriate research-data repository.
+Verified databases determine factual values; the language model interprets the
+researcher's needs and explains server-supplied results.
 
-Research Navigator is an AI-powered publishing and open access assistant built by [Khalifa University Library](https://library.ku.ac.ae/lib). It helps KU researchers navigate journal selection, self-archiving rights, and research data management — in one place.
+## Main workflows
 
----
-
-## ✨ Features
+### Journal Finder
 
-### 📰 Tab 1 — Journal Submission
-- 5 detailed journal cards + 10 quick-match suggestions (15 total per query)
-- Full Green OA breakdown per journal: Preprint / AAM / Version of Record
-- Ranking verified from 2025 journal data (32K+ journals) + OpenAlex enrichment
-- Scopus Sources direct links per journal for live metric verification
-- Khazna deposit guidance always included
+- Manuscript-fit analysis with 10 detailed recommendations and up to 20 quick matches
+- Subject browsing against a closed SCImago/Scopus classification vocabulary
+- Journal metrics enriched from the bundled journal index and OpenAlex
+- Complete quartile-by-subject-field information in manuscript and subject results
+- No model-generated self-archiving facts: DOI-based License Checking is required
+  before any manuscript version is deposited
 
-### 🛡 Tab 2 — License Checking
-- Version-specific self-archiving policy (3-version breakdown)
-- Embargo lengths, licence conditions, risk flags
-- Direct deposit guidance for Khazna
+### License Checking
 
-### 🗄 Tab 3 — Data Repository
-- 4–5 recommended repositories (domain-specific + generalist mix)
-- Khazna always first as KU institutional option
-- Metadata-first rule: register in Khazna even if data goes elsewhere
-- FAIR alignment, certification, cost, embargo support per repository
+- DOI-first article-level permissions from ShareYourPaper Permissions/OA.Works
+- Explicit permission states: `allowed`, `denied`, or `unknown`
+- Explicit embargo states: `zero`, `stated`, or `unknown`
+- Khazna holdings status kept separate from policy status
+- Missing permission or embargo information remains unknown rather than becoming
+  a permissive or prohibitive default
 
-### 🏛️ Khazna Integration
-- KU institutional repository always recommended across all three tabs
-- Article and dataset deposit guidance
-- Contact: khazna@ku.ac.ae
+### Data Repository Finder
 
----
+- Model selects exact IDs from a curated KU Library registry
+- Server hydrates all factual repository attributes
+- Unknown IDs and model-authored facts are discarded
+- Khazna is included as the KU institutional repository
 
-## 🏗️ Architecture
+## Architecture
 
-```
-GitHub Pages (frontend) ──HTTPS──► HuggingFace Spaces (FastAPI)
-                                          │
-                              ┌───────────┴──────────────┐
-                              │  scimagojr_2025.csv       │
-                              │  OpenAlex API (free)      │
-                              │  OpenAI API (gpt-4o-mini) │
-                              └───────────────────────────┘
+```text
+LibGuides frontend
+        │ HTTPS
+        ▼
+Hugging Face Space — FastAPI
+        ├── journals.db / SCImago journal data
+        ├── OpenAlex metadata and citation services
+        ├── OA.Works article permissions
+        ├── Khazna index
+        ├── Curated repository registry
+        └── OpenAI for interpretation, ranking, and explanations
 ```
 
----
+## Deployment
+
+Required Space secrets:
 
-## 🚀 Deployment
+```text
+OPENAI_API_KEY
+KHAZNA_RELOAD_TOKEN
+```
 
-### Backend — This Space
-1. Add secret: `OPENAI_API_KEY` in Space Settings → Secrets
-2. All files are in the Space root and `static/` folder
-3. Ping `/api/health` every 5 min via UptimeRobot to prevent cold starts
+Optional Khazna dataset settings:
 
-### Frontend — GitHub Pages
-1. Update `HF_BASE` in `static/js/api.js` to your Space URL
-2. Push to `gh-pages` branch of `ku-library/researchbee`
+```text
+KHAZNA_DATASET
+HF_TOKEN
+```
 
----
+See `README-DEPLOY.md` for the authenticated reload call, LibGuides replacement,
+and verification procedure.
 
-## 📊 Data Sources
+## Tests
 
-| Source | Used for | Key needed? |
-|---|---|---|
-| SCImago CSV 2025 | Offline quartile lookup (private) | No — bundled |
-| OpenAlex API | Journal enrichment, h-index, citedness | No — free |
-| OpenAI API | All AI analysis | Yes |
+```bash
+python -m pip install -r requirements-dev.txt
+pytest -q
+```
 
-> Ranking metrics sourced from journal data (2025). Always verify policies on
-> [Open Policy Finder](https://openpolicyfinder.jisc.ac.uk) and repositories on
-> [re3data](https://www.re3data.org) before deposit.
+Release A includes regression coverage for repository ID grounding, permission
+and embargo tri-state logic, journal-policy downgrading, citation-chain DOM
+preservation, safe event handling, Khazna reload authentication, and subject
+quartiles by field.
 
----
+## Data sources
 
-## 🏛️ About
+| Source | Purpose |
+|---|---|
+| Bundled journal index / SCImago data | Journal rankings, lifecycle, and subject quartiles |
+| OpenAlex | Journal and works metadata, citation discovery |
+| ShareYourPaper Permissions / OA.Works | Article-level self-archiving permissions |
+| Khazna | KU repository holdings status |
+| Curated KU Library registry | Research-data repository facts |
+| OpenAI | Manuscript understanding, closed-list selection, ranking explanations, and cover letters |
 
-Built by [Khalifa University Library](https://library.ku.ac.ae/lib) as part of the Research Navigator suite.
+Always verify time-sensitive publisher policies and repository terms at the
+linked primary sources before submission or deposit.
 
-- 📧 Library: library@ku.ac.ae
-- 🏛️ Khazna: khazna@ku.ac.ae
-- 🌐 [khazna.ku.ac.ae](https://khazna.ku.ac.ae)
-- 📚 [library.ku.ac.ae/lib](https://library.ku.ac.ae/lib)
+## Contacts
 
----
+- KU Library: `library@ku.ac.ae`
+- Khazna: `khazna@ku.ac.ae`
+- Khazna portal: <https://khazna.ku.ac.ae>
+- KU Library: <https://library.ku.ac.ae/lib>
 
 MIT License — Khalifa University Library

RELEASE_A_NOTES.md

@@ -0,0 +1,46 @@
+# ResearchBee Release A — correctness and safety gate
+
+Version: 1.5.0
+
+## Implemented requirements
+
+1. **Repository ID schema** — the prompt returns exact registry IDs; the server
+   retains only fit reason/confidence and hydrates every factual field.
+2. **Permission and embargo tri-state** — allowed/denied/unknown and
+   zero/stated/unknown are propagated through policy parsing, recommendations,
+   actions, and the frontend.
+3. **Journal-policy downgrade** — Journal Finder model output is overwritten by
+   a server-side not-checked response; unsafe policy-bearing next actions are
+   discarded.
+4. **Citation-chain preservation** — related works load only into the cited
+   results panel, leaving tabs and the citation-chain panel in the DOM.
+5. **Safe cover-letter and related-work events** — raw values are stored in Maps;
+   rendering uses `textContent`, allowlisted HTTPS URLs, and `addEventListener`.
+6. **Khazna reload authentication** — the endpoint requires a shared secret via
+   `X-ResearchBee-Reload-Token` or Bearer authorization and fails closed when the
+   secret is not configured.
+7. **Regression tests** — 18 tests cover all Release A requirements.
+8. **Subject quartiles by field** — subject browse results now retrieve and
+   display every `quartiles_by_field` entry for each journal.
+
+## Changed backend files
+
+- `app.py`
+- `prompts.py`
+- `repositories.py`
+- `oaworks.py`
+- `README.md`
+- `README-DEPLOY.md`
+- `.env.example`
+- `requirements-dev.txt`
+- `tests/`
+
+## Frontend file
+
+- `ResearchBee_LibGuides.html`
+
+## Required deployment action
+
+Create a long random `KHAZNA_RELOAD_TOKEN` in the Hugging Face Space and store
+the same value in the Khazna harvest caller. The reload endpoint is unavailable
+until this secret is configured.

admin.py

@@ -1,731 +1,731 @@
-"""
-admin.py — ResearchBee admin dashboard
-
-Ported from LibBee v3.8's admin module so the two dashboards behave
-identically: same cookie-session login, same 5-attempt IP lockout, same
-server-rendered single page, same visual language. Someone who
-administers LibBee should not have to learn a second interface.
-
-What is deliberately different
-------------------------------
-LibBee's dashboard shows recent questions. This one cannot, because
-ResearchBee never stores question text — see usage.py. The equivalent
-panels here answer "how much of each tool is being used" and "how often
-could we not answer", both from counters.
-
-Mount with:
-    from admin import router as admin_router
-    app.include_router(admin_router, prefix="/admin")
-
-Environment:
-    RESEARCHBEE_ADMIN_PASSWORD   required — the dashboard is unreachable without it
-    CLOUDFLARE_WORKER_URL        the LibBee Worker (shared)
-    CLOUDFLARE_WORKER_TOKEN      must equal the Worker's ANALYTICS_TOKEN
-"""
-
-from __future__ import annotations
-
-import json
-import os
-import secrets
-import time
-from typing import Any, Dict, Optional
-
-from fastapi import APIRouter, HTTPException, Request, Response
-from fastapi.responses import HTMLResponse, RedirectResponse
-from pydantic import BaseModel
-
-router = APIRouter()
-
-_START_TIME = time.time()
-
-ADMIN_PASSWORD = (os.environ.get("RESEARCHBEE_ADMIN_PASSWORD") or "").strip()
-WORKER_URL = (os.environ.get("CLOUDFLARE_WORKER_URL") or "").strip().strip('"').strip("'").rstrip("/")
-WORKER_TOKEN = (os.environ.get("CLOUDFLARE_WORKER_TOKEN") or "").strip()
-
-_SESSIONS_FILE = "/tmp/researchbee_admin_sessions.json"
-
-admin_sessions: Dict[str, float] = {}
-
-# Failed logins per IP. Five misses inside the window locks that IP out
-# for fifteen minutes — enough to make guessing a shared password
-# impractical without locking out a librarian who fat-fingers it twice.
-_login_attempts: Dict[str, list] = {}
-_MAX_LOGIN_ATTEMPTS = 5
-_LOCKOUT_WINDOW = 300
-_LOCKOUT_DURATION = 900
-
-
-# ── Sessions ──────────────────────────────────────────────────────────
-
-def _load_sessions() -> None:
-    try:
-        with open(_SESSIONS_FILE, "r", encoding="utf-8") as f:
-            data = json.load(f)
-        now = time.time()
-        admin_sessions.update({
-            t: exp for t, exp in data.items()
-            if isinstance(exp, (int, float)) and exp > now
-        })
-    except Exception:
-        pass
-
-
-def _save_sessions() -> None:
-    try:
-        now = time.time()
-        with open(_SESSIONS_FILE, "w", encoding="utf-8") as f:
-            json.dump({t: e for t, e in admin_sessions.items() if e > now}, f)
-    except Exception:
-        pass
-
-
-def _cleanup_sessions() -> None:
-    now = time.time()
-    for t in [t for t, exp in admin_sessions.items() if exp <= now]:
-        admin_sessions.pop(t, None)
-    _load_sessions()
-
-
-def _has_session(request: Request) -> bool:
-    _cleanup_sessions()
-    token = request.cookies.get("rb_admin_session")
-    return bool(token and token in admin_sessions)
-
-
-def _require_session(request: Request) -> None:
-    if not _has_session(request):
-        raise HTTPException(status_code=401, detail="Not authenticated")
-
-
-class LoginRequest(BaseModel):
-    password: str
-
-
-# ── Auth ──────────────────────────────────────────────────────────────
-
-@router.post("/auth")
-async def admin_auth(req: LoginRequest, request: Request):
-    client_ip = request.client.host if request.client else "unknown"
-    now = time.time()
-
-    attempts = [t for t in _login_attempts.get(client_ip, []) if now - t < _LOCKOUT_WINDOW]
-    if len(attempts) >= _MAX_LOGIN_ATTEMPTS:
-        wait = int(_LOCKOUT_DURATION - (now - attempts[0]))
-        raise HTTPException(429, f"Too many failed attempts. Try again in {wait} seconds.")
-
-    if not ADMIN_PASSWORD or not secrets.compare_digest(
-        req.password.encode(), ADMIN_PASSWORD.encode()
-    ):
-        attempts.append(now)
-        _login_attempts[client_ip] = attempts
-        remaining = _MAX_LOGIN_ATTEMPTS - len(attempts)
-        if not ADMIN_PASSWORD:
-            detail = "No admin password is set on this Space. Add RESEARCHBEE_ADMIN_PASSWORD to the Space secrets."
-        elif remaining <= 0:
-            detail = f"Too many failed attempts. Try again in {_LOCKOUT_DURATION // 60} minutes."
-        else:
-            detail = f"Incorrect password. {remaining} attempt{'s' if remaining != 1 else ''} left before lockout."
-        raise HTTPException(401, detail)
-
-    _login_attempts.pop(client_ip, None)
-    token = secrets.token_hex(32)
-    admin_sessions[token] = now + 86400
-    _save_sessions()
-    resp = Response(content='{"status":"ok"}', media_type="application/json")
-    resp.set_cookie("rb_admin_session", token, httponly=True,
-                    max_age=86400, samesite="lax", secure=True)
-    return resp
-
-
-@router.get("/logout")
-async def admin_logout(request: Request):
-    token = request.cookies.get("rb_admin_session")
-    if token:
-        admin_sessions.pop(token, None)
-    _save_sessions()
-    resp = RedirectResponse(url="/admin/login")
-    resp.delete_cookie("rb_admin_session")
-    return resp
-
-
-# ── Worker proxies ────────────────────────────────────────────────────
-# The Worker URL and token stay server-side. The browser only ever talks
-# to this Space, so neither secret reaches the page.
-
-async def _worker_get(path: str, params: Optional[dict] = None):
-    if not WORKER_URL:
-        raise HTTPException(503, "CLOUDFLARE_WORKER_URL is not set on this Space")
-    import httpx
-    headers = {"Authorization": f"Bearer {WORKER_TOKEN}"} if WORKER_TOKEN else {}
-    try:
-        async with httpx.AsyncClient(timeout=15) as client:
-            r = await client.get(f"{WORKER_URL}{path}", params=params or {}, headers=headers)
-            return r.json()
-    except HTTPException:
-        raise
-    except Exception as exc:
-        raise HTTPException(502, f"Worker unreachable: {type(exc).__name__}: {exc}")
-
-
-@router.get("/usage")
-async def admin_usage(request: Request, days: int = 30):
-    _require_session(request)
-    return await _worker_get("/rb/usage", {"days": max(0, min(days, 3650))})
-
-
-@router.get("/canary-status")
-async def admin_canary_status(request: Request):
-    _require_session(request)
-    return await _worker_get("/rb/canary/status")
-
-
-@router.get("/local-usage")
-async def admin_local_usage(request: Request):
-    """Counts held in this container since it last started. Useful when
-    the Worker is unreachable, and as a cross-check that counting is
-    actually happening before blaming the Worker."""
-    _require_session(request)
-    from usage import usage
-    return usage.snapshot()
-
-
-# ── Pull endpoints (called by the Cloudflare Worker, not by a browser) ─
-# Authenticated with the shared token rather than an admin session,
-# because the caller is a machine. This is the direction that works:
-# Cloudflare can reach this Space even though this Space cannot reach
-# Cloudflare.
-
-def _require_worker_token(request: Request) -> None:
-    if not WORKER_TOKEN:
-        raise HTTPException(503, "CLOUDFLARE_WORKER_TOKEN is not set on this Space")
-    supplied = request.headers.get("authorization", "")
-    if not secrets.compare_digest(supplied, f"Bearer {WORKER_TOKEN}"):
-        raise HTTPException(401, "Unauthorized")
-
-
-def _worker_or_session(request: Request) -> None:
-    """Accept either the Worker's token or a signed-in admin, so the same
-    endpoints back both the cron pull and the dashboard's manual send."""
-    if _has_session(request):
-        return
-    _require_worker_token(request)
-
-
-@router.get("/usage-export")
-async def usage_export(request: Request):
-    """Hand over accumulated counts. Counts are held, not deleted, until
-    the caller confirms with /usage-ack — so a caller that reads them and
-    then fails to store them cannot destroy them."""
-    _worker_or_session(request)
-    from usage import usage
-    return usage.export_snapshot()
-
-
-@router.post("/usage-ack")
-async def usage_ack(request: Request, snapshot_id: str):
-    """Confirm a snapshot reached storage. Only now is it discarded."""
-    _worker_or_session(request)
-    from usage import usage
-    return {"acknowledged": usage.ack_snapshot(snapshot_id)}
-
-
-@router.get("/worker-probe")
-async def admin_worker_probe(request: Request):
-    """Is this Space's route to the Worker broken, or just slow?
-
-    HF Spaces lose connectivity to workers.dev for stretches at a time —
-    LibBee hit the same thing and moved its analytics writes into the
-    browser because of it. Three attempts with timings tell you which
-    case you are in: three identical failures is a blocked route, a mix
-    is the intermittent case, which the flush buffer already absorbs.
-    """
-    _require_session(request)
-    import httpx, time as _t
-    if not WORKER_URL:
-        return {"error": "CLOUDFLARE_WORKER_URL is not set on this Space"}
-    attempts = []
-    for i in range(3):
-        t0 = _t.time()
-        try:
-            async with httpx.AsyncClient(timeout=20) as client:
-                r = await client.get(f"{WORKER_URL}/version")
-            attempts.append({"attempt": i + 1, "ok": r.status_code == 200,
-                             "status": r.status_code,
-                             "ms": int((_t.time() - t0) * 1000)})
-        except Exception as exc:
-            attempts.append({"attempt": i + 1, "ok": False,
-                             "error": f"{type(exc).__name__}: {exc}"[:160],
-                             "ms": int((_t.time() - t0) * 1000)})
-    ok = sum(1 for a in attempts if a.get("ok"))
-    return {
-        "worker_url": WORKER_URL,
-        "attempts": attempts,
-        "verdict": ("reachable" if ok == 3 else
-                    "intermittent" if ok else "unreachable from this Space"),
-        "note": ("Counts are buffered and retried every 60 seconds, so patchy "
-                 "egress delays the dashboard without losing data. Dashboard "
-                 "reads fall back to your browser, which usually reaches the "
-                 "Worker even when this Space cannot."),
-    }
-
-
-@router.get("/status")
-async def admin_status(request: Request):
-    _require_session(request)
-    import app as rb_app
-
-    out: Dict[str, Any] = {
-        "uptime_seconds": int(time.time() - _START_TIME),
-        "worker_url_configured": bool(WORKER_URL),
-        "worker_token_configured": bool(WORKER_TOKEN),
-        "admin_password_set": bool(ADMIN_PASSWORD),
-        "openai_key_set": bool(os.environ.get("OPENAI_API_KEY")),
-    }
-    try:
-        out["health"] = await rb_app.health()
-    except Exception as exc:
-        out["health"] = {"error": f"{type(exc).__name__}: {exc}"}
-    return out
-
-
-@router.post("/reload-khazna")
-async def admin_reload_khazna(request: Request):
-    """Session-protected wrapper around the existing reload endpoint, so
-    the button on this page works without exposing an unauthenticated
-    admin path."""
-    _require_session(request)
-    import asyncio
-    from app import khazna_index
-    ok = await asyncio.to_thread(khazna_index.reload)
-    return {"reloaded": ok, "khazna": khazna_index.status}
-
-
-# ── Login page ────────────────────────────────────────────────────────
-
-@router.get("/login", response_class=HTMLResponse)
-async def admin_login_page(request: Request):
-    if _has_session(request):
-        return RedirectResponse(url="/admin")
-    return HTMLResponse("""<!DOCTYPE html>
-<html><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
-<title>ResearchBee — Admin</title>
-<style>
-*{box-sizing:border-box;margin:0;padding:0}
-body{font-family:'Segoe UI',system-ui,sans-serif;background:#f0f4f8;display:flex;justify-content:center;align-items:center;min-height:100vh}
-.box{background:#fff;border-radius:16px;padding:40px;width:360px;box-shadow:0 4px 24px rgba(0,0,0,.1);text-align:center}
-h1{color:#003366;font-size:1.3rem;margin-bottom:6px}
-.sub{color:#6b7280;font-size:.84rem;margin-bottom:24px}
-input{width:100%;padding:12px 14px;border:1.5px solid #d1d5db;border-radius:10px;font-size:.92rem;margin-bottom:14px}
-input:focus{outline:none;border-color:#003366}
-.btn{width:100%;padding:12px;background:#003366;color:#fff;border:none;border-radius:10px;font-weight:700;font-size:.92rem;cursor:pointer}
-.btn:hover{background:#004488}
-.err{color:#dc2626;font-size:.82rem;margin-bottom:10px;display:none}
-</style></head><body>
-<div class="box">
-  <h1>🐝 ResearchBee Admin</h1>
-  <div class="sub">KU Library publishing assistant</div>
-  <div class="err" id="err"></div>
-  <form onsubmit="return go(event)">
-    <input type="password" id="pw" placeholder="Admin password" autofocus>
-    <button type="submit" class="btn">Sign in</button>
-  </form>
-</div>
-<script>
-async function go(e){
-  e.preventDefault();
-  const r=await fetch('/admin/auth',{method:'POST',headers:{'Content-Type':'application/json'},
-    body:JSON.stringify({password:document.getElementById('pw').value})});
-  if(r.ok){window.location.href='/admin';return false;}
-  const d=await r.json().catch(()=>({}));
-  const el=document.getElementById('err');
-  el.textContent=d.detail||'Incorrect password.';el.style.display='block';
-  document.getElementById('pw').value='';document.getElementById('pw').focus();
-  return false;
-}
-</script></body></html>""")
-
-
-# ── Dashboard ─────────────────────────────────────────────────────────
-
-SURFACE_LABELS = {
-    "journal_subject":    "Find a journal — browse by subject",
-    "journal_manuscript": "Find a journal — analyse manuscript",
-    "licence_deposit":    "Share your article (deposit rights)",
-    "data_repository":    "Deposit your data (repository finder)",
-    "cover_letter":       "Cover letter generator",
-    "chat":               "Chat assistant",
-}
-
-GAP_LABELS = {
-    "scimago_miss":      "Subject not found in the journal index",
-    "subject_unrecognised": "Subject did not map to any SCImago category",
-    # Model named a category that does not exist. High counts here mean the
-    # vocabulary is missing terms researchers actually use (e.g. criminology).
-    "category_invented": "Subject has no category in Scopus/SCImago",
-    # Answered, but with the wrong kind of answer: no category matched, so the
-    # list is the whole subject AREA ranked by h-index. A high count here means
-    # the vocabulary is missing terms researchers actually use.
-    "area_fallback":     "Showed a broad subject area, not a specific category",
-    "oaworks_404":       "No permissions record for that article",
-    "oaworks_unconfirmed": "Permissions record found but policy not confirmed",
-    "doi_unparseable":   "Could not read a DOI from what was entered",
-    "khazna_no_doi":     "No DOI supplied, so Khazna could not be checked",
-    "khazna_unavailable": "Khazna index not loaded",
-    "no_apc_agreement":  "No KU APC agreement for that publisher",
-    "no_repository_match": "No repository matched the dataset",
-}
-
-
-@router.get("", response_class=HTMLResponse)
-async def admin_dashboard(request: Request):
-    if not _has_session(request):
-        return RedirectResponse(url="/admin/login")
-
-    page = """<!DOCTYPE html>
-<html><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
-<title>ResearchBee — Admin</title>
-<style>
-*{box-sizing:border-box;margin:0;padding:0}
-body{font-family:'Segoe UI',system-ui,sans-serif;background:#f0f4f8;color:#1a1a2e}
-.topbar{background:#003366;color:#fff;padding:12px 20px;display:flex;align-items:center;justify-content:space-between}
-.topbar h1{font-size:1.1rem;font-weight:700}
-.topbar a{color:#C8A951;font-size:.82rem;text-decoration:none;padding:5px 12px;border:1px solid #C8A951;border-radius:8px}
-.main{max-width:900px;margin:24px auto;padding:0 16px;display:flex;flex-direction:column;gap:20px}
-.card{background:#fff;border-radius:14px;padding:20px;box-shadow:0 2px 12px rgba(0,0,0,.07)}
-.card h2{font-size:1rem;font-weight:700;color:#003366;margin-bottom:6px}
-.card .hint{font-size:.78rem;color:#6b7280;margin-bottom:14px;line-height:1.5}
-.btn{padding:9px 20px;background:#003366;color:#fff;border:none;border-radius:8px;font-weight:700;cursor:pointer;font-size:.88rem}
-.btn:hover{background:#004488}
-.btn-ghost{background:#fff;color:#003366;border:1.5px solid #d1d5db}
-.btn-ghost:hover{background:#f8fafc}
-.stat-grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(140px,1fr));gap:12px}
-.stat{background:#f8fafc;border-radius:10px;padding:14px;text-align:center;border:1px solid #e5e7eb}
-.stat .val{font-size:1.6rem;font-weight:800;color:#003366}
-.stat .lbl{font-size:.74rem;color:#6b7280;margin-top:2px}
-table{width:100%;border-collapse:collapse;font-size:.84rem}
-th{text-align:left;font-size:.72rem;text-transform:uppercase;letter-spacing:.05em;color:#6b7280;border-bottom:1px solid #e5e7eb;padding:7px 8px}
-td{padding:7px 8px;border-bottom:1px solid #f1f5f9}
-tr:last-child td{border-bottom:none}
-.bar{height:8px;background:#e5e7eb;border-radius:4px;overflow:hidden;min-width:60px}
-.bar span{display:block;height:100%;background:#003366}
-.pill{display:inline-block;padding:2px 9px;border-radius:20px;font-size:.72rem;font-weight:700}
-.pill-ok{background:#dcfce7;color:#15803d}
-.pill-bad{background:#fee2e2;color:#b91c1c}
-.pill-warn{background:#fef3c7;color:#92400e}
-.range{display:flex;gap:6px;margin-bottom:12px;flex-wrap:wrap}
-.range button{padding:4px 12px;border:1.5px solid #d1d5db;background:#fff;border-radius:20px;font-size:.78rem;cursor:pointer;color:#6b7280}
-.range button.on{border-color:#003366;background:#003366;color:#fff;font-weight:700}
-.msg{font-size:.82rem;color:#6b7280;padding:8px 0}
-.err{font-size:.82rem;color:#b91c1c;padding:8px 0}
-.mono{font-family:ui-monospace,SFMono-Regular,Menlo,monospace;font-size:.76rem}
-</style></head><body>
-<div class="topbar">
-  <h1>🐝 ResearchBee — Admin</h1>
-  <a href="/admin/logout">Sign out</a>
-</div>
-<div class="main">
-
-  <div class="card">
-    <h2>Tool usage</h2>
-    <div class="hint">How many times each tool was used. Counts only — ResearchBee never
-      stores what anyone typed, so there is no question log here by design.
-      Keep-alive traffic is excluded.</div>
-    <div class="range">
-      <button data-days="7">7 days</button>
-      <button data-days="30" class="on">30 days</button>
-      <button data-days="90">90 days</button>
-      <button data-days="0">All time</button>
-    </div>
-    <div id="usage"><div class="msg">Loading…</div></div>
-  </div>
-
-  <div class="card">
-    <h2>Questions we could not answer</h2>
-    <div class="hint">Requests that completed but found nothing to answer with — a subject
-      missing from the journal index, an article with no permissions record. Each one is a
-      candidate for improving a data source.</div>
-    <div id="gaps"><div class="msg">Loading…</div></div>
-  </div>
-
-  <div class="card">
-    <h2>Health checks</h2>
-    <div class="hint">Every six hours the Worker asks ResearchBee a real question, rotating
-      through the four tools. This keeps the Space awake and proves a different pipeline
-      still works each time.</div>
-    <div id="canary"><div class="msg">Loading…</div></div>
-  </div>
-
-  <div class="card">
-    <h2>Service status</h2>
-    <div class="hint">Data sources this Space is currently using, and its connection to the
-      analytics Worker.</div>
-    <div id="status"><div class="msg">Loading…</div></div>
-    <div style="margin-top:14px;display:flex;gap:8px;flex-wrap:wrap">
-      <button class="btn" id="reload-khazna">Reload the Khazna index</button>
-      <button class="btn btn-ghost" id="show-local">Show counts held in this container</button>
-      <button class="btn btn-ghost" id="probe-worker">Test the connection to Cloudflare</button>
-      <button class="btn btn-ghost" id="send-now">Send counts to Cloudflare now</button>
-    </div>
-    <div id="action-out"></div>
-  </div>
-
-</div>
-<script>
-// The dashboard tries the Space's server-side proxy first, then falls back to
-// calling the Worker directly from this browser. HF Spaces lose egress to
-// workers.dev for stretches at a time, while the admin's own browser reaches
-// it fine — LibBee hit exactly this and solved it the same way. This page is
-// served only to an authenticated session, so the token is not public.
-const WORKER_URL   = "__WORKER_URL__";
-const WORKER_TOKEN = "__WORKER_TOKEN__";
-const SURFACE = """ + json.dumps(SURFACE_LABELS) + """;
-const GAP = """ + json.dumps(GAP_LABELS) + """;
-const esc = s => String(s??'').replace(/&/g,'&amp;').replace(/</g,'&lt;').replace(/>/g,'&gt;');
-const num = n => Number(n||0).toLocaleString();
-
-let usedFallback = false;
-
-async function getDirect(workerPath){
-  if(!WORKER_URL || !WORKER_TOKEN) throw new Error('no direct route configured');
-  const r = await fetch(WORKER_URL + workerPath, {
-    headers: { Authorization: 'Bearer ' + WORKER_TOKEN }
-  });
-  if(!r.ok) throw new Error('Worker returned HTTP ' + r.status);
-  usedFallback = true;
-  return r.json();
-}
-
-// workerPath is the same data read straight from the Worker. Passing it makes
-// this call survive the Space losing its outbound route to Cloudflare.
-async function get(path, workerPath){
-  try {
-    const r = await fetch(path);
-    if(r.status===401){window.location.href='/admin/login';throw new Error('unauthenticated');}
-    const d = await r.json().catch(()=>({}));
-    if(!r.ok) throw new Error(d.detail || ('HTTP '+r.status));
-    return d;
-  } catch(e) {
-    if(String(e.message).includes('unauthenticated') || !workerPath) throw e;
-    // Space could not reach the Worker; try from here instead.
-    return getDirect(workerPath);
-  }
-}
-
-function fallbackNotice(){
-  return usedFallback
-    ? `<div class="hint" style="color:#92400e;background:#fef3c7;border-radius:8px;padding:7px 10px;margin-bottom:10px">
-         Read directly from Cloudflare — this Space could not reach the Worker just now.
-         Counts still being collected are buffered and will be sent when the route recovers.
-       </div>`
-    : '';
-}
-
-// ── Usage ──────────────────────────────────────────────────────
-let days = 30;
-
-async function loadUsage(){
-  const el = document.getElementById('usage');
-  const gapEl = document.getElementById('gaps');
-  el.innerHTML = '<div class="msg">Loading…</div>';
-  try {
-    const d = await get('/admin/usage?days='+days, '/usage?days='+days);
-    const t = d.totals || {};
-    const rows = (d.by_surface || []);
-    const max = Math.max(1, ...rows.map(r => r.total || 0));
-
-    el.innerHTML = fallbackNotice() +
-      `<div class="stat-grid" style="margin-bottom:16px">
-         <div class="stat"><div class="val">${num(t.total)}</div><div class="lbl">Times used</div></div>
-         <div class="stat"><div class="val">${num(t.today)}</div><div class="lbl">Today</div></div>
-         <div class="stat"><div class="val">${num(t.miss)}</div><div class="lbl">Unanswered</div></div>
-         <div class="stat"><div class="val">${num(t.errors)}</div><div class="lbl">Errors</div></div>
-       </div>` +
-      (rows.length
-        ? `<table><thead><tr><th>Tool</th><th>Times used</th><th style="width:35%"></th><th>Unanswered</th><th>Errors</th></tr></thead><tbody>` +
-          rows.map(r => `<tr>
-              <td>${esc(SURFACE[r.surface] || r.surface)}</td>
-              <td><strong>${num(r.total)}</strong></td>
-              <td><div class="bar"><span style="width:${Math.round((r.total/max)*100)}%"></span></div></td>
-              <td>${num(r.miss)}</td>
-              <td>${num(r.errors)}</td>
-            </tr>`).join('') +
-          `</tbody></table>`
-        : `<div class="msg">Nothing recorded in this period yet. Use one of the tools, wait a minute for the next flush, then reload.</div>`);
-
-    const gaps = d.gaps || [];
-    gapEl.innerHTML = gaps.length
-      ? `<table><thead><tr><th>What was missing</th><th>Tool</th><th>Times</th></tr></thead><tbody>` +
-        gaps.map(g => `<tr>
-            <td>${esc(GAP[g.gap_code] || g.gap_code)}</td>
-            <td>${esc(SURFACE[g.surface] || g.surface)}</td>
-            <td><strong>${num(g.c)}</strong></td>
-          </tr>`).join('') + `</tbody></table>`
-      : `<div class="msg">Nothing flagged in this period.</div>`;
-
-  } catch(e) {
-    el.innerHTML = '<div class="err">Could not load usage: '+esc(e.message)+'</div>';
-    gapEl.innerHTML = '';
-  }
-}
-
-document.querySelectorAll('.range button').forEach(b => {
-  b.addEventListener('click', () => {
-    document.querySelectorAll('.range button').forEach(x => x.classList.remove('on'));
-    b.classList.add('on');
-    days = parseInt(b.dataset.days, 10);
-    loadUsage();
-  });
-});
-
-// ── Canary ─────────────────────────────────────────────────────
-async function loadCanary(){
-  const el = document.getElementById('canary');
-  try {
-    const d = await get('/admin/canary-status', '/canary/status');
-    const runs = d.runs || [];
-    el.innerHTML =
-      `<div class="msg" style="padding-top:0">${esc(d.summary||'')}</div>` +
-      (runs.length
-        ? `<table><thead><tr><th>When (UTC)</th><th>Tool checked</th><th>Result</th><th>Took</th><th>Detail</th></tr></thead><tbody>` +
-          runs.map(r => `<tr>
-              <td class="mono">${esc(r.run_at)}</td>
-              <td>${esc(r.slot||'')}</td>
-              <td><span class="pill ${r.ok?'pill-ok':'pill-bad'}">${r.ok?'Passed':'Failed'}</span></td>
-              <td>${r.latency_ms!=null?(r.latency_ms/1000).toFixed(1)+'s':''}</td>
-              <td class="mono">${esc(r.error || r.detail || '')}</td>
-            </tr>`).join('') + `</tbody></table>`
-        : '');
-  } catch(e) {
-    el.innerHTML = '<div class="err">Could not load health checks: '+esc(e.message)+'</div>';
-  }
-}
-
-// ── Status ─────────────────────────────────────────────────────
-function yesNo(v){ return `<span class="pill ${v?'pill-ok':'pill-warn'}">${v?'Yes':'No'}</span>`; }
-
-async function loadStatus(){
-  const el = document.getElementById('status');
-  try {
-    const d = await get('/admin/status');
-    const h = d.health || {};
-    const src = h.sources || {};
-    const k = src.khazna || {};
-    const hrs = Math.floor((d.uptime_seconds||0)/3600);
-    const mins = Math.floor(((d.uptime_seconds||0)%3600)/60);
-    el.innerHTML = `<table><tbody>
-      <tr><td>Version</td><td>${esc(h.version||'unknown')}</td></tr>
-      <tr><td>Running for</td><td>${hrs}h ${mins}m</td></tr>
-      <tr><td>Khazna index loaded</td><td>${yesNo(k.available)}${k.records?' &middot; '+num(k.records)+' records':''}</td></tr>
-      <tr><td>KU institution ID set for permissions</td><td>${yesNo(src.oaworks && src.oaworks.ku_ror_set)}</td></tr>
-      <tr><td>Repository registry</td><td>${num((src.repository_registry||{}).entries)} entries</td></tr>
-      <tr><td>Analytics Worker configured</td><td>${yesNo(d.worker_url_configured)}</td></tr>
-      <tr><td>Worker token configured</td><td>${yesNo(d.worker_token_configured)}</td></tr>
-      <tr><td>Language model key set</td><td>${yesNo(d.openai_key_set)}</td></tr>
-    </tbody></table>`;
-  } catch(e) {
-    el.innerHTML = '<div class="err">Could not load status: '+esc(e.message)+'</div>';
-  }
-}
-
-// ── Actions ────────────────────────────────────────────────────
-document.getElementById('reload-khazna').addEventListener('click', async (ev) => {
-  const out = document.getElementById('action-out');
-  ev.target.disabled = true;
-  out.innerHTML = '<div class="msg">Reloading…</div>';
-  try {
-    const r = await fetch('/admin/reload-khazna', {method:'POST'});
-    const d = await r.json();
-    out.innerHTML = d.reloaded
-      ? `<div class="msg">Reloaded. ${num((d.khazna||{}).records)} records now in the index.</div>`
-      : `<div class="err">Reload failed. Check the Space logs for the line beginning "Khazna index download failed".</div>`;
-    loadStatus();
-  } catch(e) {
-    out.innerHTML = '<div class="err">Reload failed: '+esc(e.message)+'</div>';
-  }
-  ev.target.disabled = false;
-});
-
-document.getElementById('show-local').addEventListener('click', async () => {
-  const out = document.getElementById('action-out');
-  try {
-    const d = await get('/admin/local-usage');
-    const rows = Object.entries(d.by_surface||{});
-    out.innerHTML =
-      `<div class="msg">${esc(d.note)} Counting since ${esc(d.since)}. `+
-      `${d.pending_deltas} delta(s) waiting to be sent.</div>` +
-      (rows.length
-        ? `<table><thead><tr><th>Tool</th><th>Used</th><th>Unanswered</th><th>Errors</th></tr></thead><tbody>` +
-          rows.map(([k,v]) => `<tr><td>${esc(SURFACE[k]||k)}</td><td>${num(v.total)}</td><td>${num(v.miss)}</td><td>${num(v.error)}</td></tr>`).join('') +
-          `</tbody></table>`
-        : '<div class="msg">Nothing counted since this container started.</div>') +
-      ((d.recent_flushes||[]).length
-        ? `<div class="hint" style="margin-top:10px">Last send to the Worker: <span class="mono">${esc(d.recent_flushes[0].outcome)} — ${esc(d.recent_flushes[0].detail)}</span></div>`
-        : '');
-  } catch(e) {
-    out.innerHTML = '<div class="err">'+esc(e.message)+'</div>';
-  }
-});
-
-// This Space cannot reach Cloudflare, but your browser can — the same route
-// the dashboard reads through. So a manual send goes: Space -> browser -> Worker.
-// Worth doing before a planned redeploy, since counts sit in memory until the
-// Worker's next 6-hourly pull.
-document.getElementById('send-now').addEventListener('click', async (ev) => {
-  const out = document.getElementById('action-out');
-  ev.target.disabled = true;
-  out.innerHTML = '<div class="msg">Collecting counts…</div>';
-  try {
-    const snap = await get('/admin/usage-export');
-    if (!snap.snapshot_id || !snap.deltas.length) {
-      out.innerHTML = '<div class="msg">Nothing waiting to be sent.</div>';
-      ev.target.disabled = false; return;
-    }
-    out.innerHTML = `<div class="msg">Sending ${snap.deltas.length} item(s) via this browser…</div>`;
-    const r = await fetch(WORKER_URL + '/log', {
-      method: 'POST',
-      headers: { 'Content-Type': 'application/json', Authorization: 'Bearer ' + WORKER_TOKEN },
-      body: JSON.stringify({ deltas: snap.deltas })
-    });
-    const d = await r.json();
-    if (r.ok && d.logged) {
-      await fetch('/admin/usage-ack?snapshot_id=' + encodeURIComponent(snap.snapshot_id), {method:'POST'});
-      out.innerHTML = `<div class="msg">Sent. ${d.accepted} item(s) stored in Cloudflare.</div>`;
-      loadUsage();
-    } else {
-      out.innerHTML = `<div class="err">Cloudflare refused the counts: ${esc(d.warning||('HTTP '+r.status))}.
-        They are held and will be collected on the next scheduled pull.</div>`;
-    }
-  } catch(e) {
-    out.innerHTML = `<div class="err">Could not send: ${esc(e.message)}.
-      Counts are held and will be collected on the next scheduled pull.</div>`;
-  }
-  ev.target.disabled = false;
-});
-
-document.getElementById('probe-worker').addEventListener('click', async (ev) => {
-  const out = document.getElementById('action-out');
-  ev.target.disabled = true;
-  out.innerHTML = '<div class="msg">Testing three times…</div>';
-  try {
-    const d = await get('/admin/worker-probe');
-    const rows = (d.attempts||[]).map(a =>
-      `<tr><td>Attempt ${a.attempt}</td>
-           <td><span class="pill ${a.ok?'pill-ok':'pill-bad'}">${a.ok?'Reached':'Failed'}</span></td>
-           <td>${a.ms} ms</td><td class="mono">${esc(a.error||'')}</td></tr>`).join('');
-    out.innerHTML =
-      `<div class="msg">Verdict: <strong>${esc(d.verdict)}</strong></div>` +
-      `<table><tbody>${rows}</tbody></table>` +
-      `<div class="hint" style="margin-top:8px">${esc(d.note||'')}</div>`;
-  } catch(e) {
-    out.innerHTML = '<div class="err">'+esc(e.message)+'</div>';
-  }
-  ev.target.disabled = false;
-});
-
-loadUsage(); loadCanary(); loadStatus();
-</script></body></html>"""
-    return HTMLResponse(
-        page.replace("__WORKER_URL__", WORKER_URL)
-            .replace("__WORKER_TOKEN__", WORKER_TOKEN)
+"""
+admin.py — ResearchBee admin dashboard
+
+Ported from LibBee v3.8's admin module so the two dashboards behave
+identically: same cookie-session login, same 5-attempt IP lockout, same
+server-rendered single page, same visual language. Someone who
+administers LibBee should not have to learn a second interface.
+
+What is deliberately different
+------------------------------
+LibBee's dashboard shows recent questions. This one cannot, because
+ResearchBee never stores question text — see usage.py. The equivalent
+panels here answer "how much of each tool is being used" and "how often
+could we not answer", both from counters.
+
+Mount with:
+    from admin import router as admin_router
+    app.include_router(admin_router, prefix="/admin")
+
+Environment:
+    RESEARCHBEE_ADMIN_PASSWORD   required — the dashboard is unreachable without it
+    CLOUDFLARE_WORKER_URL        the LibBee Worker (shared)
+    CLOUDFLARE_WORKER_TOKEN      must equal the Worker's ANALYTICS_TOKEN
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import secrets
+import time
+from typing import Any, Dict, Optional
+
+from fastapi import APIRouter, HTTPException, Request, Response
+from fastapi.responses import HTMLResponse, RedirectResponse
+from pydantic import BaseModel
+
+router = APIRouter()
+
+_START_TIME = time.time()
+
+ADMIN_PASSWORD = (os.environ.get("RESEARCHBEE_ADMIN_PASSWORD") or "").strip()
+WORKER_URL = (os.environ.get("CLOUDFLARE_WORKER_URL") or "").strip().strip('"').strip("'").rstrip("/")
+WORKER_TOKEN = (os.environ.get("CLOUDFLARE_WORKER_TOKEN") or "").strip()
+
+_SESSIONS_FILE = "/tmp/researchbee_admin_sessions.json"
+
+admin_sessions: Dict[str, float] = {}
+
+# Failed logins per IP. Five misses inside the window locks that IP out
+# for fifteen minutes — enough to make guessing a shared password
+# impractical without locking out a librarian who fat-fingers it twice.
+_login_attempts: Dict[str, list] = {}
+_MAX_LOGIN_ATTEMPTS = 5
+_LOCKOUT_WINDOW = 300
+_LOCKOUT_DURATION = 900
+
+
+# ── Sessions ──────────────────────────────────────────────────────────
+
+def _load_sessions() -> None:
+    try:
+        with open(_SESSIONS_FILE, "r", encoding="utf-8") as f:
+            data = json.load(f)
+        now = time.time()
+        admin_sessions.update({
+            t: exp for t, exp in data.items()
+            if isinstance(exp, (int, float)) and exp > now
+        })
+    except Exception:
+        pass
+
+
+def _save_sessions() -> None:
+    try:
+        now = time.time()
+        with open(_SESSIONS_FILE, "w", encoding="utf-8") as f:
+            json.dump({t: e for t, e in admin_sessions.items() if e > now}, f)
+    except Exception:
+        pass
+
+
+def _cleanup_sessions() -> None:
+    now = time.time()
+    for t in [t for t, exp in admin_sessions.items() if exp <= now]:
+        admin_sessions.pop(t, None)
+    _load_sessions()
+
+
+def _has_session(request: Request) -> bool:
+    _cleanup_sessions()
+    token = request.cookies.get("rb_admin_session")
+    return bool(token and token in admin_sessions)
+
+
+def _require_session(request: Request) -> None:
+    if not _has_session(request):
+        raise HTTPException(status_code=401, detail="Not authenticated")
+
+
+class LoginRequest(BaseModel):
+    password: str
+
+
+# ── Auth ──────────────────────────────────────────────────────────────
+
+@router.post("/auth")
+async def admin_auth(req: LoginRequest, request: Request):
+    client_ip = request.client.host if request.client else "unknown"
+    now = time.time()
+
+    attempts = [t for t in _login_attempts.get(client_ip, []) if now - t < _LOCKOUT_WINDOW]
+    if len(attempts) >= _MAX_LOGIN_ATTEMPTS:
+        wait = int(_LOCKOUT_DURATION - (now - attempts[0]))
+        raise HTTPException(429, f"Too many failed attempts. Try again in {wait} seconds.")
+
+    if not ADMIN_PASSWORD or not secrets.compare_digest(
+        req.password.encode(), ADMIN_PASSWORD.encode()
+    ):
+        attempts.append(now)
+        _login_attempts[client_ip] = attempts
+        remaining = _MAX_LOGIN_ATTEMPTS - len(attempts)
+        if not ADMIN_PASSWORD:
+            detail = "No admin password is set on this Space. Add RESEARCHBEE_ADMIN_PASSWORD to the Space secrets."
+        elif remaining <= 0:
+            detail = f"Too many failed attempts. Try again in {_LOCKOUT_DURATION // 60} minutes."
+        else:
+            detail = f"Incorrect password. {remaining} attempt{'s' if remaining != 1 else ''} left before lockout."
+        raise HTTPException(401, detail)
+
+    _login_attempts.pop(client_ip, None)
+    token = secrets.token_hex(32)
+    admin_sessions[token] = now + 86400
+    _save_sessions()
+    resp = Response(content='{"status":"ok"}', media_type="application/json")
+    resp.set_cookie("rb_admin_session", token, httponly=True,
+                    max_age=86400, samesite="lax", secure=True)
+    return resp
+
+
+@router.get("/logout")
+async def admin_logout(request: Request):
+    token = request.cookies.get("rb_admin_session")
+    if token:
+        admin_sessions.pop(token, None)
+    _save_sessions()
+    resp = RedirectResponse(url="/admin/login")
+    resp.delete_cookie("rb_admin_session")
+    return resp
+
+
+# ── Worker proxies ────────────────────────────────────────────────────
+# The Worker URL and token stay server-side. The browser only ever talks
+# to this Space, so neither secret reaches the page.
+
+async def _worker_get(path: str, params: Optional[dict] = None):
+    if not WORKER_URL:
+        raise HTTPException(503, "CLOUDFLARE_WORKER_URL is not set on this Space")
+    import httpx
+    headers = {"Authorization": f"Bearer {WORKER_TOKEN}"} if WORKER_TOKEN else {}
+    try:
+        async with httpx.AsyncClient(timeout=15) as client:
+            r = await client.get(f"{WORKER_URL}{path}", params=params or {}, headers=headers)
+            return r.json()
+    except HTTPException:
+        raise
+    except Exception as exc:
+        raise HTTPException(502, f"Worker unreachable: {type(exc).__name__}: {exc}")
+
+
+@router.get("/usage")
+async def admin_usage(request: Request, days: int = 30):
+    _require_session(request)
+    return await _worker_get("/rb/usage", {"days": max(0, min(days, 3650))})
+
+
+@router.get("/canary-status")
+async def admin_canary_status(request: Request):
+    _require_session(request)
+    return await _worker_get("/rb/canary/status")
+
+
+@router.get("/local-usage")
+async def admin_local_usage(request: Request):
+    """Counts held in this container since it last started. Useful when
+    the Worker is unreachable, and as a cross-check that counting is
+    actually happening before blaming the Worker."""
+    _require_session(request)
+    from usage import usage
+    return usage.snapshot()
+
+
+# ── Pull endpoints (called by the Cloudflare Worker, not by a browser) ─
+# Authenticated with the shared token rather than an admin session,
+# because the caller is a machine. This is the direction that works:
+# Cloudflare can reach this Space even though this Space cannot reach
+# Cloudflare.
+
+def _require_worker_token(request: Request) -> None:
+    if not WORKER_TOKEN:
+        raise HTTPException(503, "CLOUDFLARE_WORKER_TOKEN is not set on this Space")
+    supplied = request.headers.get("authorization", "")
+    if not secrets.compare_digest(supplied, f"Bearer {WORKER_TOKEN}"):
+        raise HTTPException(401, "Unauthorized")
+
+
+def _worker_or_session(request: Request) -> None:
+    """Accept either the Worker's token or a signed-in admin, so the same
+    endpoints back both the cron pull and the dashboard's manual send."""
+    if _has_session(request):
+        return
+    _require_worker_token(request)
+
+
+@router.get("/usage-export")
+async def usage_export(request: Request):
+    """Hand over accumulated counts. Counts are held, not deleted, until
+    the caller confirms with /usage-ack — so a caller that reads them and
+    then fails to store them cannot destroy them."""
+    _worker_or_session(request)
+    from usage import usage
+    return usage.export_snapshot()
+
+
+@router.post("/usage-ack")
+async def usage_ack(request: Request, snapshot_id: str):
+    """Confirm a snapshot reached storage. Only now is it discarded."""
+    _worker_or_session(request)
+    from usage import usage
+    return {"acknowledged": usage.ack_snapshot(snapshot_id)}
+
+
+@router.get("/worker-probe")
+async def admin_worker_probe(request: Request):
+    """Is this Space's route to the Worker broken, or just slow?
+
+    HF Spaces lose connectivity to workers.dev for stretches at a time —
+    LibBee hit the same thing and moved its analytics writes into the
+    browser because of it. Three attempts with timings tell you which
+    case you are in: three identical failures is a blocked route, a mix
+    is the intermittent case, which the flush buffer already absorbs.
+    """
+    _require_session(request)
+    import httpx, time as _t
+    if not WORKER_URL:
+        return {"error": "CLOUDFLARE_WORKER_URL is not set on this Space"}
+    attempts = []
+    for i in range(3):
+        t0 = _t.time()
+        try:
+            async with httpx.AsyncClient(timeout=20) as client:
+                r = await client.get(f"{WORKER_URL}/version")
+            attempts.append({"attempt": i + 1, "ok": r.status_code == 200,
+                             "status": r.status_code,
+                             "ms": int((_t.time() - t0) * 1000)})
+        except Exception as exc:
+            attempts.append({"attempt": i + 1, "ok": False,
+                             "error": f"{type(exc).__name__}: {exc}"[:160],
+                             "ms": int((_t.time() - t0) * 1000)})
+    ok = sum(1 for a in attempts if a.get("ok"))
+    return {
+        "worker_url": WORKER_URL,
+        "attempts": attempts,
+        "verdict": ("reachable" if ok == 3 else
+                    "intermittent" if ok else "unreachable from this Space"),
+        "note": ("Counts are buffered and retried every 60 seconds, so patchy "
+                 "egress delays the dashboard without losing data. Dashboard "
+                 "reads fall back to your browser, which usually reaches the "
+                 "Worker even when this Space cannot."),
+    }
+
+
+@router.get("/status")
+async def admin_status(request: Request):
+    _require_session(request)
+    import app as rb_app
+
+    out: Dict[str, Any] = {
+        "uptime_seconds": int(time.time() - _START_TIME),
+        "worker_url_configured": bool(WORKER_URL),
+        "worker_token_configured": bool(WORKER_TOKEN),
+        "admin_password_set": bool(ADMIN_PASSWORD),
+        "openai_key_set": bool(os.environ.get("OPENAI_API_KEY")),
+    }
+    try:
+        out["health"] = await rb_app.health()
+    except Exception as exc:
+        out["health"] = {"error": f"{type(exc).__name__}: {exc}"}
+    return out
+
+
+@router.post("/reload-khazna")
+async def admin_reload_khazna(request: Request):
+    """Session-protected wrapper around the existing reload endpoint, so
+    the button on this page works without exposing an unauthenticated
+    admin path."""
+    _require_session(request)
+    import asyncio
+    from app import khazna_index
+    ok = await asyncio.to_thread(khazna_index.reload)
+    return {"reloaded": ok, "khazna": khazna_index.status}
+
+
+# ── Login page ────────────────────────────────────────────────────────
+
+@router.get("/login", response_class=HTMLResponse)
+async def admin_login_page(request: Request):
+    if _has_session(request):
+        return RedirectResponse(url="/admin")
+    return HTMLResponse("""<!DOCTYPE html>
+<html><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
+<title>ResearchBee — Admin</title>
+<style>
+*{box-sizing:border-box;margin:0;padding:0}
+body{font-family:'Segoe UI',system-ui,sans-serif;background:#f0f4f8;display:flex;justify-content:center;align-items:center;min-height:100vh}
+.box{background:#fff;border-radius:16px;padding:40px;width:360px;box-shadow:0 4px 24px rgba(0,0,0,.1);text-align:center}
+h1{color:#003366;font-size:1.3rem;margin-bottom:6px}
+.sub{color:#6b7280;font-size:.84rem;margin-bottom:24px}
+input{width:100%;padding:12px 14px;border:1.5px solid #d1d5db;border-radius:10px;font-size:.92rem;margin-bottom:14px}
+input:focus{outline:none;border-color:#003366}
+.btn{width:100%;padding:12px;background:#003366;color:#fff;border:none;border-radius:10px;font-weight:700;font-size:.92rem;cursor:pointer}
+.btn:hover{background:#004488}
+.err{color:#dc2626;font-size:.82rem;margin-bottom:10px;display:none}
+</style></head><body>
+<div class="box">
+  <h1>🐝 ResearchBee Admin</h1>
+  <div class="sub">KU Library publishing assistant</div>
+  <div class="err" id="err"></div>
+  <form onsubmit="return go(event)">
+    <input type="password" id="pw" placeholder="Admin password" autofocus>
+    <button type="submit" class="btn">Sign in</button>
+  </form>
+</div>
+<script>
+async function go(e){
+  e.preventDefault();
+  const r=await fetch('/admin/auth',{method:'POST',headers:{'Content-Type':'application/json'},
+    body:JSON.stringify({password:document.getElementById('pw').value})});
+  if(r.ok){window.location.href='/admin';return false;}
+  const d=await r.json().catch(()=>({}));
+  const el=document.getElementById('err');
+  el.textContent=d.detail||'Incorrect password.';el.style.display='block';
+  document.getElementById('pw').value='';document.getElementById('pw').focus();
+  return false;
+}
+</script></body></html>""")
+
+
+# ── Dashboard ─────────────────────────────────────────────────────────
+
+SURFACE_LABELS = {
+    "journal_subject":    "Find a journal — browse by subject",
+    "journal_manuscript": "Find a journal — analyse manuscript",
+    "licence_deposit":    "Share your article (deposit rights)",
+    "data_repository":    "Deposit your data (repository finder)",
+    "cover_letter":       "Cover letter generator",
+    "chat":               "Chat assistant",
+}
+
+GAP_LABELS = {
+    "scimago_miss":      "Subject not found in the journal index",
+    "subject_unrecognised": "Subject did not map to any SCImago category",
+    # Model named a category that does not exist. High counts here mean the
+    # vocabulary is missing terms researchers actually use (e.g. criminology).
+    "category_invented": "Subject has no category in Scopus/SCImago",
+    # Answered, but with the wrong kind of answer: no category matched, so the
+    # list is the whole subject AREA ranked by h-index. A high count here means
+    # the vocabulary is missing terms researchers actually use.
+    "area_fallback":     "Showed a broad subject area, not a specific category",
+    "oaworks_404":       "No permissions record for that article",
+    "oaworks_unconfirmed": "Permissions record found but policy not confirmed",
+    "doi_unparseable":   "Could not read a DOI from what was entered",
+    "khazna_no_doi":     "No DOI supplied, so Khazna could not be checked",
+    "khazna_unavailable": "Khazna index not loaded",
+    "no_apc_agreement":  "No KU APC agreement for that publisher",
+    "no_repository_match": "No repository matched the dataset",
+}
+
+
+@router.get("", response_class=HTMLResponse)
+async def admin_dashboard(request: Request):
+    if not _has_session(request):
+        return RedirectResponse(url="/admin/login")
+
+    page = """<!DOCTYPE html>
+<html><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
+<title>ResearchBee — Admin</title>
+<style>
+*{box-sizing:border-box;margin:0;padding:0}
+body{font-family:'Segoe UI',system-ui,sans-serif;background:#f0f4f8;color:#1a1a2e}
+.topbar{background:#003366;color:#fff;padding:12px 20px;display:flex;align-items:center;justify-content:space-between}
+.topbar h1{font-size:1.1rem;font-weight:700}
+.topbar a{color:#C8A951;font-size:.82rem;text-decoration:none;padding:5px 12px;border:1px solid #C8A951;border-radius:8px}
+.main{max-width:900px;margin:24px auto;padding:0 16px;display:flex;flex-direction:column;gap:20px}
+.card{background:#fff;border-radius:14px;padding:20px;box-shadow:0 2px 12px rgba(0,0,0,.07)}
+.card h2{font-size:1rem;font-weight:700;color:#003366;margin-bottom:6px}
+.card .hint{font-size:.78rem;color:#6b7280;margin-bottom:14px;line-height:1.5}
+.btn{padding:9px 20px;background:#003366;color:#fff;border:none;border-radius:8px;font-weight:700;cursor:pointer;font-size:.88rem}
+.btn:hover{background:#004488}
+.btn-ghost{background:#fff;color:#003366;border:1.5px solid #d1d5db}
+.btn-ghost:hover{background:#f8fafc}
+.stat-grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(140px,1fr));gap:12px}
+.stat{background:#f8fafc;border-radius:10px;padding:14px;text-align:center;border:1px solid #e5e7eb}
+.stat .val{font-size:1.6rem;font-weight:800;color:#003366}
+.stat .lbl{font-size:.74rem;color:#6b7280;margin-top:2px}
+table{width:100%;border-collapse:collapse;font-size:.84rem}
+th{text-align:left;font-size:.72rem;text-transform:uppercase;letter-spacing:.05em;color:#6b7280;border-bottom:1px solid #e5e7eb;padding:7px 8px}
+td{padding:7px 8px;border-bottom:1px solid #f1f5f9}
+tr:last-child td{border-bottom:none}
+.bar{height:8px;background:#e5e7eb;border-radius:4px;overflow:hidden;min-width:60px}
+.bar span{display:block;height:100%;background:#003366}
+.pill{display:inline-block;padding:2px 9px;border-radius:20px;font-size:.72rem;font-weight:700}
+.pill-ok{background:#dcfce7;color:#15803d}
+.pill-bad{background:#fee2e2;color:#b91c1c}
+.pill-warn{background:#fef3c7;color:#92400e}
+.range{display:flex;gap:6px;margin-bottom:12px;flex-wrap:wrap}
+.range button{padding:4px 12px;border:1.5px solid #d1d5db;background:#fff;border-radius:20px;font-size:.78rem;cursor:pointer;color:#6b7280}
+.range button.on{border-color:#003366;background:#003366;color:#fff;font-weight:700}
+.msg{font-size:.82rem;color:#6b7280;padding:8px 0}
+.err{font-size:.82rem;color:#b91c1c;padding:8px 0}
+.mono{font-family:ui-monospace,SFMono-Regular,Menlo,monospace;font-size:.76rem}
+</style></head><body>
+<div class="topbar">
+  <h1>🐝 ResearchBee — Admin</h1>
+  <a href="/admin/logout">Sign out</a>
+</div>
+<div class="main">
+
+  <div class="card">
+    <h2>Tool usage</h2>
+    <div class="hint">How many times each tool was used. Counts only — ResearchBee never
+      stores what anyone typed, so there is no question log here by design.
+      Keep-alive traffic is excluded.</div>
+    <div class="range">
+      <button data-days="7">7 days</button>
+      <button data-days="30" class="on">30 days</button>
+      <button data-days="90">90 days</button>
+      <button data-days="0">All time</button>
+    </div>
+    <div id="usage"><div class="msg">Loading…</div></div>
+  </div>
+
+  <div class="card">
+    <h2>Questions we could not answer</h2>
+    <div class="hint">Requests that completed but found nothing to answer with — a subject
+      missing from the journal index, an article with no permissions record. Each one is a
+      candidate for improving a data source.</div>
+    <div id="gaps"><div class="msg">Loading…</div></div>
+  </div>
+
+  <div class="card">
+    <h2>Health checks</h2>
+    <div class="hint">Every six hours the Worker asks ResearchBee a real question, rotating
+      through the four tools. This keeps the Space awake and proves a different pipeline
+      still works each time.</div>
+    <div id="canary"><div class="msg">Loading…</div></div>
+  </div>
+
+  <div class="card">
+    <h2>Service status</h2>
+    <div class="hint">Data sources this Space is currently using, and its connection to the
+      analytics Worker.</div>
+    <div id="status"><div class="msg">Loading…</div></div>
+    <div style="margin-top:14px;display:flex;gap:8px;flex-wrap:wrap">
+      <button class="btn" id="reload-khazna">Reload the Khazna index</button>
+      <button class="btn btn-ghost" id="show-local">Show counts held in this container</button>
+      <button class="btn btn-ghost" id="probe-worker">Test the connection to Cloudflare</button>
+      <button class="btn btn-ghost" id="send-now">Send counts to Cloudflare now</button>
+    </div>
+    <div id="action-out"></div>
+  </div>
+
+</div>
+<script>
+// The dashboard tries the Space's server-side proxy first, then falls back to
+// calling the Worker directly from this browser. HF Spaces lose egress to
+// workers.dev for stretches at a time, while the admin's own browser reaches
+// it fine — LibBee hit exactly this and solved it the same way. This page is
+// served only to an authenticated session, so the token is not public.
+const WORKER_URL   = "__WORKER_URL__";
+const WORKER_TOKEN = "__WORKER_TOKEN__";
+const SURFACE = """ + json.dumps(SURFACE_LABELS) + """;
+const GAP = """ + json.dumps(GAP_LABELS) + """;
+const esc = s => String(s??'').replace(/&/g,'&amp;').replace(/</g,'&lt;').replace(/>/g,'&gt;');
+const num = n => Number(n||0).toLocaleString();
+
+let usedFallback = false;
+
+async function getDirect(workerPath){
+  if(!WORKER_URL || !WORKER_TOKEN) throw new Error('no direct route configured');
+  const r = await fetch(WORKER_URL + workerPath, {
+    headers: { Authorization: 'Bearer ' + WORKER_TOKEN }
+  });
+  if(!r.ok) throw new Error('Worker returned HTTP ' + r.status);
+  usedFallback = true;
+  return r.json();
+}
+
+// workerPath is the same data read straight from the Worker. Passing it makes
+// this call survive the Space losing its outbound route to Cloudflare.
+async function get(path, workerPath){
+  try {
+    const r = await fetch(path);
+    if(r.status===401){window.location.href='/admin/login';throw new Error('unauthenticated');}
+    const d = await r.json().catch(()=>({}));
+    if(!r.ok) throw new Error(d.detail || ('HTTP '+r.status));
+    return d;
+  } catch(e) {
+    if(String(e.message).includes('unauthenticated') || !workerPath) throw e;
+    // Space could not reach the Worker; try from here instead.
+    return getDirect(workerPath);
+  }
+}
+
+function fallbackNotice(){
+  return usedFallback
+    ? `<div class="hint" style="color:#92400e;background:#fef3c7;border-radius:8px;padding:7px 10px;margin-bottom:10px">
+         Read directly from Cloudflare — this Space could not reach the Worker just now.
+         Counts still being collected are buffered and will be sent when the route recovers.
+       </div>`
+    : '';
+}
+
+// ── Usage ──────────────────────────────────────────────────────
+let days = 30;
+
+async function loadUsage(){
+  const el = document.getElementById('usage');
+  const gapEl = document.getElementById('gaps');
+  el.innerHTML = '<div class="msg">Loading…</div>';
+  try {
+    const d = await get('/admin/usage?days='+days, '/usage?days='+days);
+    const t = d.totals || {};
+    const rows = (d.by_surface || []);
+    const max = Math.max(1, ...rows.map(r => r.total || 0));
+
+    el.innerHTML = fallbackNotice() +
+      `<div class="stat-grid" style="margin-bottom:16px">
+         <div class="stat"><div class="val">${num(t.total)}</div><div class="lbl">Times used</div></div>
+         <div class="stat"><div class="val">${num(t.today)}</div><div class="lbl">Today</div></div>
+         <div class="stat"><div class="val">${num(t.miss)}</div><div class="lbl">Unanswered</div></div>
+         <div class="stat"><div class="val">${num(t.errors)}</div><div class="lbl">Errors</div></div>
+       </div>` +
+      (rows.length
+        ? `<table><thead><tr><th>Tool</th><th>Times used</th><th style="width:35%"></th><th>Unanswered</th><th>Errors</th></tr></thead><tbody>` +
+          rows.map(r => `<tr>
+              <td>${esc(SURFACE[r.surface] || r.surface)}</td>
+              <td><strong>${num(r.total)}</strong></td>
+              <td><div class="bar"><span style="width:${Math.round((r.total/max)*100)}%"></span></div></td>
+              <td>${num(r.miss)}</td>
+              <td>${num(r.errors)}</td>
+            </tr>`).join('') +
+          `</tbody></table>`
+        : `<div class="msg">Nothing recorded in this period yet. Use one of the tools, wait a minute for the next flush, then reload.</div>`);
+
+    const gaps = d.gaps || [];
+    gapEl.innerHTML = gaps.length
+      ? `<table><thead><tr><th>What was missing</th><th>Tool</th><th>Times</th></tr></thead><tbody>` +
+        gaps.map(g => `<tr>
+            <td>${esc(GAP[g.gap_code] || g.gap_code)}</td>
+            <td>${esc(SURFACE[g.surface] || g.surface)}</td>
+            <td><strong>${num(g.c)}</strong></td>
+          </tr>`).join('') + `</tbody></table>`
+      : `<div class="msg">Nothing flagged in this period.</div>`;
+
+  } catch(e) {
+    el.innerHTML = '<div class="err">Could not load usage: '+esc(e.message)+'</div>';
+    gapEl.innerHTML = '';
+  }
+}
+
+document.querySelectorAll('.range button').forEach(b => {
+  b.addEventListener('click', () => {
+    document.querySelectorAll('.range button').forEach(x => x.classList.remove('on'));
+    b.classList.add('on');
+    days = parseInt(b.dataset.days, 10);
+    loadUsage();
+  });
+});
+
+// ── Canary ─────────────────────────────────────────────────────
+async function loadCanary(){
+  const el = document.getElementById('canary');
+  try {
+    const d = await get('/admin/canary-status', '/canary/status');
+    const runs = d.runs || [];
+    el.innerHTML =
+      `<div class="msg" style="padding-top:0">${esc(d.summary||'')}</div>` +
+      (runs.length
+        ? `<table><thead><tr><th>When (UTC)</th><th>Tool checked</th><th>Result</th><th>Took</th><th>Detail</th></tr></thead><tbody>` +
+          runs.map(r => `<tr>
+              <td class="mono">${esc(r.run_at)}</td>
+              <td>${esc(r.slot||'')}</td>
+              <td><span class="pill ${r.ok?'pill-ok':'pill-bad'}">${r.ok?'Passed':'Failed'}</span></td>
+              <td>${r.latency_ms!=null?(r.latency_ms/1000).toFixed(1)+'s':''}</td>
+              <td class="mono">${esc(r.error || r.detail || '')}</td>
+            </tr>`).join('') + `</tbody></table>`
+        : '');
+  } catch(e) {
+    el.innerHTML = '<div class="err">Could not load health checks: '+esc(e.message)+'</div>';
+  }
+}
+
+// ── Status ─────────────────────────────────────────────────────
+function yesNo(v){ return `<span class="pill ${v?'pill-ok':'pill-warn'}">${v?'Yes':'No'}</span>`; }
+
+async function loadStatus(){
+  const el = document.getElementById('status');
+  try {
+    const d = await get('/admin/status');
+    const h = d.health || {};
+    const src = h.sources || {};
+    const k = src.khazna || {};
+    const hrs = Math.floor((d.uptime_seconds||0)/3600);
+    const mins = Math.floor(((d.uptime_seconds||0)%3600)/60);
+    el.innerHTML = `<table><tbody>
+      <tr><td>Version</td><td>${esc(h.version||'unknown')}</td></tr>
+      <tr><td>Running for</td><td>${hrs}h ${mins}m</td></tr>
+      <tr><td>Khazna index loaded</td><td>${yesNo(k.available)}${k.records?' &middot; '+num(k.records)+' records':''}</td></tr>
+      <tr><td>KU institution ID set for permissions</td><td>${yesNo(src.oaworks && src.oaworks.ku_ror_set)}</td></tr>
+      <tr><td>Repository registry</td><td>${num((src.repository_registry||{}).entries)} entries</td></tr>
+      <tr><td>Analytics Worker configured</td><td>${yesNo(d.worker_url_configured)}</td></tr>
+      <tr><td>Worker token configured</td><td>${yesNo(d.worker_token_configured)}</td></tr>
+      <tr><td>Language model key set</td><td>${yesNo(d.openai_key_set)}</td></tr>
+    </tbody></table>`;
+  } catch(e) {
+    el.innerHTML = '<div class="err">Could not load status: '+esc(e.message)+'</div>';
+  }
+}
+
+// ── Actions ────────────────────────────────────────────────────
+document.getElementById('reload-khazna').addEventListener('click', async (ev) => {
+  const out = document.getElementById('action-out');
+  ev.target.disabled = true;
+  out.innerHTML = '<div class="msg">Reloading…</div>';
+  try {
+    const r = await fetch('/admin/reload-khazna', {method:'POST'});
+    const d = await r.json();
+    out.innerHTML = d.reloaded
+      ? `<div class="msg">Reloaded. ${num((d.khazna||{}).records)} records now in the index.</div>`
+      : `<div class="err">Reload failed. Check the Space logs for the line beginning "Khazna index download failed".</div>`;
+    loadStatus();
+  } catch(e) {
+    out.innerHTML = '<div class="err">Reload failed: '+esc(e.message)+'</div>';
+  }
+  ev.target.disabled = false;
+});
+
+document.getElementById('show-local').addEventListener('click', async () => {
+  const out = document.getElementById('action-out');
+  try {
+    const d = await get('/admin/local-usage');
+    const rows = Object.entries(d.by_surface||{});
+    out.innerHTML =
+      `<div class="msg">${esc(d.note)} Counting since ${esc(d.since)}. `+
+      `${d.pending_deltas} delta(s) waiting to be sent.</div>` +
+      (rows.length
+        ? `<table><thead><tr><th>Tool</th><th>Used</th><th>Unanswered</th><th>Errors</th></tr></thead><tbody>` +
+          rows.map(([k,v]) => `<tr><td>${esc(SURFACE[k]||k)}</td><td>${num(v.total)}</td><td>${num(v.miss)}</td><td>${num(v.error)}</td></tr>`).join('') +
+          `</tbody></table>`
+        : '<div class="msg">Nothing counted since this container started.</div>') +
+      ((d.recent_flushes||[]).length
+        ? `<div class="hint" style="margin-top:10px">Last send to the Worker: <span class="mono">${esc(d.recent_flushes[0].outcome)} — ${esc(d.recent_flushes[0].detail)}</span></div>`
+        : '');
+  } catch(e) {
+    out.innerHTML = '<div class="err">'+esc(e.message)+'</div>';
+  }
+});
+
+// This Space cannot reach Cloudflare, but your browser can — the same route
+// the dashboard reads through. So a manual send goes: Space -> browser -> Worker.
+// Worth doing before a planned redeploy, since counts sit in memory until the
+// Worker's next 6-hourly pull.
+document.getElementById('send-now').addEventListener('click', async (ev) => {
+  const out = document.getElementById('action-out');
+  ev.target.disabled = true;
+  out.innerHTML = '<div class="msg">Collecting counts…</div>';
+  try {
+    const snap = await get('/admin/usage-export');
+    if (!snap.snapshot_id || !snap.deltas.length) {
+      out.innerHTML = '<div class="msg">Nothing waiting to be sent.</div>';
+      ev.target.disabled = false; return;
+    }
+    out.innerHTML = `<div class="msg">Sending ${snap.deltas.length} item(s) via this browser…</div>`;
+    const r = await fetch(WORKER_URL + '/log', {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json', Authorization: 'Bearer ' + WORKER_TOKEN },
+      body: JSON.stringify({ deltas: snap.deltas })
+    });
+    const d = await r.json();
+    if (r.ok && d.logged) {
+      await fetch('/admin/usage-ack?snapshot_id=' + encodeURIComponent(snap.snapshot_id), {method:'POST'});
+      out.innerHTML = `<div class="msg">Sent. ${d.accepted} item(s) stored in Cloudflare.</div>`;
+      loadUsage();
+    } else {
+      out.innerHTML = `<div class="err">Cloudflare refused the counts: ${esc(d.warning||('HTTP '+r.status))}.
+        They are held and will be collected on the next scheduled pull.</div>`;
+    }
+  } catch(e) {
+    out.innerHTML = `<div class="err">Could not send: ${esc(e.message)}.
+      Counts are held and will be collected on the next scheduled pull.</div>`;
+  }
+  ev.target.disabled = false;
+});
+
+document.getElementById('probe-worker').addEventListener('click', async (ev) => {
+  const out = document.getElementById('action-out');
+  ev.target.disabled = true;
+  out.innerHTML = '<div class="msg">Testing three times…</div>';
+  try {
+    const d = await get('/admin/worker-probe');
+    const rows = (d.attempts||[]).map(a =>
+      `<tr><td>Attempt ${a.attempt}</td>
+           <td><span class="pill ${a.ok?'pill-ok':'pill-bad'}">${a.ok?'Reached':'Failed'}</span></td>
+           <td>${a.ms} ms</td><td class="mono">${esc(a.error||'')}</td></tr>`).join('');
+    out.innerHTML =
+      `<div class="msg">Verdict: <strong>${esc(d.verdict)}</strong></div>` +
+      `<table><tbody>${rows}</tbody></table>` +
+      `<div class="hint" style="margin-top:8px">${esc(d.note||'')}</div>`;
+  } catch(e) {
+    out.innerHTML = '<div class="err">'+esc(e.message)+'</div>';
+  }
+  ev.target.disabled = false;
+});
+
+loadUsage(); loadCanary(); loadStatus();
+</script></body></html>"""
+    return HTMLResponse(
+        page.replace("__WORKER_URL__", WORKER_URL)
+            .replace("__WORKER_TOKEN__", WORKER_TOKEN)
     )

app.py

@@ -1,5 +1,5 @@
 """
-ResearchBee — FastAPI backend v1.4.0
+ResearchBee — FastAPI backend v1.5.0 (Release A correctness and safety gate)
 
 Changes from v1.3 (verified-sources release):
 - /api/check-license is now DOI-first and GROUNDED. Policy facts come from the
@@ -23,6 +23,7 @@ import json
 import re
 import asyncio
 import logging
+import secrets
 import time
 from collections import defaultdict
 from contextlib import asynccontextmanager
@@ -217,6 +218,34 @@ def get_client_ip(request: Request) -> str:
         return forwarded.split(",")[0].strip()
     return request.client.host if request.client else "unknown"
 
+
+KHAZNA_RELOAD_HEADER = "X-ResearchBee-Reload-Token"
+
+
+def require_khazna_reload_auth(request: Request) -> None:
+    """Require the Space secret before allowing a Khazna index reload."""
+    expected = os.environ.get("KHAZNA_RELOAD_TOKEN", "").strip()
+    if not expected:
+        raise HTTPException(
+            503,
+            "Khazna reload authentication is not configured.",
+        )
+
+    supplied = request.headers.get(KHAZNA_RELOAD_HEADER, "").strip()
+    if not supplied:
+        auth = request.headers.get("Authorization", "").strip()
+        if auth.lower().startswith("bearer "):
+            supplied = auth[7:].strip()
+
+    if not supplied:
+        raise HTTPException(
+            401,
+            "Khazna reload token required.",
+            headers={"WWW-Authenticate": "Bearer"},
+        )
+    if not secrets.compare_digest(supplied, expected):
+        raise HTTPException(403, "Invalid Khazna reload token.")
+
 OPF_BASE = "https://v2.sherpa.ac.uk/cgi/retrieve"  # Open Policy Finder (Jisc)
 
 
@@ -447,7 +476,7 @@ async def lifespan(app: FastAPI):
 
 
 # ── App ────────────────────────────────────────────────────────────────────
-app = FastAPI(title="ResearchBee API", version="1.4.1", lifespan=lifespan)
+app = FastAPI(title="ResearchBee API", version="1.5.0", lifespan=lifespan)
 
 app.add_middleware(
     CORSMiddleware,
@@ -465,7 +494,7 @@ app.add_middleware(
         "http://127.0.0.1:3000",
     ],
     allow_methods=["GET", "POST", "OPTIONS"],
-    allow_headers=["Content-Type"],
+    allow_headers=["Content-Type", "Authorization", "X-ResearchBee-Reload-Token"],
 )
 
 # Registered AFTER the CORS middleware on purpose. Starlette runs middleware
@@ -669,6 +698,106 @@ def enrich_extended(extended: list) -> list:
         result.append(j)
     return result
 
+_POLICY_ACTION_RE = re.compile(
+    r"(?:self[- ]?archiv|embargo|deposit|repository|licen[cs]e|preprint|postprint|"
+    r"accepted manuscript|version of record|green oa|\baam\b)",
+    re.IGNORECASE,
+)
+
+
+def enforce_journal_policy_guard(parsed: dict) -> dict:
+    """
+    Make the journal-finder workflow incapable of returning LLM policy facts.
+
+    The model may rank and explain journals, but every policy-bearing field is
+    replaced with a canonical, three-valued "unknown" response. This guard is
+    deliberately server-side so a future prompt regression cannot reintroduce
+    apparent permissions, embargoes, or licences.
+    """
+    if not isinstance(parsed, dict):
+        return {}
+
+    for journal in parsed.get("journals") or []:
+        if not isinstance(journal, dict):
+            continue
+        green = oaworks.not_confirmed(
+            journal.get("name", ""),
+            journal.get("issn", ""),
+            reason="Journal Finder evaluates fit only; no article-level permission lookup was performed",
+        )
+        green["repository_action_note"] = (
+            "Use ResearchBee License Checking with the article DOI before depositing any manuscript version."
+        )
+        green["evidence_note"] = (
+            "No article-level permission lookup is performed by Journal Finder. "
+            "Policy values from the language model are discarded by the server."
+        )
+        green["risk_flag"] = "Deposit rights not checked — verify with the article DOI."
+        journal["green_oa"] = green
+        journal["oa_compliance_note"] = (
+            "Journal Finder does not determine self-archiving rights. Use the DOI-based "
+            "License Checking tool before depositing any version."
+        )
+
+    unknown = oaworks.not_confirmed(
+        reason="Journal Finder does not perform an article-level permission lookup"
+    )
+    parsed["repository_recommendation"] = oaworks.recommendation_from(unknown)
+
+    safe_actions = []
+    for action in parsed.get("next_actions") or []:
+        action = " ".join(str(action or "").split())
+        if action and not _POLICY_ACTION_RE.search(action):
+            safe_actions.append(action[:600])
+    safe_actions.extend([
+        "Verify the journal scope, author guidelines, current fees, and indexation before submission.",
+        "Use ResearchBee License Checking with the article DOI before depositing any manuscript version.",
+    ])
+    # Stable de-duplication.
+    parsed["next_actions"] = list(dict.fromkeys(safe_actions))
+    parsed["global_notes"] = (
+        "Journal Finder evaluates publication fit and verified journal metrics only. "
+        "Self-archiving permissions, embargoes, and licences require a DOI-based check."
+    )
+    parsed["policy_source_mode"] = "not_checked_in_journal_finder"
+    return parsed
+
+
+def build_subject_journal_result(journal: dict) -> dict:
+    """Build one subject-browse row, including all DB-backed field quartiles."""
+    issn = journal.get("issn")
+    scimago = lookup_any([issn]) if issn else None
+    if not scimago:
+        scimago = _scimago_title_fallback(journal.get("title", ""))
+
+    source = scimago or journal
+    final_issn = source.get("matched_issn") or issn or ""
+    quartile = source.get("quartile") or journal.get("quartile", "")
+    h_index = source.get("h_index", journal.get("h_index"))
+    try:
+        h_index = int(h_index) if h_index not in (None, "") else None
+    except (TypeError, ValueError):
+        h_index = None
+
+    result = {
+        "name": journal.get("title", ""),
+        "issn": final_issn,
+        "quartile": quartile if quartile and quartile != "-" else None,
+        "h_index": h_index,
+        "publisher": source.get("publisher", "") or journal.get("publisher", ""),
+        "open_access": source.get("open_access", journal.get("open_access", "No")),
+        "country": source.get("country", journal.get("country", "")),
+        "categories": source.get("categories", journal.get("categories", "")),
+        "verify_links": build_verify_links(
+            journal.get("title", ""),
+            final_issn,
+            source.get("sourceid", journal.get("sourceid", "")),
+        ),
+    }
+    result.update(_index_extras(scimago))
+    result["ku_apc_covered"] = is_ku_apc_covered(result["publisher"], quartile)
+    return result
+
 
 # ── OpenAI call with retry ─────────────────────────────────────────────────
 @retry(
@@ -887,7 +1016,7 @@ async def health():
     return {
         "status": "ok",
         "service": "ResearchBee",
-        "version": "1.4.0",
+        "version": "1.5.0",
         # Restart detection for the uptime monitor. boot_ts is the stable
         # identity of this process; when it changes between two polls the
         # container was replaced, whatever the HTTP status said.
@@ -919,6 +1048,7 @@ async def reload_khazna(request: Request):
     picks up new data within seconds instead of waiting for the 23:30 refresh.
     Idempotent and harmless: it only re-reads a public dataset.
     """
+    require_khazna_reload_auth(request)
     ip = get_client_ip(request)
     if not check_rate_limit(ip, "global"):
         raise HTTPException(429, "Too many requests.")
@@ -976,6 +1106,52 @@ async def openalex_works(request: Request, source_id: str, per_page: int = 5):
         return {"works": []}
 
 
+@app.get("/api/openalex-cited-by")
+async def openalex_cited_by(request: Request, work_id: str, per_page: int = 5):
+    """Return the most-cited papers that cite one validated OpenAlex work id."""
+    ip = get_client_ip(request)
+    if not check_rate_limit(ip, "global"):
+        raise HTTPException(429, "Too many requests. Please wait a moment.")
+
+    clean_id = str(work_id or "").strip().replace("https://openalex.org/", "")
+    if not re.fullmatch(r"W\d+", clean_id):
+        raise HTTPException(400, "Invalid OpenAlex work_id")
+
+    try:
+        client = openalex._get_client()
+        response = await client.get(
+            "https://api.openalex.org/works",
+            params={
+                "filter": f"cites:{clean_id},is_retracted:false",
+                "sort": "cited_by_count:desc",
+                "per_page": min(max(per_page, 1), 8),
+                "select": "id,title,publication_year,doi,cited_by_count,authorships",
+            },
+            timeout=6.0,
+        )
+        if not response.is_success:
+            return {"works": []}
+
+        works = []
+        for work in response.json().get("results", []):
+            authors = [
+                authorship.get("author", {}).get("display_name", "")
+                for authorship in (work.get("authorships") or [])[:3]
+            ]
+            works.append({
+                "title": work.get("title", ""),
+                "year": work.get("publication_year"),
+                "doi": work.get("doi", ""),
+                "cited_by": work.get("cited_by_count", 0),
+                "authors": [author for author in authors if author],
+                "openalex_url": work.get("id", ""),
+            })
+        return {"works": works}
+    except Exception as exc:
+        logger.warning("[OpenAlex cited-by] %s", exc)
+        return {"works": []}
+
+
 # ── Trending papers by concept/topic ─────────────────────────────────────
 @app.get("/api/trending-papers")
 async def trending_papers(request: Request, concept: str = "", concept_id: str = "", per_page: int = 5):
@@ -1105,6 +1281,10 @@ async def analyze_journal(req: JournalRequest, request: Request):
     if "error" in parsed:
         raise HTTPException(500, parsed["error"])
 
+    # Release A hard guard: the language model ranks journals but may never
+    # provide self-archiving facts in this workflow, even if a prompt regresses.
+    parsed = enforce_journal_policy_guard(parsed)
+
     if isinstance(parsed.get("journals"), list):
         parsed["journals"] = await enrich_journals(parsed["journals"])
     if isinstance(parsed.get("extended_list"), list):
@@ -1166,12 +1346,14 @@ def khazna_status_for(doi: str) -> dict:
 async def _advisory_note(green: dict, li: "LicenseInput", lang: str) -> str:
     """The LLM writes prose from verified facts only. It never supplies a value."""
     facts = {
+        "permission_status":  green.get("permission_status", "unknown"),
+        "embargo_status":     green.get("embargo_status", "unknown"),
         "can_archive":        green.get("_can_archive"),
         "versions_permitted": green.get("_versions"),
         "best_version":       green.get("_best_version"),
         "locations":          green.get("_locations"),
         "embargo_months":     green.get("_embargo_months"),
-        "embargo_stated":     green.get("_embargo_months") is not None,
+        "embargo_stated":     green.get("embargo_status") != "unknown",
         "embargo_end":        green.get("embargo_end"),
         "licence":            green.get("_licence"),
         "journal_oa_type":    green.get("journal_oa_type"),
@@ -1262,48 +1444,41 @@ async def check_license(req: LicenseRequest, request: Request):
     if not li.journal_name:
         raise HTTPException(422, "Provide either an article DOI/URL or a journal name.")
 
-    client = get_client()
     issn = doi_utils.normalise_issn(li.issn) or li.issn
-    user_msg = (
-        f"{lang}Explain what this author should check regarding self-archiving. "
-        f"Return JSON only. You have NO policy database — do not state facts.\n\n"
-        f"Journal name: {li.journal_name}\nISSN: {issn or 'Not provided'}\n"
-        f"Publisher: {li.publisher or 'Not provided'}\nVersion: {li.manuscript_version}\n"
-        f"Funder: {li.funder or 'None'}\nIntended repository: {li.intended_repository}\n"
-        f"Intended licence: {li.intended_licence}\nNotes: {li.notes or 'None'}"
+    green = oaworks.not_confirmed(
+        li.journal_name,
+        issn,
+        reason="no article DOI was supplied, so no article-level permission lookup was possible",
+    )
+    green["repository_action_note"] = (
+        "Enter the article DOI for a verified permission result. Until then, do not infer "
+        "a permitted version, embargo, licence, or deposit timing from the journal name alone."
     )
-    parsed = await call_openai(client, LICENSE_SYSTEM_PROMPT, user_msg,
-                               max_tokens=1200, model=MODEL_LIGHT)
-    if "error" in parsed:
-        raise HTTPException(500, parsed["error"])
-
-    # HARD GUARD: the model has no database, so it may never claim confirmation.
-    verify = "https://openpolicyfinder.jisc.ac.uk/" + (f"search?term={issn}" if issn else "")
-    for j_ in (parsed.get("journals") or []):
-        g = j_.setdefault("green_oa", {})
-        g["policy_status"] = "Not confirmed"
-        g["risk_flag"] = ("Journal-level estimate only — not verified against a policy "
-                          "database. Enter your article's DOI for a confirmed answer.")
-        g["evidence_note"] = ("No verified permission record was retrieved. "
-                              f"Verify at {verify}")
-        g["_verify_url"] = verify
-        if not j_.get("issn"):
-            j_["issn"] = issn
 
-    if isinstance(parsed.get("journals"), list):
-        parsed["journals"] = await enrich_journals(parsed["journals"])
+    journal = {
+        "name": li.journal_name,
+        "issn": issn,
+        "publisher": li.publisher,
+        "green_oa": green,
+    }
+    enriched = await enrich_journals([journal])
+    parsed = {
+        "journals": enriched,
+        "repository_recommendation": oaworks.recommendation_from(green),
+        "next_actions": oaworks.next_actions_from(green, ""),
+        "global_notes": (
+            "No DOI was supplied. ResearchBee therefore reports permission and embargo "
+            "as unknown rather than estimating them from a journal-level model response."
+        ),
+        "khazna_status": {"checked": False, "reason": "no DOI supplied"},
+        "source_mode": "journal",
+        "khazna": KHAZNA_DEPOSIT_CARD,
+    }
 
-    rec = parsed.setdefault("repository_recommendation", {})
-    rec["manual_checks_required"] = (rec.get("manual_checks_required") or []) + [
-        "Verify the policy at Open Policy Finder before depositing."]
-    parsed["khazna_status"] = {"checked": False, "reason": "no DOI supplied"}
     # Path B never produces a verified answer. Counting it as a miss keeps the
-    # dashboard honest about how often people arrive without a DOI — if that
-    # number is high, the form needs to ask for one more insistently.
+    # dashboard honest about how often people arrive without a DOI.
     usage.usage.record_gap("licence_deposit", "khazna_no_doi")
     request.state.usage_recorded = True
-    parsed["source_mode"] = "journal"
-    parsed["khazna"] = KHAZNA_DEPOSIT_CARD
     return {"result": parsed}
 
 
@@ -1328,6 +1503,9 @@ async def find_repository(req: RepoRequest, request: Request):
     # The model selected ids and wrote fit_reasons; the SERVER supplies every
     # factual field. Unknown ids are silently dropped.
     repos = hydrate_selection(parsed.get("repositories", []))
+    parsed["deposit_strategy"] = repositories.hydrate_deposit_strategy(
+        parsed.get("deposit_strategy"), repos
+    )
 
     # Khazna always leads, and is never a model choice.
     repos = [r for r in repos if not r.get("is_khazna")]
@@ -1347,6 +1525,7 @@ async def find_repository(req: RepoRequest, request: Request):
             }
 
     parsed["repositories"] = repos
+    parsed["selection_schema_version"] = "release-a-id-only-v1"
     parsed["registry_note"] = (
         "Repositories are selected from a registry curated by KU Library. "
         "Factual details are served from verified records, not generated."
@@ -1388,6 +1567,10 @@ def subject_prompt_with_vocabulary() -> str:
 
 @app.post("/api/browse-subject")
 async def browse_subject(req: SubjectRequest, request: Request):
+    ip = get_client_ip(request)
+    if not check_rate_limit(ip, "global"):
+        raise HTTPException(429, "Too many requests. Please wait a moment and try again.")
+
     # Guard: ensure SCImago index is loaded (cold-start protection)
     from scimago_index import _subject_cache, _scimago_cache
     if not _subject_cache or not _scimago_cache:
@@ -1528,24 +1711,9 @@ async def browse_subject(req: SubjectRequest, request: Request):
             "journals": [],
         }}
 
-    enriched = []
-    for j in journals:
-        issn     = j.get("issn")
-        sourceid = j.get("sourceid", "")
-        vl = build_verify_links(j["title"], issn, sourceid)
-        q  = j.get("quartile", "")
-        enriched.append({
-            "name":        j["title"],
-            "issn":        issn or "",
-            "quartile":    q if q and q != "-" else None,
-            "h_index":     j.get("h_index"),
-            "publisher":   j.get("publisher", ""),
-            "open_access": j.get("open_access", "No"),
-            "country":     j.get("country", ""),
-            "categories":  j.get("categories", ""),
-            "verify_links": vl,
-            "ku_apc_covered": is_ku_apc_covered(j.get("publisher", ""), j.get("quartile", "")),
-        })
+    # Enrich each subject result through journals.db so the response contains
+    # every field-specific quartile, not only SCIMago's best headline quartile.
+    enriched = [build_subject_journal_result(journal) for journal in journals]
 
     msg = user_message
     if fallback_used:

asjc_classifier.py

@@ -1,216 +1,216 @@
-"""
-Abstract -> ASJC subject codes, for populating ResearchBee's recommendation filter.
-
-    from asjc_classifier import AsjcClassifier
-
-    clf = AsjcClassifier("asjc_model.joblib")           # local model only
-    clf = AsjcClassifier("asjc_model.joblib", llm=True) # + LLM adjudication
-
-    result = clf.classify(title=..., abstract=...)
-    result.codes        # ['3309', '1710'] — feed straight to recommend(asjc=...)
-    result.method       # 'linear+llm' | 'linear-topk'
-    result.candidates   # what the LLM was asked to choose between
-
-Architecture, and why it is not the obvious one
------------------------------------------------
-The obvious design is "run the linear model, escalate to an LLM when confidence
-is low". Measured on the held-out abstract set, that design cannot work: the
-top-1 probability averages 0.367 when the prediction is correct and 0.330 when
-it is wrong. There is no threshold separating them — the model is wrong often
-and unbothered about it.
-
-What the model *is* good at is narrowing. A correct code appears within:
-
-    top-1   30%
-    top-3   70%
-    top-5   95%     <- and flat thereafter
-
-So it is used as a candidate generator, not a decider. The linear model cuts 334
-codes to 5 with ~95% recall; the LLM picks among those 5. That inverts the usual
-cost argument in our favour — the prompt carries five labelled options rather
-than a 334-entry taxonomy, so the call is small, fast and hard to get wrong.
-
-Degradation without an API key is graceful: `codes` becomes the top 3, which the
-recommender applies as an OR filter. Recall is 70% rather than 95%, and the
-candidate pool is broader, but nothing breaks.
-
-Retrain after the OpenAlex harvest
-----------------------------------
-The shipped model is trained on journal titles, because that is the labelled text
-available without a network. It is applied to abstracts. `harvest_openalex.py`
-produces the matched corpus; retraining on it and re-running `asjc_eval.jsonl`
-measures the gain on the same 20 abstracts, so the upgrade is quantified rather
-than assumed.
-"""
-from __future__ import annotations
-
-import json
-import os
-import re
-from dataclasses import dataclass, field
-from pathlib import Path
-
-import joblib
-import numpy as np
-from scipy import sparse
-
-__all__ = ["AsjcClassifier", "AsjcResult"]
-
-#: Where the linear model's recall saturates. Measured, not guessed.
-DEFAULT_K = 5
-#: Codes returned when no LLM is available. top-3 = 70% recall.
-FALLBACK_K = 3
-
-
-@dataclass
-class AsjcResult:
-    codes: list[str]
-    names: list[str]
-    method: str
-    candidates: list[dict] = field(default_factory=list)
-    llm_error: str | None = None
-
-    def __bool__(self) -> bool:
-        return bool(self.codes)
-
-
-_PROMPT = """You are classifying an academic manuscript into Scopus ASJC subject areas.
-
-A statistical model has narrowed 334 possible subject areas to the {n} candidates \
-below. One or more of them is correct.
-
-Candidates:
-{candidates}
-
-Manuscript title: {title}
-
-Abstract: {abstract}
-
-Choose the 1-3 candidates that best describe where this manuscript belongs, most \
-appropriate first. Choose only from the candidate list — do not invent codes. If \
-the manuscript is genuinely interdisciplinary, return more than one; if it clearly \
-belongs to a single area, return just that one.
-
-Respond with JSON only, no other text:
-{{"codes": ["NNNN", ...], "reason": "one short sentence"}}"""
-
-
-class AsjcClassifier:
-    def __init__(
-        self,
-        model_path: str | Path = "asjc_model.joblib",
-        llm: bool = False,
-        llm_model: str | None = None,
-        api_key: str | None = None,
-    ):
-        self.bundle = joblib.load(model_path)
-        self.classes: np.ndarray = self.bundle["classes"]
-        self.names: dict[str, str] = self.bundle["names"]
-        self.parent_of: dict[str, str] = self.bundle["parent_of"]
-        self.trained_on: str = self.bundle.get("corpus", "unknown")
-
-        self._client = None
-        # Model id is configuration, not a constant — set ASJC_LLM_MODEL to
-        # whichever model you are entitled to. Verify it against your account's
-        # available models rather than trusting a default baked in here.
-        self.llm_model = llm_model or os.getenv("ASJC_LLM_MODEL", "claude-sonnet-4-5")
-        if llm:
-            key = api_key or os.getenv("ANTHROPIC_API_KEY")
-            if key:
-                try:
-                    import anthropic
-                    self._client = anthropic.Anthropic(api_key=key)
-                except ImportError:
-                    pass   # falls back to top-k; reported in AsjcResult.method
-
-    # -- stage 1: narrow ----------------------------------------------------
-    def candidates(self, text: str, k: int = DEFAULT_K) -> list[dict]:
-        """The linear model's top-k, with names attached. ~95% recall at k=5."""
-        X = sparse.hstack([
-            self.bundle["word"].transform([text]),
-            self.bundle["char"].transform([text]),
-        ]).tocsr()
-        p = self.bundle["clf"].predict_proba(X)[0]
-        order = np.argsort(-p)[:k]
-        return [
-            {
-                "code": str(self.classes[i]),
-                "name": self.names.get(str(self.classes[i]), str(self.classes[i])),
-                "score": round(float(p[i]), 4),
-            }
-            for i in order
-        ]
-
-    # -- stage 2: adjudicate ------------------------------------------------
-    def classify(
-        self,
-        title: str = "",
-        abstract: str = "",
-        k: int = DEFAULT_K,
-        max_codes: int = 3,
-    ) -> AsjcResult:
-        text = f"{title} {title} {abstract}".strip()
-        if not text:
-            return AsjcResult([], [], "empty")
-
-        cands = self.candidates(text, k=k)
-        if not cands:
-            return AsjcResult([], [], "no-candidates")
-
-        if self._client is None:
-            picked = [c["code"] for c in cands[:FALLBACK_K]]
-            return AsjcResult(picked, [self.names.get(c, c) for c in picked],
-                              "linear-topk", cands)
-
-        listing = "\n".join(f"  {c['code']}  {c['name']}" for c in cands)
-        prompt = _PROMPT.format(
-            n=len(cands), candidates=listing,
-            title=title or "(not supplied)",
-            abstract=(abstract or "(not supplied)")[:4000],
-        )
-        try:
-            msg = self._client.messages.create(
-                model=self.llm_model,
-                max_tokens=300,
-                temperature=0,
-                messages=[{"role": "user", "content": prompt}],
-            )
-            raw = "".join(b.text for b in msg.content if getattr(b, "type", "") == "text")
-            picked = self._parse(raw, {c["code"] for c in cands})[:max_codes]
-            if picked:
-                return AsjcResult(picked, [self.names.get(c, c) for c in picked],
-                                  "linear+llm", cands)
-            err = "LLM returned no valid code from the candidate set"
-        except Exception as e:                      # noqa: BLE001
-            err = f"{type(e).__name__}: {e}"
-
-        # Any LLM failure degrades to the statistical answer rather than raising.
-        # A broader-than-ideal subject filter is a far better outcome for the
-        # caller than a recommendation request that errors out.
-        picked = [c["code"] for c in cands[:FALLBACK_K]]
-        return AsjcResult(picked, [self.names.get(c, c) for c in picked],
-                          "linear-topk", cands, llm_error=err)
-
-    @staticmethod
-    def _parse(raw: str, allowed: set[str]) -> list[str]:
-        """
-        Pull codes out of the reply and hard-filter to the candidate set.
-
-        The filter is the point: it makes an invented or hallucinated code
-        structurally impossible to return, so the worst case is fewer codes
-        rather than a wrong one that silently poisons the journal filter.
-        """
-        codes: list[str] = []
-        m = re.search(r"\{.*\}", raw, re.S)
-        if m:
-            try:
-                codes = [str(c).strip() for c in json.loads(m.group(0)).get("codes", [])]
-            except (json.JSONDecodeError, AttributeError, TypeError):
-                codes = []
-        if not codes:
-            codes = re.findall(r"\b\d{4}\b", raw)
-        out: list[str] = []
-        for c in codes:
-            if c in allowed and c not in out:
-                out.append(c)
-        return out
+"""
+Abstract -> ASJC subject codes, for populating ResearchBee's recommendation filter.
+
+    from asjc_classifier import AsjcClassifier
+
+    clf = AsjcClassifier("asjc_model.joblib")           # local model only
+    clf = AsjcClassifier("asjc_model.joblib", llm=True) # + LLM adjudication
+
+    result = clf.classify(title=..., abstract=...)
+    result.codes        # ['3309', '1710'] — feed straight to recommend(asjc=...)
+    result.method       # 'linear+llm' | 'linear-topk'
+    result.candidates   # what the LLM was asked to choose between
+
+Architecture, and why it is not the obvious one
+-----------------------------------------------
+The obvious design is "run the linear model, escalate to an LLM when confidence
+is low". Measured on the held-out abstract set, that design cannot work: the
+top-1 probability averages 0.367 when the prediction is correct and 0.330 when
+it is wrong. There is no threshold separating them — the model is wrong often
+and unbothered about it.
+
+What the model *is* good at is narrowing. A correct code appears within:
+
+    top-1   30%
+    top-3   70%
+    top-5   95%     <- and flat thereafter
+
+So it is used as a candidate generator, not a decider. The linear model cuts 334
+codes to 5 with ~95% recall; the LLM picks among those 5. That inverts the usual
+cost argument in our favour — the prompt carries five labelled options rather
+than a 334-entry taxonomy, so the call is small, fast and hard to get wrong.
+
+Degradation without an API key is graceful: `codes` becomes the top 3, which the
+recommender applies as an OR filter. Recall is 70% rather than 95%, and the
+candidate pool is broader, but nothing breaks.
+
+Retrain after the OpenAlex harvest
+----------------------------------
+The shipped model is trained on journal titles, because that is the labelled text
+available without a network. It is applied to abstracts. `harvest_openalex.py`
+produces the matched corpus; retraining on it and re-running `asjc_eval.jsonl`
+measures the gain on the same 20 abstracts, so the upgrade is quantified rather
+than assumed.
+"""
+from __future__ import annotations
+
+import json
+import os
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+
+import joblib
+import numpy as np
+from scipy import sparse
+
+__all__ = ["AsjcClassifier", "AsjcResult"]
+
+#: Where the linear model's recall saturates. Measured, not guessed.
+DEFAULT_K = 5
+#: Codes returned when no LLM is available. top-3 = 70% recall.
+FALLBACK_K = 3
+
+
+@dataclass
+class AsjcResult:
+    codes: list[str]
+    names: list[str]
+    method: str
+    candidates: list[dict] = field(default_factory=list)
+    llm_error: str | None = None
+
+    def __bool__(self) -> bool:
+        return bool(self.codes)
+
+
+_PROMPT = """You are classifying an academic manuscript into Scopus ASJC subject areas.
+
+A statistical model has narrowed 334 possible subject areas to the {n} candidates \
+below. One or more of them is correct.
+
+Candidates:
+{candidates}
+
+Manuscript title: {title}
+
+Abstract: {abstract}
+
+Choose the 1-3 candidates that best describe where this manuscript belongs, most \
+appropriate first. Choose only from the candidate list — do not invent codes. If \
+the manuscript is genuinely interdisciplinary, return more than one; if it clearly \
+belongs to a single area, return just that one.
+
+Respond with JSON only, no other text:
+{{"codes": ["NNNN", ...], "reason": "one short sentence"}}"""
+
+
+class AsjcClassifier:
+    def __init__(
+        self,
+        model_path: str | Path = "asjc_model.joblib",
+        llm: bool = False,
+        llm_model: str | None = None,
+        api_key: str | None = None,
+    ):
+        self.bundle = joblib.load(model_path)
+        self.classes: np.ndarray = self.bundle["classes"]
+        self.names: dict[str, str] = self.bundle["names"]
+        self.parent_of: dict[str, str] = self.bundle["parent_of"]
+        self.trained_on: str = self.bundle.get("corpus", "unknown")
+
+        self._client = None
+        # Model id is configuration, not a constant — set ASJC_LLM_MODEL to
+        # whichever model you are entitled to. Verify it against your account's
+        # available models rather than trusting a default baked in here.
+        self.llm_model = llm_model or os.getenv("ASJC_LLM_MODEL", "claude-sonnet-4-5")
+        if llm:
+            key = api_key or os.getenv("ANTHROPIC_API_KEY")
+            if key:
+                try:
+                    import anthropic
+                    self._client = anthropic.Anthropic(api_key=key)
+                except ImportError:
+                    pass   # falls back to top-k; reported in AsjcResult.method
+
+    # -- stage 1: narrow ----------------------------------------------------
+    def candidates(self, text: str, k: int = DEFAULT_K) -> list[dict]:
+        """The linear model's top-k, with names attached. ~95% recall at k=5."""
+        X = sparse.hstack([
+            self.bundle["word"].transform([text]),
+            self.bundle["char"].transform([text]),
+        ]).tocsr()
+        p = self.bundle["clf"].predict_proba(X)[0]
+        order = np.argsort(-p)[:k]
+        return [
+            {
+                "code": str(self.classes[i]),
+                "name": self.names.get(str(self.classes[i]), str(self.classes[i])),
+                "score": round(float(p[i]), 4),
+            }
+            for i in order
+        ]
+
+    # -- stage 2: adjudicate ------------------------------------------------
+    def classify(
+        self,
+        title: str = "",
+        abstract: str = "",
+        k: int = DEFAULT_K,
+        max_codes: int = 3,
+    ) -> AsjcResult:
+        text = f"{title} {title} {abstract}".strip()
+        if not text:
+            return AsjcResult([], [], "empty")
+
+        cands = self.candidates(text, k=k)
+        if not cands:
+            return AsjcResult([], [], "no-candidates")
+
+        if self._client is None:
+            picked = [c["code"] for c in cands[:FALLBACK_K]]
+            return AsjcResult(picked, [self.names.get(c, c) for c in picked],
+                              "linear-topk", cands)
+
+        listing = "\n".join(f"  {c['code']}  {c['name']}" for c in cands)
+        prompt = _PROMPT.format(
+            n=len(cands), candidates=listing,
+            title=title or "(not supplied)",
+            abstract=(abstract or "(not supplied)")[:4000],
+        )
+        try:
+            msg = self._client.messages.create(
+                model=self.llm_model,
+                max_tokens=300,
+                temperature=0,
+                messages=[{"role": "user", "content": prompt}],
+            )
+            raw = "".join(b.text for b in msg.content if getattr(b, "type", "") == "text")
+            picked = self._parse(raw, {c["code"] for c in cands})[:max_codes]
+            if picked:
+                return AsjcResult(picked, [self.names.get(c, c) for c in picked],
+                                  "linear+llm", cands)
+            err = "LLM returned no valid code from the candidate set"
+        except Exception as e:                      # noqa: BLE001
+            err = f"{type(e).__name__}: {e}"
+
+        # Any LLM failure degrades to the statistical answer rather than raising.
+        # A broader-than-ideal subject filter is a far better outcome for the
+        # caller than a recommendation request that errors out.
+        picked = [c["code"] for c in cands[:FALLBACK_K]]
+        return AsjcResult(picked, [self.names.get(c, c) for c in picked],
+                          "linear-topk", cands, llm_error=err)
+
+    @staticmethod
+    def _parse(raw: str, allowed: set[str]) -> list[str]:
+        """
+        Pull codes out of the reply and hard-filter to the candidate set.
+
+        The filter is the point: it makes an invented or hallucinated code
+        structurally impossible to return, so the worst case is fewer codes
+        rather than a wrong one that silently poisons the journal filter.
+        """
+        codes: list[str] = []
+        m = re.search(r"\{.*\}", raw, re.S)
+        if m:
+            try:
+                codes = [str(c).strip() for c in json.loads(m.group(0)).get("codes", [])]
+            except (json.JSONDecodeError, AttributeError, TypeError):
+                codes = []
+        if not codes:
+            codes = re.findall(r"\b\d{4}\b", raw)
+        out: list[str] = []
+        for c in codes:
+            if c in allowed and c not in out:
+                out.append(c)
+        return out

asjc_eval.jsonl

@@ -1,20 +1,20 @@
-{"id": "lis-chatbot", "field": "Library & Information Science", "accept": ["3309", "1710", "1706"], "text": "We present a retrieval-augmented generation chatbot integrated with the Primo discovery layer at a university library. The system combines dense vector retrieval over catalogue records and LibGuides content with a large language model to answer patron reference questions at the point of need. We evaluate retrieval precision, answer groundedness and user satisfaction across 1,200 real reference queries submitted by academic library users over one semester."}
-{"id": "bibliometrics", "field": "Scientometrics", "accept": ["3309", "1706", "3312"], "text": "This scientometric analysis maps the growth of research output across national research systems using bibliometric indicators derived from Scopus and Web of Science. We compute field-normalised citation impact, international collaboration rates and disciplinary specialisation indices for 42 countries between 2000 and 2024, and examine the relationship between public research funding intensity and scholarly productivity."}
-{"id": "cardiology-rct", "field": "Cardiology", "accept": ["2705", "2700", "2736"], "text": "This multicentre randomised controlled trial evaluated the efficacy and safety of direct oral anticoagulation in patients with non-valvular atrial fibrillation. 840 patients were randomised to receive either apixaban or dose-adjusted warfarin, with a primary composite endpoint of ischaemic stroke or systemic embolism at 24 months. Cardiovascular mortality, major bleeding and all-cause hospitalisation were prespecified secondary outcomes."}
-{"id": "perovskite", "field": "Materials / Energy", "accept": ["2504", "2500", "2508", "2102", "3104", "3107", "1600", "2200"], "text": "We report a perovskite solar cell architecture achieving 25.6% power conversion efficiency. Thin-film deposition of methylammonium lead iodide was optimised through antisolvent engineering and interfacial passivation, and long-term device stability under continuous one-sun illumination was characterised over 1000 hours using X-ray diffraction, photoluminescence spectroscopy and impedance analysis."}
-{"id": "nlp-transformer", "field": "Computer Science / AI", "accept": ["1702", "1706", "1712", "1700"], "text": "We introduce a parameter-efficient fine-tuning method for large transformer language models that adapts fewer than 0.1% of weights while matching full fine-tuning performance on downstream natural language understanding benchmarks. We evaluate on GLUE and SuperGLUE, analyse the rank structure of the learned updates, and show consistent gains in low-resource transfer settings."}
-{"id": "groundwater", "field": "Earth / Environmental Science", "accept": ["2312", "1908", "1909", "1900", "2300", "2305", "2215"], "text": "We assess groundwater recharge dynamics in an arid alluvial aquifer using a coupled unsaturated-zone and finite-difference flow model calibrated against a decade of piezometric observations. Stable isotope signatures and chloride mass balance were used to partition recharge sources, and abstraction scenarios were simulated to evaluate long-term aquifer sustainability under projected climate forcing."}
-{"id": "nursing-education", "field": "Nursing", "accept": ["2900", "2922", "2901", "3304", "2908"], "text": "This mixed-methods study examined the effect of high-fidelity simulation on clinical reasoning among undergraduate nursing students. Ninety-four students completed simulated deteriorating-patient scenarios followed by structured debriefing. Clinical reasoning was assessed with a validated rubric, and semi-structured interviews explored students' experiences of psychological safety during debriefing."}
-{"id": "corporate-finance", "field": "Economics & Finance", "accept": ["2003", "2002", "2000", "1402", "1403"], "text": "We examine the effect of corporate governance quality on the cost of debt financing using a panel of listed firms across 18 emerging markets. Employing instrumental-variable estimation to address endogeneity, we find that board independence and disclosure quality are associated with materially lower credit spreads, with the effect concentrated among firms facing greater information asymmetry."}
-{"id": "medieval-history", "field": "History / Humanities", "accept": ["1202", "1200", "1208"], "text": "Drawing on previously unexamined chancery records, this article reconstructs patterns of monastic landholding in the late medieval Rhineland. It argues that the conventional narrative of continuous ecclesiastical accumulation obscures a sustained period of alienation driven by fiscal pressure, and situates these transactions within wider debates about the economic agency of religious houses."}
-{"id": "sociolinguistics", "field": "Linguistics / Arts & Humanities", "accept": ["3310", "1203", "1200", "3316"], "text": "This study investigates code-switching practices among second-generation multilingual speakers in an urban migration context. Using a corpus of naturalistic conversational recordings, we analyse patterns of intra-sentential switching and examine how speakers deploy linguistic resources to index identity and negotiate group membership across community settings."}
-{"id": "supply-chain", "field": "Operations / Decision Sciences", "accept": ["1803", "1800", "1802", "2209", "1408"], "text": "We formulate a stochastic mixed-integer programming model for resilient supply chain network design under correlated disruption risk. A Benders decomposition scheme with valid inequalities is developed to solve realistically sized instances, and computational experiments on a multi-echelon distribution case demonstrate substantial cost savings over deterministic and robust-optimisation baselines."}
-{"id": "immunology", "field": "Immunology & Microbiology", "accept": ["2403", "2404", "2400", "2406", "1312"], "text": "We characterise the tissue-resident memory T cell compartment in the human respiratory mucosa following viral infection. Single-cell RNA sequencing combined with flow cytometry revealed transcriptionally distinct subsets with divergent cytokine profiles, and adoptive transfer experiments in a murine model demonstrated that these populations confer durable protection against heterologous challenge."}
-{"id": "civil-engineering", "field": "Civil / Structural Engineering", "accept": ["2205", "2200", "2215", "2210"], "text": "This paper presents a performance-based seismic assessment of reinforced concrete moment frames designed to pre-1980 detailing provisions. Nonlinear time-history analyses were conducted across a suite of ground motions scaled to multiple hazard levels, and fragility surfaces were developed to quantify collapse probability and inform retrofit prioritisation for existing building stock."}
-{"id": "education-policy", "field": "Education", "accept": ["3304", "3300", "3312"], "text": "We evaluate the impact of a national school autonomy reform on student attainment using a difference-in-differences design across administrative regions. Drawing on eight years of standardised assessment data linked to school-level governance records, we find heterogeneous effects that vary systematically with baseline school capacity and local labour market conditions."}
-{"id": "pharmacology", "field": "Pharmacology & Pharmaceutics", "accept": ["3004", "3003", "3002", "3000", "1313"], "text": "We report the pharmacokinetic and pharmacodynamic characterisation of a novel small-molecule kinase inhibitor in preclinical models. Dose-ranging studies established plasma exposure and tissue distribution, hepatic microsomal assays quantified metabolic stability and CYP inhibition liability, and target engagement was confirmed by phosphoproteomic profiling in tumour xenografts."}
-{"id": "psychology-cognition", "field": "Psychology / Neuroscience", "accept": ["3205", "3206", "3200", "2805", "2802"], "text": "Across three preregistered experiments we examined how working memory load modulates attentional capture by emotionally salient distractors. Participants performed a visual search task under concurrent memory demand while eye movements were recorded. Results indicate that capture persists under high load for threat-related stimuli but is attenuated for neutral salient items."}
-{"id": "agriculture", "field": "Agricultural & Biological Sciences", "accept": ["1111", "1102", "1100", "2300", "1106"], "text": "We evaluate the effect of conservation tillage and cover cropping on soil organic carbon stocks and crop yield stability across a nine-year field experiment in a semi-arid cropping system. Soil sampling at multiple depths, combined with yield gap decomposition, indicates that carbon accrual is concentrated in surface horizons and that yield benefits emerge only in drier seasons."}
-{"id": "veterinary", "field": "Veterinary", "accept": ["3404", "3400", "3401", "3403"], "text": "This retrospective cohort study describes clinical presentation, treatment and outcome in dogs diagnosed with immune-mediated haemolytic anaemia at a referral hospital over twelve years. Prognostic factors were assessed by multivariable survival analysis, and the association between transfusion requirement, thromboembolic complications and 30-day mortality was quantified."}
-{"id": "dentistry", "field": "Dentistry", "accept": ["3504", "3506", "3500", "3501"], "text": "We compared marginal bone loss around platform-switched and conventional implant-abutment connections in a randomised split-mouth clinical trial. Sixty implants were placed in thirty partially edentulous patients, with standardised periapical radiography at baseline, six and twenty-four months, and peri-implant soft tissue parameters recorded alongside patient-reported outcomes."}
-{"id": "open-access-policy", "field": "LIS / Scholarly Communication", "accept": ["3309", "3312", "1710", "3300"], "text": "This study analyses institutional compliance with funder open access mandates across a national university system. Combining repository deposit records with publisher policy data from Open Policy Finder, we quantify green and gold route uptake, embargo adherence and article processing charge expenditure, and identify institutional factors associated with higher compliance rates."}
+{"id": "lis-chatbot", "field": "Library & Information Science", "accept": ["3309", "1710", "1706"], "text": "We present a retrieval-augmented generation chatbot integrated with the Primo discovery layer at a university library. The system combines dense vector retrieval over catalogue records and LibGuides content with a large language model to answer patron reference questions at the point of need. We evaluate retrieval precision, answer groundedness and user satisfaction across 1,200 real reference queries submitted by academic library users over one semester."}
+{"id": "bibliometrics", "field": "Scientometrics", "accept": ["3309", "1706", "3312"], "text": "This scientometric analysis maps the growth of research output across national research systems using bibliometric indicators derived from Scopus and Web of Science. We compute field-normalised citation impact, international collaboration rates and disciplinary specialisation indices for 42 countries between 2000 and 2024, and examine the relationship between public research funding intensity and scholarly productivity."}
+{"id": "cardiology-rct", "field": "Cardiology", "accept": ["2705", "2700", "2736"], "text": "This multicentre randomised controlled trial evaluated the efficacy and safety of direct oral anticoagulation in patients with non-valvular atrial fibrillation. 840 patients were randomised to receive either apixaban or dose-adjusted warfarin, with a primary composite endpoint of ischaemic stroke or systemic embolism at 24 months. Cardiovascular mortality, major bleeding and all-cause hospitalisation were prespecified secondary outcomes."}
+{"id": "perovskite", "field": "Materials / Energy", "accept": ["2504", "2500", "2508", "2102", "3104", "3107", "1600", "2200"], "text": "We report a perovskite solar cell architecture achieving 25.6% power conversion efficiency. Thin-film deposition of methylammonium lead iodide was optimised through antisolvent engineering and interfacial passivation, and long-term device stability under continuous one-sun illumination was characterised over 1000 hours using X-ray diffraction, photoluminescence spectroscopy and impedance analysis."}
+{"id": "nlp-transformer", "field": "Computer Science / AI", "accept": ["1702", "1706", "1712", "1700"], "text": "We introduce a parameter-efficient fine-tuning method for large transformer language models that adapts fewer than 0.1% of weights while matching full fine-tuning performance on downstream natural language understanding benchmarks. We evaluate on GLUE and SuperGLUE, analyse the rank structure of the learned updates, and show consistent gains in low-resource transfer settings."}
+{"id": "groundwater", "field": "Earth / Environmental Science", "accept": ["2312", "1908", "1909", "1900", "2300", "2305", "2215"], "text": "We assess groundwater recharge dynamics in an arid alluvial aquifer using a coupled unsaturated-zone and finite-difference flow model calibrated against a decade of piezometric observations. Stable isotope signatures and chloride mass balance were used to partition recharge sources, and abstraction scenarios were simulated to evaluate long-term aquifer sustainability under projected climate forcing."}
+{"id": "nursing-education", "field": "Nursing", "accept": ["2900", "2922", "2901", "3304", "2908"], "text": "This mixed-methods study examined the effect of high-fidelity simulation on clinical reasoning among undergraduate nursing students. Ninety-four students completed simulated deteriorating-patient scenarios followed by structured debriefing. Clinical reasoning was assessed with a validated rubric, and semi-structured interviews explored students' experiences of psychological safety during debriefing."}
+{"id": "corporate-finance", "field": "Economics & Finance", "accept": ["2003", "2002", "2000", "1402", "1403"], "text": "We examine the effect of corporate governance quality on the cost of debt financing using a panel of listed firms across 18 emerging markets. Employing instrumental-variable estimation to address endogeneity, we find that board independence and disclosure quality are associated with materially lower credit spreads, with the effect concentrated among firms facing greater information asymmetry."}
+{"id": "medieval-history", "field": "History / Humanities", "accept": ["1202", "1200", "1208"], "text": "Drawing on previously unexamined chancery records, this article reconstructs patterns of monastic landholding in the late medieval Rhineland. It argues that the conventional narrative of continuous ecclesiastical accumulation obscures a sustained period of alienation driven by fiscal pressure, and situates these transactions within wider debates about the economic agency of religious houses."}
+{"id": "sociolinguistics", "field": "Linguistics / Arts & Humanities", "accept": ["3310", "1203", "1200", "3316"], "text": "This study investigates code-switching practices among second-generation multilingual speakers in an urban migration context. Using a corpus of naturalistic conversational recordings, we analyse patterns of intra-sentential switching and examine how speakers deploy linguistic resources to index identity and negotiate group membership across community settings."}
+{"id": "supply-chain", "field": "Operations / Decision Sciences", "accept": ["1803", "1800", "1802", "2209", "1408"], "text": "We formulate a stochastic mixed-integer programming model for resilient supply chain network design under correlated disruption risk. A Benders decomposition scheme with valid inequalities is developed to solve realistically sized instances, and computational experiments on a multi-echelon distribution case demonstrate substantial cost savings over deterministic and robust-optimisation baselines."}
+{"id": "immunology", "field": "Immunology & Microbiology", "accept": ["2403", "2404", "2400", "2406", "1312"], "text": "We characterise the tissue-resident memory T cell compartment in the human respiratory mucosa following viral infection. Single-cell RNA sequencing combined with flow cytometry revealed transcriptionally distinct subsets with divergent cytokine profiles, and adoptive transfer experiments in a murine model demonstrated that these populations confer durable protection against heterologous challenge."}
+{"id": "civil-engineering", "field": "Civil / Structural Engineering", "accept": ["2205", "2200", "2215", "2210"], "text": "This paper presents a performance-based seismic assessment of reinforced concrete moment frames designed to pre-1980 detailing provisions. Nonlinear time-history analyses were conducted across a suite of ground motions scaled to multiple hazard levels, and fragility surfaces were developed to quantify collapse probability and inform retrofit prioritisation for existing building stock."}
+{"id": "education-policy", "field": "Education", "accept": ["3304", "3300", "3312"], "text": "We evaluate the impact of a national school autonomy reform on student attainment using a difference-in-differences design across administrative regions. Drawing on eight years of standardised assessment data linked to school-level governance records, we find heterogeneous effects that vary systematically with baseline school capacity and local labour market conditions."}
+{"id": "pharmacology", "field": "Pharmacology & Pharmaceutics", "accept": ["3004", "3003", "3002", "3000", "1313"], "text": "We report the pharmacokinetic and pharmacodynamic characterisation of a novel small-molecule kinase inhibitor in preclinical models. Dose-ranging studies established plasma exposure and tissue distribution, hepatic microsomal assays quantified metabolic stability and CYP inhibition liability, and target engagement was confirmed by phosphoproteomic profiling in tumour xenografts."}
+{"id": "psychology-cognition", "field": "Psychology / Neuroscience", "accept": ["3205", "3206", "3200", "2805", "2802"], "text": "Across three preregistered experiments we examined how working memory load modulates attentional capture by emotionally salient distractors. Participants performed a visual search task under concurrent memory demand while eye movements were recorded. Results indicate that capture persists under high load for threat-related stimuli but is attenuated for neutral salient items."}
+{"id": "agriculture", "field": "Agricultural & Biological Sciences", "accept": ["1111", "1102", "1100", "2300", "1106"], "text": "We evaluate the effect of conservation tillage and cover cropping on soil organic carbon stocks and crop yield stability across a nine-year field experiment in a semi-arid cropping system. Soil sampling at multiple depths, combined with yield gap decomposition, indicates that carbon accrual is concentrated in surface horizons and that yield benefits emerge only in drier seasons."}
+{"id": "veterinary", "field": "Veterinary", "accept": ["3404", "3400", "3401", "3403"], "text": "This retrospective cohort study describes clinical presentation, treatment and outcome in dogs diagnosed with immune-mediated haemolytic anaemia at a referral hospital over twelve years. Prognostic factors were assessed by multivariable survival analysis, and the association between transfusion requirement, thromboembolic complications and 30-day mortality was quantified."}
+{"id": "dentistry", "field": "Dentistry", "accept": ["3504", "3506", "3500", "3501"], "text": "We compared marginal bone loss around platform-switched and conventional implant-abutment connections in a randomised split-mouth clinical trial. Sixty implants were placed in thirty partially edentulous patients, with standardised periapical radiography at baseline, six and twenty-four months, and peri-implant soft tissue parameters recorded alongside patient-reported outcomes."}
+{"id": "open-access-policy", "field": "LIS / Scholarly Communication", "accept": ["3309", "3312", "1710", "3300"], "text": "This study analyses institutional compliance with funder open access mandates across a national university system. Combining repository deposit records with publisher policy data from Open Policy Finder, we quantify green and gold route uptake, embargo adherence and article processing charge expenditure, and identify institutional factors associated with higher compliance rates."}

build_centroids.py

@@ -1,130 +1,130 @@
-# /// script
-# requires-python = ">=3.10"
-# dependencies = ["numpy", "sentence-transformers", "huggingface_hub"]
-# ///
-"""
-Turn the harvest into journal centroid vectors for the recommender's stage 2.
-
-    python build_centroids.py --db journals.db \
-        --journal-text journal_text.jsonl --out centroids.npz
-
-Run this AFTER harvest_openalex.py. It does no network I/O of its own beyond
-downloading the sentence-transformer weights once.
-
-What gets embedded
-------------------
-Per journal, in descending order of signal:
-
-  1. OpenAlex topics, repeated in proportion to how much the journal publishes
-     on each — an article-derived fingerprint, pre-aggregated by OpenAlex.
-  2. The journal's ASJC subject names, from the index.
-  3. The journal title.
-
-Journals OpenAlex could not match still get (2) and (3), so every journal has a
-vector and none silently drops out of the reranking.
-
-Cost note
----------
-all-MiniLM-L6-v2 is 384-dimensional and ~90 MB. 32,050 active journals × 384
-float32 = ~49 MB uncompressed, well under what a Space can hold in memory. A
-larger model buys little here: the input is a bag of topic labels, not prose,
-so the extra capacity has nothing to work with.
-"""
-from __future__ import annotations
-
-import argparse
-import json
-import sqlite3
-from pathlib import Path
-
-import numpy as np
-
-
-def load_topics(path: str) -> dict[str, str]:
-    out: dict[str, str] = {}
-    p = Path(path)
-    if not p.exists():
-        print(f"note: {path} not found — building from index metadata only, "
-              f"which is much weaker. Run harvest_openalex.py first.")
-        return out
-    with p.open(encoding="utf-8") as f:
-        for line in f:
-            try:
-                rec = json.loads(line)
-            except json.JSONDecodeError:
-                continue
-            sid = rec.get("scopus_id")
-            if sid:
-                out[sid] = (out.get(sid, "") + " " + rec.get("topics", "")).strip()
-    return out
-
-
-def main() -> None:
-    ap = argparse.ArgumentParser()
-    ap.add_argument("--db", default="journals.db")
-    ap.add_argument("--journal-text", default="journal_text.jsonl")
-    ap.add_argument("--out", default="centroids.npz")
-    ap.add_argument("--model", default="sentence-transformers/all-MiniLM-L6-v2")
-    ap.add_argument("--batch-size", type=int, default=128)
-    ap.add_argument("--active-only", action="store_true", default=True)
-    ap.add_argument("--push-to-dataset", default=None)
-    args = ap.parse_args()
-
-    topics = load_topics(args.journal_text)
-    con = sqlite3.connect(args.db)
-
-    subs: dict[str, list[str]] = {}
-    for sid, name in con.execute(
-        """SELECT ja.scopus_id, coalesce(a.name, a.parent_name)
-           FROM journal_asjc ja JOIN asjc a ON a.code = ja.asjc"""
-    ):
-        subs.setdefault(sid, []).append(name or "")
-
-    where = "WHERE active = 1" if args.active_only else ""
-    rows = con.execute(f"SELECT scopus_id, title FROM journals {where}").fetchall()
-    con.close()
-
-    ids, docs = [], []
-    for sid, title in rows:
-        parts = [title or ""]
-        s = subs.get(sid)
-        if s:
-            parts.append(". ".join(sorted(set(s))))
-        t = topics.get(sid)
-        if t:
-            parts.append(t)
-        ids.append(sid)
-        docs.append(". ".join(p for p in parts if p)[:2000])
-
-    covered = sum(1 for i in ids if i in topics)
-    print(f"{len(ids)} journals; {covered} ({covered/max(len(ids),1):.0%}) "
-          f"have OpenAlex topics")
-
-    from sentence_transformers import SentenceTransformer
-    model = SentenceTransformer(args.model)
-    vecs = model.encode(
-        docs, batch_size=args.batch_size, show_progress_bar=True,
-        normalize_embeddings=True, convert_to_numpy=True,
-    ).astype("float32")
-
-    np.savez_compressed(args.out, ids=np.array(ids), vecs=vecs)
-    size = Path(args.out).stat().st_size / 1e6
-    print(f"wrote {args.out}  {vecs.shape}  ({size:.1f} MB)")
-
-    if args.push_to_dataset:
-        from huggingface_hub import HfApi
-        api = HfApi()
-        api.create_repo(args.push_to_dataset, repo_type="dataset", exist_ok=True)
-        api.upload_file(path_or_fileobj=args.out, path_in_repo=Path(args.out).name,
-                        repo_id=args.push_to_dataset, repo_type="dataset")
-        print(f"pushed {args.out} -> {args.push_to_dataset}")
-
-
-def load_centroids(path: str = "centroids.npz") -> dict[str, np.ndarray]:
-    """Load into the {scopus_id: vector} mapping recommend() expects."""
-    d = np.load(path, allow_pickle=False)
-    return {str(i): v for i, v in zip(d["ids"], d["vecs"])}
-
-
-if __name__ == "__main__":
-    main()
+# /// script
+# requires-python = ">=3.10"
+# dependencies = ["numpy", "sentence-transformers", "huggingface_hub"]
+# ///
+"""
+Turn the harvest into journal centroid vectors for the recommender's stage 2.
+
+    python build_centroids.py --db journals.db \
+        --journal-text journal_text.jsonl --out centroids.npz
+
+Run this AFTER harvest_openalex.py. It does no network I/O of its own beyond
+downloading the sentence-transformer weights once.
+
+What gets embedded
+------------------
+Per journal, in descending order of signal:
+
+  1. OpenAlex topics, repeated in proportion to how much the journal publishes
+     on each — an article-derived fingerprint, pre-aggregated by OpenAlex.
+  2. The journal's ASJC subject names, from the index.
+  3. The journal title.
+
+Journals OpenAlex could not match still get (2) and (3), so every journal has a
+vector and none silently drops out of the reranking.
+
+Cost note
+---------
+all-MiniLM-L6-v2 is 384-dimensional and ~90 MB. 32,050 active journals × 384
+float32 = ~49 MB uncompressed, well under what a Space can hold in memory. A
+larger model buys little here: the input is a bag of topic labels, not prose,
+so the extra capacity has nothing to work with.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import sqlite3
+from pathlib import Path
+
+import numpy as np
+
+
+def load_topics(path: str) -> dict[str, str]:
+    out: dict[str, str] = {}
+    p = Path(path)
+    if not p.exists():
+        print(f"note: {path} not found — building from index metadata only, "
+              f"which is much weaker. Run harvest_openalex.py first.")
+        return out
+    with p.open(encoding="utf-8") as f:
+        for line in f:
+            try:
+                rec = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+            sid = rec.get("scopus_id")
+            if sid:
+                out[sid] = (out.get(sid, "") + " " + rec.get("topics", "")).strip()
+    return out
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--db", default="journals.db")
+    ap.add_argument("--journal-text", default="journal_text.jsonl")
+    ap.add_argument("--out", default="centroids.npz")
+    ap.add_argument("--model", default="sentence-transformers/all-MiniLM-L6-v2")
+    ap.add_argument("--batch-size", type=int, default=128)
+    ap.add_argument("--active-only", action="store_true", default=True)
+    ap.add_argument("--push-to-dataset", default=None)
+    args = ap.parse_args()
+
+    topics = load_topics(args.journal_text)
+    con = sqlite3.connect(args.db)
+
+    subs: dict[str, list[str]] = {}
+    for sid, name in con.execute(
+        """SELECT ja.scopus_id, coalesce(a.name, a.parent_name)
+           FROM journal_asjc ja JOIN asjc a ON a.code = ja.asjc"""
+    ):
+        subs.setdefault(sid, []).append(name or "")
+
+    where = "WHERE active = 1" if args.active_only else ""
+    rows = con.execute(f"SELECT scopus_id, title FROM journals {where}").fetchall()
+    con.close()
+
+    ids, docs = [], []
+    for sid, title in rows:
+        parts = [title or ""]
+        s = subs.get(sid)
+        if s:
+            parts.append(". ".join(sorted(set(s))))
+        t = topics.get(sid)
+        if t:
+            parts.append(t)
+        ids.append(sid)
+        docs.append(". ".join(p for p in parts if p)[:2000])
+
+    covered = sum(1 for i in ids if i in topics)
+    print(f"{len(ids)} journals; {covered} ({covered/max(len(ids),1):.0%}) "
+          f"have OpenAlex topics")
+
+    from sentence_transformers import SentenceTransformer
+    model = SentenceTransformer(args.model)
+    vecs = model.encode(
+        docs, batch_size=args.batch_size, show_progress_bar=True,
+        normalize_embeddings=True, convert_to_numpy=True,
+    ).astype("float32")
+
+    np.savez_compressed(args.out, ids=np.array(ids), vecs=vecs)
+    size = Path(args.out).stat().st_size / 1e6
+    print(f"wrote {args.out}  {vecs.shape}  ({size:.1f} MB)")
+
+    if args.push_to_dataset:
+        from huggingface_hub import HfApi
+        api = HfApi()
+        api.create_repo(args.push_to_dataset, repo_type="dataset", exist_ok=True)
+        api.upload_file(path_or_fileobj=args.out, path_in_repo=Path(args.out).name,
+                        repo_id=args.push_to_dataset, repo_type="dataset")
+        print(f"pushed {args.out} -> {args.push_to_dataset}")
+
+
+def load_centroids(path: str = "centroids.npz") -> dict[str, np.ndarray]:
+    """Load into the {scopus_id: vector} mapping recommend() expects."""
+    d = np.load(path, allow_pickle=False)
+    return {str(i): v for i, v in zip(d["ids"], d["vecs"])}
+
+
+if __name__ == "__main__":
+    main()

doi_utils.py

@@ -1,92 +1,92 @@
-"""
-doi_utils.py — deterministic DOI extraction and ISSN validation.
-
-No LLM. No guessing. If a DOI cannot be extracted, we say so and ask the user.
-"""
-
-from __future__ import annotations
-
-import re
-from urllib.parse import unquote, urlparse
-
-# DOIs are 10.<registrant>/<suffix>. The suffix may contain almost anything,
-# so we trim trailing punctuation and known URL cruft afterwards.
-DOI_RE = re.compile(r"(10\.\d{4,9}/[^\s\"'<>&?#]+)", re.I)
-
-_TRAILING = ".,;:)]}>'\"-"
-_SUFFIX_JUNK = (
-    "/full", "/abstract", "/pdf", "/epub", "/meta", "/html",
-    ".full", ".pdf", ".abstract", ".long",
-)
-
-ISSN_RE = re.compile(r"^(\d{4})-?(\d{3}[\dXx])$")
-
-
-def extract_doi(text: str) -> str:
-    """
-    Pull a DOI out of a URL, a citation, or raw text.
-
-    Handles publisher URL shapes:
-        https://www.tandfonline.com/doi/full/10.1080/19322909.2023.2221477
-        https://link.springer.com/article/10.1007/s11192-023-04812-4
-        https://doi.org/10.1016/j.jclepro.2023.136775
-        https://onlinelibrary.wiley.com/doi/10.1002/adma.202301234
-        10.1109/TCOMM.2023.1234567
-    Returns "" when no DOI is present.
-    """
-    if not text:
-        return ""
-    s = unquote(str(text).strip())
-
-    m = DOI_RE.search(s)
-    if not m:
-        return ""
-    doi = m.group(1)
-
-    # Strip publisher URL suffixes appended after the DOI.
-    low = doi.lower()
-    for junk in _SUFFIX_JUNK:
-        if low.endswith(junk):
-            doi = doi[: -len(junk)]
-            low = doi.lower()
-
-    doi = doi.rstrip(_TRAILING)
-    return doi if "/" in doi and len(doi) > 7 else ""
-
-
-def looks_like_url(text: str) -> bool:
-    try:
-        p = urlparse((text or "").strip())
-        return p.scheme in ("http", "https") and bool(p.netloc)
-    except ValueError:
-        return False
-
-
-def normalise_issn(raw: str) -> str:
-    """Return a hyphenated, checksum-valid ISSN, or '' if invalid."""
-    m = ISSN_RE.match((raw or "").strip())
-    if not m:
-        return ""
-    digits = (m.group(1) + m.group(2)).upper()
-    if not issn_checksum_ok(digits):
-        return ""
-    return f"{digits[:4]}-{digits[4:]}"
-
-
-def issn_checksum_ok(eight: str) -> bool:
-    """ISSN mod-11 check digit. Catches typos before we hit the network."""
-    e = eight.replace("-", "").upper()
-    if len(e) != 8:
-        return False
-    try:
-        total = sum(int(e[i]) * (8 - i) for i in range(7))
-    except ValueError:
-        return False
-    remainder = total % 11
-    check = 0 if remainder == 0 else 11 - remainder
-    expected = "X" if check == 10 else str(check)
-    return e[7] == expected
-
-
-def doi_url(doi: str) -> str:
-    return f"https://doi.org/{doi}" if doi else ""
+"""
+doi_utils.py — deterministic DOI extraction and ISSN validation.
+
+No LLM. No guessing. If a DOI cannot be extracted, we say so and ask the user.
+"""
+
+from __future__ import annotations
+
+import re
+from urllib.parse import unquote, urlparse
+
+# DOIs are 10.<registrant>/<suffix>. The suffix may contain almost anything,
+# so we trim trailing punctuation and known URL cruft afterwards.
+DOI_RE = re.compile(r"(10\.\d{4,9}/[^\s\"'<>&?#]+)", re.I)
+
+_TRAILING = ".,;:)]}>'\"-"
+_SUFFIX_JUNK = (
+    "/full", "/abstract", "/pdf", "/epub", "/meta", "/html",
+    ".full", ".pdf", ".abstract", ".long",
+)
+
+ISSN_RE = re.compile(r"^(\d{4})-?(\d{3}[\dXx])$")
+
+
+def extract_doi(text: str) -> str:
+    """
+    Pull a DOI out of a URL, a citation, or raw text.
+
+    Handles publisher URL shapes:
+        https://www.tandfonline.com/doi/full/10.1080/19322909.2023.2221477
+        https://link.springer.com/article/10.1007/s11192-023-04812-4
+        https://doi.org/10.1016/j.jclepro.2023.136775
+        https://onlinelibrary.wiley.com/doi/10.1002/adma.202301234
+        10.1109/TCOMM.2023.1234567
+    Returns "" when no DOI is present.
+    """
+    if not text:
+        return ""
+    s = unquote(str(text).strip())
+
+    m = DOI_RE.search(s)
+    if not m:
+        return ""
+    doi = m.group(1)
+
+    # Strip publisher URL suffixes appended after the DOI.
+    low = doi.lower()
+    for junk in _SUFFIX_JUNK:
+        if low.endswith(junk):
+            doi = doi[: -len(junk)]
+            low = doi.lower()
+
+    doi = doi.rstrip(_TRAILING)
+    return doi if "/" in doi and len(doi) > 7 else ""
+
+
+def looks_like_url(text: str) -> bool:
+    try:
+        p = urlparse((text or "").strip())
+        return p.scheme in ("http", "https") and bool(p.netloc)
+    except ValueError:
+        return False
+
+
+def normalise_issn(raw: str) -> str:
+    """Return a hyphenated, checksum-valid ISSN, or '' if invalid."""
+    m = ISSN_RE.match((raw or "").strip())
+    if not m:
+        return ""
+    digits = (m.group(1) + m.group(2)).upper()
+    if not issn_checksum_ok(digits):
+        return ""
+    return f"{digits[:4]}-{digits[4:]}"
+
+
+def issn_checksum_ok(eight: str) -> bool:
+    """ISSN mod-11 check digit. Catches typos before we hit the network."""
+    e = eight.replace("-", "").upper()
+    if len(e) != 8:
+        return False
+    try:
+        total = sum(int(e[i]) * (8 - i) for i in range(7))
+    except ValueError:
+        return False
+    remainder = total % 11
+    check = 0 if remainder == 0 else 11 - remainder
+    expected = "X" if check == 10 else str(check)
+    return e[7] == expected
+
+
+def doi_url(doi: str) -> str:
+    return f"https://doi.org/{doi}" if doi else ""

harvest_openalex.py

@@ -1,353 +1,353 @@
-# /// script
-# requires-python = ">=3.10"
-# dependencies = ["requests", "huggingface_hub"]
-# ///
-"""
-One OpenAlex pass that produces BOTH deliverables:
-
-  journal_text.jsonl   topical fingerprint per journal  -> centroids (recommender stage 2)
-  asjc_train.jsonl     labelled article abstracts       -> ASJC classifier training
-
-    python harvest_openalex.py --db journals.db --mailto you@ku.ac.ae
-    python harvest_openalex.py --db journals.db --mailto you@ku.ac.ae --resume
-
-These are the same job. The centroids need to know what each journal publishes
-about; the classifier needs text labelled with subject codes. Both are answered
-by reading OpenAlex once. Running them as separate harvests would double the
-request count for no benefit.
-
-Two phases
-----------
-PHASE 1  sources    ~1,400 requests, minutes.
-    Cursor-pages every journal source and takes its `topics`, which OpenAlex has
-    already inferred from that journal's article corpus. An article-derived
-    signal delivered pre-aggregated — most of the value of reading the articles,
-    at ~4% of the requests.
-
-PHASE 2  works      ~8,000 requests, 1-2 hours.
-    Pulls recent abstracts per journal to train the classifier. Sampling is
-    STRATIFIED BY ASJC CODE, not uniform over journals: ASJC is wildly
-    imbalanced (History has 2,009 journals, Medical Terminology has 2), and
-    uniform sampling would produce a classifier that has never seen the rare
-    half of the taxonomy. `--per-code` journals are drawn per subject area, so
-    thin codes get represented.
-
-Resumability
-------------
-HF Space containers restart without warning, and a two-hour harvest that loses
-everything on restart is not a harvest. Both outputs are append-only JSONL and
-progress is checkpointed to `<out>.state.json` after every batch. `--resume`
-picks up from the last checkpoint. Re-running without `--resume` starts over.
-
-Rate limits
------------
-OpenAlex allows 100k requests/day and ~10/sec in the polite pool. `--mailto` is
-what puts you in that pool — without it you are throttled harder and less
-predictably. It is required for a reason.
-
-Where to run this
------------------
-Not inside a Space. A Space's disk is ephemeral — "content is lost if your Space
-restarts or is stopped" — so a two-hour harvest writing 250k abstracts there is
-one restart away from nothing, and free Spaces give you no terminal to launch it
-from. Persistent storage on a Space is a paid Storage Bucket.
-
-Run it as an HF Job instead — same infrastructure, purpose-built for exactly
-this, billed per second:
-
-    hf jobs uv run --with requests,huggingface_hub \\
-        harvest_openalex.py -- --db journals.db --mailto you@ku.ac.ae \\
-        --push-to-dataset nikeshn/researchbee-index
-
-The PEP 723 header above declares the dependencies, so `hf jobs uv run` resolves
-them without a Dockerfile. `--push-to-dataset` uploads the outputs to a Dataset
-repo as they complete, which is what makes the run durable: the Job container is
-as ephemeral as the Space, so anything not pushed is lost when it exits.
-"""
-from __future__ import annotations
-
-import argparse
-import json
-import random
-import sqlite3
-import time
-from collections import defaultdict
-from pathlib import Path
-
-import requests
-
-API = "https://api.openalex.org"
-SESSION = requests.Session()
-SESSION.headers.update({"User-Agent": "ResearchBee/1.0 (journal index build)"})
-
-
-# ---------------------------------------------------------------------------
-# HTTP
-# ---------------------------------------------------------------------------
-def get(url: str, params: dict, mailto: str, tries: int = 5) -> dict:
-    params = {**params, "mailto": mailto}
-    for attempt in range(tries):
-        try:
-            r = SESSION.get(url, params=params, timeout=60)
-            if r.status_code in (429, 503):
-                time.sleep(min(60, 2 ** attempt * 2))
-                continue
-            r.raise_for_status()
-            return r.json()
-        except requests.RequestException:
-            if attempt == tries - 1:
-                return {}
-            time.sleep(min(60, 2 ** attempt))
-    return {}
-
-
-def reconstruct_abstract(inv: dict | None) -> str:
-    """
-    OpenAlex ships abstracts as an inverted index ({word: [positions]}) because
-    of publisher redistribution terms. Rebuild the linear text.
-    """
-    if not inv:
-        return ""
-    positions: list[tuple[int, str]] = []
-    for word, idxs in inv.items():
-        for i in idxs:
-            positions.append((i, word))
-    if not positions:
-        return ""
-    positions.sort()
-    return " ".join(w for _, w in positions)
-
-
-# ---------------------------------------------------------------------------
-# Checkpointing
-# ---------------------------------------------------------------------------
-class Checkpoint:
-    def __init__(self, path: Path, resume: bool):
-        self.path = path
-        self.state = {"phase": 1, "cursor": "*", "done_journals": [], "n_works": 0}
-        if resume and path.exists():
-            self.state = json.loads(path.read_text())
-            print(f"resuming: phase {self.state['phase']}, "
-                  f"{len(self.state['done_journals'])} journals done, "
-                  f"{self.state['n_works']} abstracts written")
-
-    def save(self) -> None:
-        tmp = self.path.with_suffix(".tmp")
-        tmp.write_text(json.dumps(self.state))
-        tmp.replace(self.path)      # atomic — a restart mid-write must not corrupt it
-
-
-# ---------------------------------------------------------------------------
-# Index helpers
-# ---------------------------------------------------------------------------
-def issn_to_scopus(con: sqlite3.Connection) -> dict[str, str]:
-    m: dict[str, str] = {}
-    for sid, p, e in con.execute(
-        "SELECT scopus_id, issn_print, issn_e FROM journals WHERE active = 1"
-    ):
-        for i in (p, e):
-            if i:
-                m.setdefault(f"{i[:4]}-{i[4:]}", sid)
-    return m
-
-
-def stratified_journals(con: sqlite3.Connection, per_code: int, seed: int) -> list[tuple[str, str]]:
-    """
-    Pick up to `per_code` journals per ASJC code, preferring higher-SJR titles
-    (better-edited abstracts, cleaner topical signal), then de-duplicate.
-    Returns [(scopus_id, issn)].
-    """
-    rng = random.Random(seed)
-    by_code: dict[str, list[tuple[str, str]]] = defaultdict(list)
-    for sid, issn, code in con.execute(
-        """SELECT j.scopus_id, coalesce(j.issn_e, j.issn_print), ja.asjc
-           FROM journals j JOIN journal_asjc ja ON ja.scopus_id = j.scopus_id
-           WHERE j.active = 1 AND j.source_type = 'Journal'
-             AND coalesce(j.issn_e, j.issn_print) IS NOT NULL
-             AND j.has_english = 1
-           ORDER BY j.sjr DESC NULLS LAST"""
-    ):
-        by_code[code].append((sid, issn))
-
-    picked: dict[str, str] = {}
-    for code, rows in by_code.items():
-        # Take the strongest half, then sample within it, so the corpus is not
-        # entirely the same handful of megajournals per field.
-        pool = rows[: max(per_code * 3, per_code)]
-        rng.shuffle(pool)
-        for sid, issn in pool[:per_code]:
-            picked.setdefault(sid, issn)
-    return sorted(picked.items())
-
-
-# ---------------------------------------------------------------------------
-# Phase 1 — journal topics (centroids)
-# ---------------------------------------------------------------------------
-def phase1(con, mailto: str, out: Path, ck: Checkpoint) -> None:
-    keymap = issn_to_scopus(con)
-    seen_ids: set[str] = set()
-    if out.exists() and ck.state["phase"] > 1:
-        return
-    fh = out.open("a", encoding="utf-8")
-    if out.exists():
-        with out.open(encoding="utf-8") as f:
-            for line in f:
-                try:
-                    seen_ids.add(json.loads(line)["scopus_id"])
-                except (json.JSONDecodeError, KeyError):
-                    pass
-
-    cursor = ck.state.get("cursor") or "*"
-    page = 0
-    while cursor:
-        data = get(f"{API}/sources", {
-            "filter": "type:journal", "per-page": 200, "cursor": cursor,
-            "select": "id,display_name,issn,topics",
-        }, mailto)
-        results = data.get("results", [])
-        if not results:
-            break
-        for s in results:
-            sid = next((keymap[i] for i in (s.get("issn") or []) if i in keymap), None)
-            if not sid or sid in seen_ids:
-                continue
-            terms: list[str] = []
-            for t in (s.get("topics") or [])[:12]:
-                n = int(t.get("count", 1) or 1)
-                # Repeat proportional to prevalence so the embedding reflects
-                # what the journal mostly publishes, not its long tail.
-                terms += [t.get("display_name", "")] * max(1, min(4, n // 50))
-                for lvl in ("subfield", "field"):
-                    if (t.get(lvl) or {}).get("display_name"):
-                        terms.append(t[lvl]["display_name"])
-            if terms:
-                fh.write(json.dumps({"scopus_id": sid, "topics": " ".join(terms)}) + "\n")
-                seen_ids.add(sid)
-        fh.flush()
-        cursor = (data.get("meta") or {}).get("next_cursor")
-        ck.state["cursor"] = cursor
-        ck.save()
-        page += 1
-        if page % 10 == 0:
-            print(f"  phase1 page {page}  matched {len(seen_ids)}", flush=True)
-
-    fh.close()
-    ck.state.update(phase=2, cursor=None)
-    ck.save()
-    print(f"phase 1 done: {len(seen_ids)} journals with topics")
-
-
-# ---------------------------------------------------------------------------
-# Phase 2 — labelled abstracts (classifier)
-# ---------------------------------------------------------------------------
-def phase2(con, mailto: str, out: Path, ck: Checkpoint,
-           per_code: int, per_journal: int, seed: int, from_year: int) -> None:
-    targets = stratified_journals(con, per_code, seed)
-    done = set(ck.state["done_journals"])
-    todo = [(s, i) for s, i in targets if s not in done]
-    print(f"phase 2: {len(targets)} journals selected, {len(todo)} remaining")
-
-    fh = out.open("a", encoding="utf-8")
-    for n, (sid, issn) in enumerate(todo, 1):
-        fmt = f"{issn[:4]}-{issn[4:]}"
-        data = get(f"{API}/works", {
-            "filter": (f"primary_location.source.issn:{fmt},"
-                       f"from_publication_date:{from_year}-01-01,"
-                       "type:article,has_abstract:true"),
-            "per-page": per_journal,
-            "select": "title,abstract_inverted_index",
-        }, mailto)
-        wrote = 0
-        for w in data.get("results", []):
-            abstract = reconstruct_abstract(w.get("abstract_inverted_index"))
-            if len(abstract) < 200:
-                continue
-            text = f"{w.get('title') or ''}. {abstract}".strip()
-            fh.write(json.dumps({"scopus_id": sid, "text": text[:4000]}) + "\n")
-            wrote += 1
-        ck.state["n_works"] += wrote
-        done.add(sid)
-
-        if n % 25 == 0:
-            fh.flush()
-            ck.state["done_journals"] = sorted(done)
-            ck.save()
-            print(f"  phase2 {n}/{len(todo)} journals, "
-                  f"{ck.state['n_works']} abstracts", flush=True)
-        time.sleep(0.11)          # ~9 req/s, inside the polite-pool ceiling
-
-    fh.flush()
-    fh.close()
-    ck.state["done_journals"] = sorted(done)
-    ck.state["phase"] = 3
-    ck.save()
-    print(f"phase 2 done: {ck.state['n_works']} abstracts")
-
-
-def main() -> None:
-    ap = argparse.ArgumentParser()
-    ap.add_argument("--db", default="journals.db")
-    ap.add_argument("--mailto", required=True,
-                    help="your email — required for OpenAlex's polite pool")
-    ap.add_argument("--resume", action="store_true")
-    ap.add_argument("--skip-phase1", action="store_true")
-    ap.add_argument("--skip-phase2", action="store_true")
-    ap.add_argument("--journal-text", default="journal_text.jsonl")
-    ap.add_argument("--train-out", default="asjc_train.jsonl")
-    ap.add_argument("--per-code", type=int, default=25,
-                    help="journals sampled per ASJC code (stratification depth)")
-    ap.add_argument("--per-journal", type=int, default=40,
-                    help="abstracts pulled per journal")
-    ap.add_argument("--from-year", type=int, default=2021)
-    ap.add_argument("--seed", type=int, default=42)
-    ap.add_argument("--push-to-dataset", default=None,
-                    help="HF Dataset repo id to upload outputs to, e.g. user/researchbee-index. "
-                         "Required for Job runs — the container's disk does not survive it.")
-    args = ap.parse_args()
-
-    con = sqlite3.connect(args.db)
-    ck = Checkpoint(Path(args.train_out + ".state.json"), args.resume)
-
-    if not args.resume:
-        for p in (args.journal_text, args.train_out):
-            Path(p).unlink(missing_ok=True)
-
-    if not args.skip_phase1 and ck.state["phase"] <= 1:
-        phase1(con, args.mailto, Path(args.journal_text), ck)
-    if not args.skip_phase2 and ck.state["phase"] <= 2:
-        phase2(con, args.mailto, Path(args.train_out), ck,
-               args.per_code, args.per_journal, args.seed, args.from_year)
-
-    for p in (args.journal_text, args.train_out):
-        path = Path(p)
-        if path.exists():
-            n = sum(1 for _ in path.open(encoding="utf-8"))
-            print(f"{p}: {n} lines, {path.stat().st_size/1e6:.1f} MB")
-    con.close()
-
-    if args.push_to_dataset:
-        push(args.push_to_dataset, [args.journal_text, args.train_out,
-                                    args.train_out + ".state.json"])
-
-
-def push(repo_id: str, paths: list[str]) -> None:
-    """
-    Upload outputs to a Dataset repo. This is not a convenience — a Job's disk
-    is discarded when the run ends, so anything not pushed did not happen.
-    """
-    from huggingface_hub import HfApi
-    api = HfApi()
-    api.create_repo(repo_id, repo_type="dataset", exist_ok=True)
-    for p in paths:
-        path = Path(p)
-        if not path.exists():
-            continue
-        api.upload_file(
-            path_or_fileobj=str(path), path_in_repo=path.name,
-            repo_id=repo_id, repo_type="dataset",
-        )
-        print(f"pushed {path.name} -> {repo_id}")
-
-
-if __name__ == "__main__":
-    main()
+# /// script
+# requires-python = ">=3.10"
+# dependencies = ["requests", "huggingface_hub"]
+# ///
+"""
+One OpenAlex pass that produces BOTH deliverables:
+
+  journal_text.jsonl   topical fingerprint per journal  -> centroids (recommender stage 2)
+  asjc_train.jsonl     labelled article abstracts       -> ASJC classifier training
+
+    python harvest_openalex.py --db journals.db --mailto you@ku.ac.ae
+    python harvest_openalex.py --db journals.db --mailto you@ku.ac.ae --resume
+
+These are the same job. The centroids need to know what each journal publishes
+about; the classifier needs text labelled with subject codes. Both are answered
+by reading OpenAlex once. Running them as separate harvests would double the
+request count for no benefit.
+
+Two phases
+----------
+PHASE 1  sources    ~1,400 requests, minutes.
+    Cursor-pages every journal source and takes its `topics`, which OpenAlex has
+    already inferred from that journal's article corpus. An article-derived
+    signal delivered pre-aggregated — most of the value of reading the articles,
+    at ~4% of the requests.
+
+PHASE 2  works      ~8,000 requests, 1-2 hours.
+    Pulls recent abstracts per journal to train the classifier. Sampling is
+    STRATIFIED BY ASJC CODE, not uniform over journals: ASJC is wildly
+    imbalanced (History has 2,009 journals, Medical Terminology has 2), and
+    uniform sampling would produce a classifier that has never seen the rare
+    half of the taxonomy. `--per-code` journals are drawn per subject area, so
+    thin codes get represented.
+
+Resumability
+------------
+HF Space containers restart without warning, and a two-hour harvest that loses
+everything on restart is not a harvest. Both outputs are append-only JSONL and
+progress is checkpointed to `<out>.state.json` after every batch. `--resume`
+picks up from the last checkpoint. Re-running without `--resume` starts over.
+
+Rate limits
+-----------
+OpenAlex allows 100k requests/day and ~10/sec in the polite pool. `--mailto` is
+what puts you in that pool — without it you are throttled harder and less
+predictably. It is required for a reason.
+
+Where to run this
+-----------------
+Not inside a Space. A Space's disk is ephemeral — "content is lost if your Space
+restarts or is stopped" — so a two-hour harvest writing 250k abstracts there is
+one restart away from nothing, and free Spaces give you no terminal to launch it
+from. Persistent storage on a Space is a paid Storage Bucket.
+
+Run it as an HF Job instead — same infrastructure, purpose-built for exactly
+this, billed per second:
+
+    hf jobs uv run --with requests,huggingface_hub \\
+        harvest_openalex.py -- --db journals.db --mailto you@ku.ac.ae \\
+        --push-to-dataset nikeshn/researchbee-index
+
+The PEP 723 header above declares the dependencies, so `hf jobs uv run` resolves
+them without a Dockerfile. `--push-to-dataset` uploads the outputs to a Dataset
+repo as they complete, which is what makes the run durable: the Job container is
+as ephemeral as the Space, so anything not pushed is lost when it exits.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import random
+import sqlite3
+import time
+from collections import defaultdict
+from pathlib import Path
+
+import requests
+
+API = "https://api.openalex.org"
+SESSION = requests.Session()
+SESSION.headers.update({"User-Agent": "ResearchBee/1.0 (journal index build)"})
+
+
+# ---------------------------------------------------------------------------
+# HTTP
+# ---------------------------------------------------------------------------
+def get(url: str, params: dict, mailto: str, tries: int = 5) -> dict:
+    params = {**params, "mailto": mailto}
+    for attempt in range(tries):
+        try:
+            r = SESSION.get(url, params=params, timeout=60)
+            if r.status_code in (429, 503):
+                time.sleep(min(60, 2 ** attempt * 2))
+                continue
+            r.raise_for_status()
+            return r.json()
+        except requests.RequestException:
+            if attempt == tries - 1:
+                return {}
+            time.sleep(min(60, 2 ** attempt))
+    return {}
+
+
+def reconstruct_abstract(inv: dict | None) -> str:
+    """
+    OpenAlex ships abstracts as an inverted index ({word: [positions]}) because
+    of publisher redistribution terms. Rebuild the linear text.
+    """
+    if not inv:
+        return ""
+    positions: list[tuple[int, str]] = []
+    for word, idxs in inv.items():
+        for i in idxs:
+            positions.append((i, word))
+    if not positions:
+        return ""
+    positions.sort()
+    return " ".join(w for _, w in positions)
+
+
+# ---------------------------------------------------------------------------
+# Checkpointing
+# ---------------------------------------------------------------------------
+class Checkpoint:
+    def __init__(self, path: Path, resume: bool):
+        self.path = path
+        self.state = {"phase": 1, "cursor": "*", "done_journals": [], "n_works": 0}
+        if resume and path.exists():
+            self.state = json.loads(path.read_text())
+            print(f"resuming: phase {self.state['phase']}, "
+                  f"{len(self.state['done_journals'])} journals done, "
+                  f"{self.state['n_works']} abstracts written")
+
+    def save(self) -> None:
+        tmp = self.path.with_suffix(".tmp")
+        tmp.write_text(json.dumps(self.state))
+        tmp.replace(self.path)      # atomic — a restart mid-write must not corrupt it
+
+
+# ---------------------------------------------------------------------------
+# Index helpers
+# ---------------------------------------------------------------------------
+def issn_to_scopus(con: sqlite3.Connection) -> dict[str, str]:
+    m: dict[str, str] = {}
+    for sid, p, e in con.execute(
+        "SELECT scopus_id, issn_print, issn_e FROM journals WHERE active = 1"
+    ):
+        for i in (p, e):
+            if i:
+                m.setdefault(f"{i[:4]}-{i[4:]}", sid)
+    return m
+
+
+def stratified_journals(con: sqlite3.Connection, per_code: int, seed: int) -> list[tuple[str, str]]:
+    """
+    Pick up to `per_code` journals per ASJC code, preferring higher-SJR titles
+    (better-edited abstracts, cleaner topical signal), then de-duplicate.
+    Returns [(scopus_id, issn)].
+    """
+    rng = random.Random(seed)
+    by_code: dict[str, list[tuple[str, str]]] = defaultdict(list)
+    for sid, issn, code in con.execute(
+        """SELECT j.scopus_id, coalesce(j.issn_e, j.issn_print), ja.asjc
+           FROM journals j JOIN journal_asjc ja ON ja.scopus_id = j.scopus_id
+           WHERE j.active = 1 AND j.source_type = 'Journal'
+             AND coalesce(j.issn_e, j.issn_print) IS NOT NULL
+             AND j.has_english = 1
+           ORDER BY j.sjr DESC NULLS LAST"""
+    ):
+        by_code[code].append((sid, issn))
+
+    picked: dict[str, str] = {}
+    for code, rows in by_code.items():
+        # Take the strongest half, then sample within it, so the corpus is not
+        # entirely the same handful of megajournals per field.
+        pool = rows[: max(per_code * 3, per_code)]
+        rng.shuffle(pool)
+        for sid, issn in pool[:per_code]:
+            picked.setdefault(sid, issn)
+    return sorted(picked.items())
+
+
+# ---------------------------------------------------------------------------
+# Phase 1 — journal topics (centroids)
+# ---------------------------------------------------------------------------
+def phase1(con, mailto: str, out: Path, ck: Checkpoint) -> None:
+    keymap = issn_to_scopus(con)
+    seen_ids: set[str] = set()
+    if out.exists() and ck.state["phase"] > 1:
+        return
+    fh = out.open("a", encoding="utf-8")
+    if out.exists():
+        with out.open(encoding="utf-8") as f:
+            for line in f:
+                try:
+                    seen_ids.add(json.loads(line)["scopus_id"])
+                except (json.JSONDecodeError, KeyError):
+                    pass
+
+    cursor = ck.state.get("cursor") or "*"
+    page = 0
+    while cursor:
+        data = get(f"{API}/sources", {
+            "filter": "type:journal", "per-page": 200, "cursor": cursor,
+            "select": "id,display_name,issn,topics",
+        }, mailto)
+        results = data.get("results", [])
+        if not results:
+            break
+        for s in results:
+            sid = next((keymap[i] for i in (s.get("issn") or []) if i in keymap), None)
+            if not sid or sid in seen_ids:
+                continue
+            terms: list[str] = []
+            for t in (s.get("topics") or [])[:12]:
+                n = int(t.get("count", 1) or 1)
+                # Repeat proportional to prevalence so the embedding reflects
+                # what the journal mostly publishes, not its long tail.
+                terms += [t.get("display_name", "")] * max(1, min(4, n // 50))
+                for lvl in ("subfield", "field"):
+                    if (t.get(lvl) or {}).get("display_name"):
+                        terms.append(t[lvl]["display_name"])
+            if terms:
+                fh.write(json.dumps({"scopus_id": sid, "topics": " ".join(terms)}) + "\n")
+                seen_ids.add(sid)
+        fh.flush()
+        cursor = (data.get("meta") or {}).get("next_cursor")
+        ck.state["cursor"] = cursor
+        ck.save()
+        page += 1
+        if page % 10 == 0:
+            print(f"  phase1 page {page}  matched {len(seen_ids)}", flush=True)
+
+    fh.close()
+    ck.state.update(phase=2, cursor=None)
+    ck.save()
+    print(f"phase 1 done: {len(seen_ids)} journals with topics")
+
+
+# ---------------------------------------------------------------------------
+# Phase 2 — labelled abstracts (classifier)
+# ---------------------------------------------------------------------------
+def phase2(con, mailto: str, out: Path, ck: Checkpoint,
+           per_code: int, per_journal: int, seed: int, from_year: int) -> None:
+    targets = stratified_journals(con, per_code, seed)
+    done = set(ck.state["done_journals"])
+    todo = [(s, i) for s, i in targets if s not in done]
+    print(f"phase 2: {len(targets)} journals selected, {len(todo)} remaining")
+
+    fh = out.open("a", encoding="utf-8")
+    for n, (sid, issn) in enumerate(todo, 1):
+        fmt = f"{issn[:4]}-{issn[4:]}"
+        data = get(f"{API}/works", {
+            "filter": (f"primary_location.source.issn:{fmt},"
+                       f"from_publication_date:{from_year}-01-01,"
+                       "type:article,has_abstract:true"),
+            "per-page": per_journal,
+            "select": "title,abstract_inverted_index",
+        }, mailto)
+        wrote = 0
+        for w in data.get("results", []):
+            abstract = reconstruct_abstract(w.get("abstract_inverted_index"))
+            if len(abstract) < 200:
+                continue
+            text = f"{w.get('title') or ''}. {abstract}".strip()
+            fh.write(json.dumps({"scopus_id": sid, "text": text[:4000]}) + "\n")
+            wrote += 1
+        ck.state["n_works"] += wrote
+        done.add(sid)
+
+        if n % 25 == 0:
+            fh.flush()
+            ck.state["done_journals"] = sorted(done)
+            ck.save()
+            print(f"  phase2 {n}/{len(todo)} journals, "
+                  f"{ck.state['n_works']} abstracts", flush=True)
+        time.sleep(0.11)          # ~9 req/s, inside the polite-pool ceiling
+
+    fh.flush()
+    fh.close()
+    ck.state["done_journals"] = sorted(done)
+    ck.state["phase"] = 3
+    ck.save()
+    print(f"phase 2 done: {ck.state['n_works']} abstracts")
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--db", default="journals.db")
+    ap.add_argument("--mailto", required=True,
+                    help="your email — required for OpenAlex's polite pool")
+    ap.add_argument("--resume", action="store_true")
+    ap.add_argument("--skip-phase1", action="store_true")
+    ap.add_argument("--skip-phase2", action="store_true")
+    ap.add_argument("--journal-text", default="journal_text.jsonl")
+    ap.add_argument("--train-out", default="asjc_train.jsonl")
+    ap.add_argument("--per-code", type=int, default=25,
+                    help="journals sampled per ASJC code (stratification depth)")
+    ap.add_argument("--per-journal", type=int, default=40,
+                    help="abstracts pulled per journal")
+    ap.add_argument("--from-year", type=int, default=2021)
+    ap.add_argument("--seed", type=int, default=42)
+    ap.add_argument("--push-to-dataset", default=None,
+                    help="HF Dataset repo id to upload outputs to, e.g. user/researchbee-index. "
+                         "Required for Job runs — the container's disk does not survive it.")
+    args = ap.parse_args()
+
+    con = sqlite3.connect(args.db)
+    ck = Checkpoint(Path(args.train_out + ".state.json"), args.resume)
+
+    if not args.resume:
+        for p in (args.journal_text, args.train_out):
+            Path(p).unlink(missing_ok=True)
+
+    if not args.skip_phase1 and ck.state["phase"] <= 1:
+        phase1(con, args.mailto, Path(args.journal_text), ck)
+    if not args.skip_phase2 and ck.state["phase"] <= 2:
+        phase2(con, args.mailto, Path(args.train_out), ck,
+               args.per_code, args.per_journal, args.seed, args.from_year)
+
+    for p in (args.journal_text, args.train_out):
+        path = Path(p)
+        if path.exists():
+            n = sum(1 for _ in path.open(encoding="utf-8"))
+            print(f"{p}: {n} lines, {path.stat().st_size/1e6:.1f} MB")
+    con.close()
+
+    if args.push_to_dataset:
+        push(args.push_to_dataset, [args.journal_text, args.train_out,
+                                    args.train_out + ".state.json"])
+
+
+def push(repo_id: str, paths: list[str]) -> None:
+    """
+    Upload outputs to a Dataset repo. This is not a convenience — a Job's disk
+    is discarded when the run ends, so anything not pushed did not happen.
+    """
+    from huggingface_hub import HfApi
+    api = HfApi()
+    api.create_repo(repo_id, repo_type="dataset", exist_ok=True)
+    for p in paths:
+        path = Path(p)
+        if not path.exists():
+            continue
+        api.upload_file(
+            path_or_fileobj=str(path), path_in_repo=path.name,
+            repo_id=repo_id, repo_type="dataset",
+        )
+        print(f"pushed {path.name} -> {repo_id}")
+
+
+if __name__ == "__main__":
+    main()

journal_lookup.py

@@ -1,225 +1,225 @@
-"""
-Drop-in replacement for scimago_index, backed by journals.db.
-
-Swap ONE import in app.py and nothing else:
-
-    - from scimago_index import (lookup_any, load_scimago, search_by_subject, ...)
-    + from journal_lookup import (lookup_any, load_scimago, search_by_subject, ...)
-
-Keep the import list exactly as it is — however long it is, and whatever gets
-added to it later.
-
-Why it forwards everything instead of naming functions
--------------------------------------------------------
-The first version of this file re-exported four functions by name. That broke
-the moment `scimago_index` grew `subject_vocabulary`, `is_real_category`,
-`is_real_area` and `suggest_categories` — the import in app.py would have
-raised ImportError at startup and taken the Space down.
-
-So this module now forwards **every** attribute it does not define itself to
-`scimago_index`, via PEP 562 module `__getattr__`. Add ten more functions to
-`scimago_index` tomorrow and this file needs no change. The only name it
-deliberately overrides is `lookup_any`.
-
-That also means the subject-search work in `scimago_index` — the parsed
-category/area sets, the tiered match levels, `with_level` — passes through
-untouched, including its return types. This module has no opinion about it.
-
-Fail-safe by design
--------------------
-If journals.db is missing, unreadable or the wrong schema, every call falls
-back to the CSV path and the Space behaves exactly as it does today. This
-module never raises at import and never raises from a lookup.
-
-What lookup_any() gains
------------------------
-Same keys, same types, same values app.py already reads — plus:
-
-`quartiles_by_field`  Every (subject area, quartile) pair, not just the best.
-                      46% of ranked journals rank differently across their own
-                      fields. `quartile` still returns the best one, so nothing
-                      existing changes.
-`state` / `warn`      `warn` is True only for the 967 titles Scopus
-                      discontinued for cause — never for the 6,855 renamed.
-`successor_title`     Where a renamed journal continued.
-`oa_diamond`          No APC and no subscription.
-"""
-from __future__ import annotations
-
-import logging
-import os
-from typing import Optional
-
-import scimago_index
-
-logger = logging.getLogger(__name__)
-
-_DB_PATH = os.path.join(os.path.dirname(__file__), "journals.db")
-
-
-# ---------------------------------------------------------------------------
-# Forward every name we do not define to scimago_index.
-# ---------------------------------------------------------------------------
-def __getattr__(name: str):
-    """
-    PEP 562 module-level attribute hook. Anything app.py imports from here that
-    is not defined below is served from scimago_index unchanged — same object,
-    same signature, same behaviour. This is what makes the import swap safe
-    against future additions to scimago_index.
-    """
-    try:
-        return getattr(scimago_index, name)
-    except AttributeError:
-        raise AttributeError(
-            f"module 'journal_lookup' has no attribute {name!r}, and neither "
-            f"does 'scimago_index'"
-        ) from None
-
-
-def __dir__():
-    return sorted(set(globals()) | set(dir(scimago_index)))
-
-
-_csv_lookup_any = scimago_index.lookup_any
-
-
-# ---------------------------------------------------------------------------
-# Index bootstrap — never allowed to raise
-# ---------------------------------------------------------------------------
-_index = None
-_index_tried = False
-
-
-def _get_index():
-    global _index, _index_tried
-    if _index_tried:
-        return _index
-    _index_tried = True
-    try:
-        from researchbee_journals import JournalIndex
-        idx = JournalIndex(_DB_PATH)
-        meta = idx.meta()
-        logger.info(
-            "[JournalIndex] loaded %s journals (%s active, %s discontinued) built %s",
-            meta.get("n_journals"), meta.get("n_active"),
-            meta.get("n_discontinued"), meta.get("built_at", "")[:10],
-        )
-        print(f"[JournalIndex] loaded {meta.get('n_journals')} journals "
-              f"({meta.get('n_active')} active, {meta.get('n_discontinued')} "
-              f"discontinued) built {meta.get('built_at','')[:10]}", flush=True)
-        _index = idx
-    except Exception as e:                                    # noqa: BLE001
-        logger.warning("[JournalIndex] unavailable (%s) — falling back to CSV", e)
-        print(f"[JournalIndex] UNAVAILABLE ({e}) — falling back to CSV", flush=True)
-        _index = None
-    return _index
-
-
-def index_available() -> bool:
-    return _get_index() is not None
-
-
-# ---------------------------------------------------------------------------
-# Shape translation
-# ---------------------------------------------------------------------------
-def _to_csv_shape(rec, matched_issn: str) -> dict:
-    """
-    Render a JournalRecord in the dict shape app.py already consumes, then add
-    the new fields. The key names and value types below MUST match what
-    scimago_index.lookup_any returns — app.py reads .get("quartile"),
-    .get("rank"), .get("h_index"), .get("categories"), .get("publisher"),
-    .get("open_access"), .get("country"), .get("sourceid"), .get("matched_issn").
-    """
-    # "Hematology (Q1); Oncology (Q1)" — the exact format the CSV produced.
-    cats = "; ".join(
-        f"{s['name']} ({s['quartile']})" if s.get("quartile") else s["name"]
-        for s in rec.subjects if s.get("name")
-    )
-    areas = "; ".join(sorted({s["parent"] for s in rec.subjects if s.get("parent")}))
-
-    return {
-        # ---- keys app.py already uses (same names, same types) ----
-        "title":       rec.title or "",
-        "rank":        str(rec.sjr_rank) if rec.sjr_rank else "",
-        "quartile":    rec.sjr_best_quartile or "",
-        "h_index":     str(rec.h_index) if rec.h_index is not None else "",
-        "categories":  cats,
-        "areas":       areas,
-        "publisher":   rec.publisher_group or rec.publisher or "",
-        # SCImago's own OA flag first, for parity with the CSV path;
-        # Scopus/Unpaywall's flag as backup for titles SCImago does not rank.
-        "open_access": "Yes" if (rec.scimago_oa or rec.oa_diamond or rec.oa_unpaywall) else "No",
-        "country":     rec.country or "",
-        "sourceid":    rec.scimago_id or rec.scopus_id or "",
-        "matched_issn": matched_issn,
-
-        # ---- provenance, so the UI can say where each half came from ----
-        "from_scopus":  True,
-        "from_scimago": rec.sjr is not None,
-
-        # ---- new keys; app.py can adopt these incrementally ----
-        "quartiles_by_field": [
-            {"code": s["asjc"], "field": s["name"], "quartile": s["quartile"]}
-            for s in rec.subjects if s.get("quartile")
-        ],
-        "state":            rec.state,
-        "state_message":    rec.state_message,
-        "warn":             rec.warn,
-        "currently_indexed": rec.currently_indexed,
-        "successor_title":  rec.successor_title,
-        "oa_diamond":       rec.oa_diamond,
-        "sjr":              rec.sjr,
-        "coverage_start":   rec.coverage_start,
-        "coverage_end":     rec.coverage_end,
-        "source_type":      rec.source_type,
-        "scopus_id":        rec.scopus_id,
-        "medline":          rec.medline,
-    }
-
-
-def lookup_any(issns: list) -> Optional[dict]:
-    """
-    Same contract as scimago_index.lookup_any: first ISSN that resolves wins,
-    None if none do.
-    """
-    idx = _get_index()
-    if idx is None:
-        return _csv_lookup_any(issns)
-    try:
-        for issn in issns:
-            if not issn:
-                continue
-            rec = idx.lookup(issn=str(issn))
-            if rec:
-                from researchbee_journals import norm_issn
-                return _to_csv_shape(rec, norm_issn(str(issn)) or str(issn))
-    except Exception as e:                                    # noqa: BLE001
-        logger.warning("[JournalIndex] lookup failed (%s) — CSV fallback", e)
-        return _csv_lookup_any(issns)
-    # A miss must still try the CSV. journals.db is NOT a strict superset:
-    # ~700 SCImago records (mostly conference series) have no matching Scopus
-    # source record, so returning None here would silently LOSE coverage the
-    # app has today.
-    result = _csv_lookup_any(issns)
-    if result is not None:
-        result.setdefault("from_scopus", False)
-        result.setdefault("from_scimago", True)
-    return result
-
-
-def journal_detail(issn: str = "", title: str = "") -> Optional[dict]:
-    """Richer lookup for one journal. None when the index is unavailable."""
-    idx = _get_index()
-    if idx is None:
-        return None
-    try:
-        rec = idx.lookup(issn=issn or None, title=title or None)
-    except Exception:                                          # noqa: BLE001
-        return None
-    if not rec:
-        return None
-    d = _to_csv_shape(rec, rec.issn_e or rec.issn_print or "")
-    d["issn_print"] = rec.issn_print
-    d["issn_electronic"] = rec.issn_e
-    d["coverage_has_gap"] = rec.coverage_has_gap
+"""
+Drop-in replacement for scimago_index, backed by journals.db.
+
+Swap ONE import in app.py and nothing else:
+
+    - from scimago_index import (lookup_any, load_scimago, search_by_subject, ...)
+    + from journal_lookup import (lookup_any, load_scimago, search_by_subject, ...)
+
+Keep the import list exactly as it is — however long it is, and whatever gets
+added to it later.
+
+Why it forwards everything instead of naming functions
+-------------------------------------------------------
+The first version of this file re-exported four functions by name. That broke
+the moment `scimago_index` grew `subject_vocabulary`, `is_real_category`,
+`is_real_area` and `suggest_categories` — the import in app.py would have
+raised ImportError at startup and taken the Space down.
+
+So this module now forwards **every** attribute it does not define itself to
+`scimago_index`, via PEP 562 module `__getattr__`. Add ten more functions to
+`scimago_index` tomorrow and this file needs no change. The only name it
+deliberately overrides is `lookup_any`.
+
+That also means the subject-search work in `scimago_index` — the parsed
+category/area sets, the tiered match levels, `with_level` — passes through
+untouched, including its return types. This module has no opinion about it.
+
+Fail-safe by design
+-------------------
+If journals.db is missing, unreadable or the wrong schema, every call falls
+back to the CSV path and the Space behaves exactly as it does today. This
+module never raises at import and never raises from a lookup.
+
+What lookup_any() gains
+-----------------------
+Same keys, same types, same values app.py already reads — plus:
+
+`quartiles_by_field`  Every (subject area, quartile) pair, not just the best.
+                      46% of ranked journals rank differently across their own
+                      fields. `quartile` still returns the best one, so nothing
+                      existing changes.
+`state` / `warn`      `warn` is True only for the 967 titles Scopus
+                      discontinued for cause — never for the 6,855 renamed.
+`successor_title`     Where a renamed journal continued.
+`oa_diamond`          No APC and no subscription.
+"""
+from __future__ import annotations
+
+import logging
+import os
+from typing import Optional
+
+import scimago_index
+
+logger = logging.getLogger(__name__)
+
+_DB_PATH = os.path.join(os.path.dirname(__file__), "journals.db")
+
+
+# ---------------------------------------------------------------------------
+# Forward every name we do not define to scimago_index.
+# ---------------------------------------------------------------------------
+def __getattr__(name: str):
+    """
+    PEP 562 module-level attribute hook. Anything app.py imports from here that
+    is not defined below is served from scimago_index unchanged — same object,
+    same signature, same behaviour. This is what makes the import swap safe
+    against future additions to scimago_index.
+    """
+    try:
+        return getattr(scimago_index, name)
+    except AttributeError:
+        raise AttributeError(
+            f"module 'journal_lookup' has no attribute {name!r}, and neither "
+            f"does 'scimago_index'"
+        ) from None
+
+
+def __dir__():
+    return sorted(set(globals()) | set(dir(scimago_index)))
+
+
+_csv_lookup_any = scimago_index.lookup_any
+
+
+# ---------------------------------------------------------------------------
+# Index bootstrap — never allowed to raise
+# ---------------------------------------------------------------------------
+_index = None
+_index_tried = False
+
+
+def _get_index():
+    global _index, _index_tried
+    if _index_tried:
+        return _index
+    _index_tried = True
+    try:
+        from researchbee_journals import JournalIndex
+        idx = JournalIndex(_DB_PATH)
+        meta = idx.meta()
+        logger.info(
+            "[JournalIndex] loaded %s journals (%s active, %s discontinued) built %s",
+            meta.get("n_journals"), meta.get("n_active"),
+            meta.get("n_discontinued"), meta.get("built_at", "")[:10],
+        )
+        print(f"[JournalIndex] loaded {meta.get('n_journals')} journals "
+              f"({meta.get('n_active')} active, {meta.get('n_discontinued')} "
+              f"discontinued) built {meta.get('built_at','')[:10]}", flush=True)
+        _index = idx
+    except Exception as e:                                    # noqa: BLE001
+        logger.warning("[JournalIndex] unavailable (%s) — falling back to CSV", e)
+        print(f"[JournalIndex] UNAVAILABLE ({e}) — falling back to CSV", flush=True)
+        _index = None
+    return _index
+
+
+def index_available() -> bool:
+    return _get_index() is not None
+
+
+# ---------------------------------------------------------------------------
+# Shape translation
+# ---------------------------------------------------------------------------
+def _to_csv_shape(rec, matched_issn: str) -> dict:
+    """
+    Render a JournalRecord in the dict shape app.py already consumes, then add
+    the new fields. The key names and value types below MUST match what
+    scimago_index.lookup_any returns — app.py reads .get("quartile"),
+    .get("rank"), .get("h_index"), .get("categories"), .get("publisher"),
+    .get("open_access"), .get("country"), .get("sourceid"), .get("matched_issn").
+    """
+    # "Hematology (Q1); Oncology (Q1)" — the exact format the CSV produced.
+    cats = "; ".join(
+        f"{s['name']} ({s['quartile']})" if s.get("quartile") else s["name"]
+        for s in rec.subjects if s.get("name")
+    )
+    areas = "; ".join(sorted({s["parent"] for s in rec.subjects if s.get("parent")}))
+
+    return {
+        # ---- keys app.py already uses (same names, same types) ----
+        "title":       rec.title or "",
+        "rank":        str(rec.sjr_rank) if rec.sjr_rank else "",
+        "quartile":    rec.sjr_best_quartile or "",
+        "h_index":     str(rec.h_index) if rec.h_index is not None else "",
+        "categories":  cats,
+        "areas":       areas,
+        "publisher":   rec.publisher_group or rec.publisher or "",
+        # SCImago's own OA flag first, for parity with the CSV path;
+        # Scopus/Unpaywall's flag as backup for titles SCImago does not rank.
+        "open_access": "Yes" if (rec.scimago_oa or rec.oa_diamond or rec.oa_unpaywall) else "No",
+        "country":     rec.country or "",
+        "sourceid":    rec.scimago_id or rec.scopus_id or "",
+        "matched_issn": matched_issn,
+
+        # ---- provenance, so the UI can say where each half came from ----
+        "from_scopus":  True,
+        "from_scimago": rec.sjr is not None,
+
+        # ---- new keys; app.py can adopt these incrementally ----
+        "quartiles_by_field": [
+            {"code": s["asjc"], "field": s["name"], "quartile": s["quartile"]}
+            for s in rec.subjects if s.get("quartile")
+        ],
+        "state":            rec.state,
+        "state_message":    rec.state_message,
+        "warn":             rec.warn,
+        "currently_indexed": rec.currently_indexed,
+        "successor_title":  rec.successor_title,
+        "oa_diamond":       rec.oa_diamond,
+        "sjr":              rec.sjr,
+        "coverage_start":   rec.coverage_start,
+        "coverage_end":     rec.coverage_end,
+        "source_type":      rec.source_type,
+        "scopus_id":        rec.scopus_id,
+        "medline":          rec.medline,
+    }
+
+
+def lookup_any(issns: list) -> Optional[dict]:
+    """
+    Same contract as scimago_index.lookup_any: first ISSN that resolves wins,
+    None if none do.
+    """
+    idx = _get_index()
+    if idx is None:
+        return _csv_lookup_any(issns)
+    try:
+        for issn in issns:
+            if not issn:
+                continue
+            rec = idx.lookup(issn=str(issn))
+            if rec:
+                from researchbee_journals import norm_issn
+                return _to_csv_shape(rec, norm_issn(str(issn)) or str(issn))
+    except Exception as e:                                    # noqa: BLE001
+        logger.warning("[JournalIndex] lookup failed (%s) — CSV fallback", e)
+        return _csv_lookup_any(issns)
+    # A miss must still try the CSV. journals.db is NOT a strict superset:
+    # ~700 SCImago records (mostly conference series) have no matching Scopus
+    # source record, so returning None here would silently LOSE coverage the
+    # app has today.
+    result = _csv_lookup_any(issns)
+    if result is not None:
+        result.setdefault("from_scopus", False)
+        result.setdefault("from_scimago", True)
+    return result
+
+
+def journal_detail(issn: str = "", title: str = "") -> Optional[dict]:
+    """Richer lookup for one journal. None when the index is unavailable."""
+    idx = _get_index()
+    if idx is None:
+        return None
+    try:
+        rec = idx.lookup(issn=issn or None, title=title or None)
+    except Exception:                                          # noqa: BLE001
+        return None
+    if not rec:
+        return None
+    d = _to_csv_shape(rec, rec.issn_e or rec.issn_print or "")
+    d["issn_print"] = rec.issn_print
+    d["issn_electronic"] = rec.issn_e
+    d["coverage_has_gap"] = rec.coverage_has_gap
     return d

khazna_loader.py

@@ -1,136 +1,136 @@
-"""
-khazna_loader.py — Space-side loading of the Khazna index.
-
-The harvest runs in GitHub Actions (nightly, 23:00 Dubai) and publishes
-khazna_index.json to an HF Dataset repo. The Space downloads it — it never
-harvests, because HF Spaces have an ephemeral filesystem and a 20-minute
-harvest would block startup.
-
-Behaviour:
-    * startup      -> download once, load into memory (non-blocking)
-    * daily 23:30  -> re-download (a few seconds)
-    * unavailable  -> lookups return None, and the caller says "not checked"
-                      rather than implying the article is absent from Khazna.
-
-Add to requirements.txt:
-    huggingface_hub
-    apscheduler
-"""
-
-from __future__ import annotations
-
-import asyncio
-import json
-import logging
-import os
-from pathlib import Path
-from typing import Optional
-
-logger = logging.getLogger("khazna")
-
-HF_DATASET = os.getenv("KHAZNA_DATASET", "nikeshn/researchbee-data")
-INDEX_FILE = "khazna_index.json"
-LOCAL_PATH = Path("/tmp/khazna_index.json")
-
-
-class KhaznaIndex:
-    def __init__(self) -> None:
-        self._idx: dict[str, dict] = {}
-        self._loaded_at: Optional[str] = None
-
-    # ── loading ────────────────────────────────────────────────────────────
-    def _download(self) -> Optional[Path]:
-        try:
-            from huggingface_hub import hf_hub_download
-            p = hf_hub_download(
-                repo_id=HF_DATASET,
-                filename=INDEX_FILE,
-                repo_type="dataset",
-                token=os.getenv("HF_TOKEN"),   # omit if the dataset is public
-            )
-            return Path(p)
-        except Exception as e:
-            logger.warning("Khazna index download failed: %s", e)
-            return LOCAL_PATH if LOCAL_PATH.exists() else None
-
-    def reload(self) -> bool:
-        path = self._download()
-        if not path or not path.exists():
-            logger.warning("Khazna index unavailable — lookups will return None")
-            return False
-        try:
-            data = json.loads(path.read_text(encoding="utf-8"))
-        except json.JSONDecodeError as e:
-            logger.error("Khazna index malformed, keeping previous copy: %s", e)
-            return False
-        # Never replace a good index with an empty one.
-        if not data and self._idx:
-            logger.error("Downloaded index was empty — keeping previous copy")
-            return False
-        self._idx = data
-        from datetime import datetime, timezone
-        self._loaded_at = datetime.now(timezone.utc).isoformat(timespec="seconds")
-        logger.info("Khazna index loaded: %d DOIs", len(self._idx))
-        return True
-
-    # ── lookup ─────────────────────────────────────────────────────────────
-    def get(self, doi: str) -> Optional[dict]:
-        d = (doi or "").strip().lower()
-        for p in ("https://doi.org/", "http://doi.org/", "https://dx.doi.org/", "doi:"):
-            if d.startswith(p):
-                d = d[len(p):]
-                break
-        return self._idx.get(d.strip("/"))
-
-    @property
-    def available(self) -> bool:
-        return bool(self._idx)
-
-    @property
-    def status(self) -> dict:
-        return {"available": self.available,
-                "records": len(self._idx),
-                "loaded_at": self._loaded_at,
-                "source": HF_DATASET}
-
-
-khazna_index = KhaznaIndex()
-
-
-# ── FastAPI wiring (app.py) ────────────────────────────────────────────────
-#
-# from khazna_loader import khazna_index
-# from apscheduler.schedulers.asyncio import AsyncIOScheduler
-# from apscheduler.triggers.cron import CronTrigger
-#
-# @app.on_event("startup")
-# async def _startup():
-#     # Non-blocking: the Space serves traffic immediately; Khazna lookups
-#     # report "not checked" for the few seconds before the index lands.
-#     asyncio.create_task(asyncio.to_thread(khazna_index.reload))
-#
-#     sched = AsyncIOScheduler(timezone="Asia/Dubai")
-#     sched.add_job(lambda: asyncio.create_task(asyncio.to_thread(khazna_index.reload)),
-#                   CronTrigger(hour=23, minute=30))   # 30 min after the harvest
-#     sched.start()
-#
-# @app.get("/api/health/khazna")
-# async def khazna_health():
-#     return khazna_index.status
-#
-#
-# ── Using it in the license flow ───────────────────────────────────────────
-#
-# def khazna_status_for(doi: str) -> dict:
-#     """Three-valued, never guesses absence when the index is missing."""
-#     if not khazna_index.available:
-#         return {"checked": False,
-#                 "note": "Khazna could not be checked right now."}
-#     rec = khazna_index.get(doi)
-#     if rec is None:
-#         return {"checked": True, "in_khazna": False}
-#     return {"checked": True, "in_khazna": True,
-#             "deposit_state": rec["deposit_state"],       # open|embargoed|...|metadata_only
-#             "embargo_end":   rec.get("embargo_end"),
-#             "portal_url":    rec.get("portal_url"),
-#             "needs_deposit": rec["deposit_state"] in ("metadata_only", "closed")}
+"""
+khazna_loader.py — Space-side loading of the Khazna index.
+
+The harvest runs in GitHub Actions (nightly, 23:00 Dubai) and publishes
+khazna_index.json to an HF Dataset repo. The Space downloads it — it never
+harvests, because HF Spaces have an ephemeral filesystem and a 20-minute
+harvest would block startup.
+
+Behaviour:
+    * startup      -> download once, load into memory (non-blocking)
+    * daily 23:30  -> re-download (a few seconds)
+    * unavailable  -> lookups return None, and the caller says "not checked"
+                      rather than implying the article is absent from Khazna.
+
+Add to requirements.txt:
+    huggingface_hub
+    apscheduler
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import os
+from pathlib import Path
+from typing import Optional
+
+logger = logging.getLogger("khazna")
+
+HF_DATASET = os.getenv("KHAZNA_DATASET", "nikeshn/researchbee-data")
+INDEX_FILE = "khazna_index.json"
+LOCAL_PATH = Path("/tmp/khazna_index.json")
+
+
+class KhaznaIndex:
+    def __init__(self) -> None:
+        self._idx: dict[str, dict] = {}
+        self._loaded_at: Optional[str] = None
+
+    # ── loading ────────────────────────────────────────────────────────────
+    def _download(self) -> Optional[Path]:
+        try:
+            from huggingface_hub import hf_hub_download
+            p = hf_hub_download(
+                repo_id=HF_DATASET,
+                filename=INDEX_FILE,
+                repo_type="dataset",
+                token=os.getenv("HF_TOKEN"),   # omit if the dataset is public
+            )
+            return Path(p)
+        except Exception as e:
+            logger.warning("Khazna index download failed: %s", e)
+            return LOCAL_PATH if LOCAL_PATH.exists() else None
+
+    def reload(self) -> bool:
+        path = self._download()
+        if not path or not path.exists():
+            logger.warning("Khazna index unavailable — lookups will return None")
+            return False
+        try:
+            data = json.loads(path.read_text(encoding="utf-8"))
+        except json.JSONDecodeError as e:
+            logger.error("Khazna index malformed, keeping previous copy: %s", e)
+            return False
+        # Never replace a good index with an empty one.
+        if not data and self._idx:
+            logger.error("Downloaded index was empty — keeping previous copy")
+            return False
+        self._idx = data
+        from datetime import datetime, timezone
+        self._loaded_at = datetime.now(timezone.utc).isoformat(timespec="seconds")
+        logger.info("Khazna index loaded: %d DOIs", len(self._idx))
+        return True
+
+    # ── lookup ─────────────────────────────────────────────────────────────
+    def get(self, doi: str) -> Optional[dict]:
+        d = (doi or "").strip().lower()
+        for p in ("https://doi.org/", "http://doi.org/", "https://dx.doi.org/", "doi:"):
+            if d.startswith(p):
+                d = d[len(p):]
+                break
+        return self._idx.get(d.strip("/"))
+
+    @property
+    def available(self) -> bool:
+        return bool(self._idx)
+
+    @property
+    def status(self) -> dict:
+        return {"available": self.available,
+                "records": len(self._idx),
+                "loaded_at": self._loaded_at,
+                "source": HF_DATASET}
+
+
+khazna_index = KhaznaIndex()
+
+
+# ── FastAPI wiring (app.py) ────────────────────────────────────────────────
+#
+# from khazna_loader import khazna_index
+# from apscheduler.schedulers.asyncio import AsyncIOScheduler
+# from apscheduler.triggers.cron import CronTrigger
+#
+# @app.on_event("startup")
+# async def _startup():
+#     # Non-blocking: the Space serves traffic immediately; Khazna lookups
+#     # report "not checked" for the few seconds before the index lands.
+#     asyncio.create_task(asyncio.to_thread(khazna_index.reload))
+#
+#     sched = AsyncIOScheduler(timezone="Asia/Dubai")
+#     sched.add_job(lambda: asyncio.create_task(asyncio.to_thread(khazna_index.reload)),
+#                   CronTrigger(hour=23, minute=30))   # 30 min after the harvest
+#     sched.start()
+#
+# @app.get("/api/health/khazna")
+# async def khazna_health():
+#     return khazna_index.status
+#
+#
+# ── Using it in the license flow ───────────────────────────────────────────
+#
+# def khazna_status_for(doi: str) -> dict:
+#     """Three-valued, never guesses absence when the index is missing."""
+#     if not khazna_index.available:
+#         return {"checked": False,
+#                 "note": "Khazna could not be checked right now."}
+#     rec = khazna_index.get(doi)
+#     if rec is None:
+#         return {"checked": True, "in_khazna": False}
+#     return {"checked": True, "in_khazna": True,
+#             "deposit_state": rec["deposit_state"],       # open|embargoed|...|metadata_only
+#             "embargo_end":   rec.get("embargo_end"),
+#             "portal_url":    rec.get("portal_url"),
+#             "needs_deposit": rec["deposit_state"] in ("metadata_only", "closed")}

ku_knowledge.py

@@ -1,369 +1,369 @@
-"""
-KU Open Access Knowledge Base
-Extracted from: https://library.ku.ac.ae/oa (all tabs)
-Last extracted: May 2026
-Loaded once at startup and injected into CHAT_SYSTEM_PROMPT.
-"""
-
-KU_OA_KNOWLEDGE = """
-═══════════════════════════════════════════════════════════════
-KU LIBRARY — OPEN ACCESS PUBLISHING KNOWLEDGE BASE
-Source: https://library.ku.ac.ae/oa | Extracted May 2026
-═══════════════════════════════════════════════════════════════
-
-━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
-1. WHAT IS OPEN ACCESS?
-━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
-Open Access (OA) publishing means published work can be accessed
-by readers without a subscription or institutional access.
-Because readers do not pay, publishers generate revenue by charging
-the author an Article Processing Charge (APC).
-
-KU supports three OA models:
-
-GOLD OA:
-- Journal is completely Open Access
-- Publisher charges an APC to the author/institution
-- All articles in the journal are freely available
-
-HYBRID:
-- Journal has a mix of OA and closed-access articles
-- OA articles have an APC; traditional articles do not
-- Author chooses at submission whether to go OA
-
-GREEN OA:
-- Author/institution publishes a copy in an institutional repository
-- The repository copy is freely accessible even if the journal version is not
-- KU's institutional repository is Khazna (khazna.ku.ac.ae)
-
-Important: OA does not change the peer-review process. Articles must
-still be accepted and pass peer review before publication.
-
-━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
-2. KU OPEN ACCESS POLICY
-━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
-KU administration is committed to encouraging and supporting faculty
-to use Open Access publishing for their scholarly work.
-
-Full policy document:
-https://portal.ku.ac.ae/Policy/KU%20Poliy%20and%20Procedure%20Library/LIB%208540%20Open%20Access%20Publishing.pdf
-
-OA contact email: openaccess@ku.ac.ae
-OA guide: https://library.ku.ac.ae/oa/
-
-━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
-3. KU OA FUNDING OPTIONS — THREE TIERS
-━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
-
-TIER 1 — TOP 15% JOURNALS (Library pays APC only):
-  Conditions (ALL three must be met):
-  - At least one author's PRIMARY affiliation on the paper must be KU
-  - Paper must be in a journal from one of KU's 6 partner publishers:
-    ACS, Elsevier, IEEE, Springer Nature, Taylor & Francis, Wiley
-  - Journal must be rated in the TOP 15% by Scopus
-  
-  Publisher-specific OA process pages:
-  - ACS:            https://library.ku.ac.ae/oa/acs
-  - IEEE:           https://library.ku.ac.ae/oa/ieee
-  - Elsevier:       https://library.ku.ac.ae/oa/elsevier
-  - Springer Nature:https://library.ku.ac.ae/oa/SpringerNature
-  - Taylor & Francis:https://library.ku.ac.ae/oa/TandF
-  - Wiley:          https://library.ku.ac.ae/oa/wiley
-  - Check ratings:  https://library.ku.ac.ae/oa/ratings
-  
-  What is covered: Full basic APC cost
-  What is NOT covered: Overlength page charges, colour page charges,
-    or any charges beyond the basic APC
-  
-  How to apply: Process is handled automatically through the publisher
-    portal when the author selects KU as their institution.
-  
-  Contact: openaccess@ku.ac.ae
-
-TIER 2 — TOP 10% JOURNALS (APC + EPC via Provost office):
-  Conditions:
-  - Paper accepted in journal rated TOP 10% by Scopus
-  - KU must be author's primary affiliation
-  - Must have received email from library confirming they don't
-    cover these charges (i.e. not in Tier 1 agreement)
-  
-  How to apply:
-  - Submit Payment Request (PMR) through KU Portal:
-    https://kuconnect.ku.ac.ae/group/guest/services#/service-details/831545
-  - Attach: Invoice + Final accepted manuscript + Library rejection email
-  
-  Important notes:
-  - This is NOT a guarantee of approval — Provost office reviews each case
-  - Payments completed by finance team (their SLAs not in Provost control)
-  - Direct PMR only — expense reimbursement is NOT allowed
-
-TIER 3 — Q1 JOURNALS (Department budget):
-  Conditions:
-  - Neither Tier 1 nor Tier 2 applies
-  - Paper published in a Q1 journal (Scopus-determined)
-  
-  How to apply:
-  - Contact your department head to inquire about department budget support
-  - Direct PMR only — expense reimbursement is NOT allowed
-
-━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
-4. HOW KU DETERMINES JOURNAL RATINGS (TOP 15%)
-━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
-KU uses Scopus to determine journal ratings.
-- Ratings page: https://library.ku.ac.ae/oa/ratings
-
-Guidelines history:
-- From 24 October 2023: journals must be in top 20% by Scopus
-- From 1 January 2024 onwards: journals must be in top 15% by Scopus
-  (current requirement)
-
-How to interpret Scopus scores:
-https://service-elsevier-com.libconnect.ku.ac.ae/app/answers/detail/a_id/14834/supporthub/scopus/
-
-━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
-5. PUBLISHER-SPECIFIC PROCESSES
-━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
-
-── WILEY ──────────────────────────────────────────
-Agreement: Active. Covers Gold OA and Hybrid journals.
-Covered journals list (top 15%):
-  Excel: https://library.ku.ac.ae/ld.php?content_id=84913015
-  PDF: https://library.ku.ac.ae/ld.php?content_id=84913016
-
-Submission process:
-1. Author submits article via Wiley Author Center
-2. When asked who pays APC, select:
-   "My institution or funder has an agreement with Wiley and may pay"
-3. Select Khalifa University using the institutional lookup tool
-4. APC administrators notified — approved IF journal is top 15% (Scopus)
-5. Paper goes through editing and peer review
-6. If accepted, author acknowledges licence agreement
-7. Paper published
-
-Wiley Author Portal: http://www.wileyauthors.com/
-OA resources: https://www.wiley.com/network/researchers/licensing-and-open-access
-
-── ELSEVIER ──────────────────────────────────────
-Agreement: Active. Covers Gold OA and Hybrid journals.
-CRITICAL: Corresponding author MUST be affiliated with KU.
-  If corresponding author is from another university, that university
-  or the authors must pay the APC — KU cannot cover it.
-
-Covered journals list (top 5%):
-  Excel: https://library.ku.ac.ae/ld.php?content_id=84910855
-  PDF: https://library.ku.ac.ae/ld.php?content_id=84910856
-Note: Elsevier list covers top 5% (stricter than other publishers).
-
-Submission process:
-1. After acceptance, corresponding author receives email with link
-2. Click "Rights and Access" form link
-3. Corresponding author selects KU as affiliation
-4. Add co-authors and their affiliations
-5. Optionally add funder details
-6. Choose Gold Open Access or Subscription
-7. If OA chosen, select CC licence
-8. Accept agreement and submit
-9. KU administrators notified directly to pay the APC
-
-── SPRINGER NATURE ────────────────────────────────
-Agreement: Active from agreement date. Covers HYBRID journals only.
-IMPORTANT EXCLUSIONS:
-- Agreement covers hybrid journals ONLY — NOT pure Gold OA journals
-- Scientific Reports is specifically EXCLUDED from the agreement
-  (KU cannot pay APCs for Scientific Reports)
-- Corresponding author MUST be affiliated with KU
-
-Covered journals list (top 15%):
-  Excel: https://library.ku.ac.ae/ld.php?content_id=84909217
-  PDF: https://library.ku.ac.ae/ld.php?content_id=84909223
-
-Submission process:
-1. After editorial acceptance, corresponding author receives form
-2. Confirm country
-3. Confirm institutional affiliation (select KU)
-4. Publisher checks eligibility — confirmation given if eligible
-5. Author submits
-6. KU staff notified
-7. If journal meets KU guidelines → APC approved and paid
-8. If journal does NOT meet guidelines → request rejected,
-   authors must arrange alternative payment
-
-── TAYLOR & FRANCIS ───────────────────────────────
-Agreement: Active from 1 January 2023.
-Corresponding author MUST be affiliated with KU and use KU email
-address — this allows publisher to automatically request APC from KU.
-If corresponding author not affiliated with KU, KU cannot pay.
-
-Note: Journal list currently being updated to reflect newest Scopus ratings.
-
-Helpful T&F author resources:
-- Why OA: https://authorservices.taylorandfrancis.com/choose-open/publishing-open-access/
-- Writing advice: https://authorservices.taylorandfrancis.com/publishing-your-research/writing-your-paper/
-- Submission info: https://authorservices.taylorandfrancis.com/publishing-your-research/making-your-submission/
-
-── ACS (American Chemical Society) ────────────────
-Agreement: Active. 42 journals meet KU guidelines.
-NOTE: ACS Omega is NOT rated in top 15% and is NOT eligible for KU OA payment.
-
-Covered journals list (top 15%):
-  Excel: https://library.ku.ac.ae/ld.php?content_id=75321482
-  PDF: https://library.ku.ac.ae/ld.php?content_id=75321479
-
-Submission process:
-1. Submit article via ACS Publication Center: https://publish.acs.org/publish/
-2. When asked who pays APC, select that the university may pay
-3. Select Khalifa University using institutional lookup
-4. APC administrators notified
-   - If journal is top 15% (Scopus) → APC approved
-   - If NOT top 15% → funding request rejected
-5. Paper goes through editing and peer review
-6. If accepted, author acknowledges licence agreement
-7. Paper published
-
-ACS tools: https://publish.acs.org/publish/
-ACS OA info: https://acsopenaccess.org/
-
-── IEEE ────────────────────────────────────────────
-Agreement: Active. KU has agreement with IEEE.
-Contact openaccess@ku.ac.ae for current IEEE journal list and process details.
-IEEE journal list: https://library.ku.ac.ae/oa/ieee
-
-━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
-6. KHAZNA — KU INSTITUTIONAL REPOSITORY
-━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
-URL: https://khazna.ku.ac.ae
-Contact: khazna@ku.ac.ae
-
-Khazna is KU's institutional repository (Green OA).
-- All KU published articles should be deposited in Khazna
-- Even under embargo, deposit the metadata record immediately
-- Full text deposited once embargo expires
-- Deposit increases visibility of KU research globally
-- Supports KU OA compliance
-
-━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
-7. RESEARCHBEE TOOLS — WHAT EACH DOES
-━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
-ResearchBee has THREE tools available as tabs on the main page:
-
-TOOL 1 — JOURNAL SUBMISSION TAB:
-  Two modes:
-  a) Browse by Subject: Type a discipline → get top 15 ranked journals
-     instantly from 53,000+ indexed titles. No manuscript needed.
-  b) Analyse Manuscript: Paste abstract → AI matches journals with full
-     Green OA policy breakdown, submission strategy, cover letter generation.
-  Use this for: finding where to publish, journal rankings, OA compliance.
-
-TOOL 2 — LICENSE CHECKING TAB:
-  Look up self-archiving rights for a specific known journal.
-  Gives: version-specific permissions (preprint/AAM/VoR), embargo periods,
-  licence conditions, repository restrictions.
-  Use this for: "Can I deposit my accepted manuscript in Khazna?"
-  or "What's the embargo on this journal?"
-
-TOOL 3 — DATA REPOSITORY TAB:
-  Describe a dataset → get matched repositories (domain-specific +
-  generalist like Zenodo, Dryad) + Khazna.
-  Gives: FAIR alignment, licence support, embargo support, cost, DOI provision.
-  Use this for: "Where should I deposit my research data?"
-
-━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
-8. KEY CONTACTS & SUPPORT
-━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
-OA Services:          openaccess@ku.ac.ae
-Khazna Repository:    khazna@ku.ac.ae
-Library general:      library@ku.ac.ae
-KU Library:           https://library.ku.ac.ae/lib
-OA Guide:             https://library.ku.ac.ae/oa
-
-RESEARCH SUPPORT — PRIMARY CONTACT:
-  Nikesh Narayanan, Research Librarian
-  Email: nikesh.narayanan@ku.ac.ae
-  For: journal selection advice, OA publishing support, research data management,
-       data management plans, repository selection, FAIR data guidance,
-       any publishing or research support question not answered by the tools.
-
-PUBLISHER RECOMMENDATION FORM:
-  URL: https://kustar.libwizard.com/f/oapub
-  Purpose: Use this form to suggest a publisher or journal to be added to
-  KU Library's OA APC agreement. If a researcher wants KU to negotiate
-  an APC agreement with a publisher not currently covered, they submit
-  this form. The library reviews all suggestions.
-
-━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
-8b. KU JOURNAL FINDER TOOLS GUIDE
-━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
-KU Library maintains a comprehensive guide to journal finder tools:
-URL: https://library.ku.ac.ae/journalfinder
-Created by: Nikesh Narayanan, Research Librarian
-
-The guide covers:
-- ResearchBee (KU's own AI tool — recommended starting point)
-- Cross-publisher AI matchers: EndNote Find a Journal, Trinka, JournalGuide, Paperpile, JANE, Editage
-- Publisher-specific finders: Elsevier, Springer Nature, Wiley, Taylor & Francis, IEEE, SAGE, Emerald, MDPI
-- Indexing & verification tools: Scopus Sources, WoS Master Journal List, DOAJ, SCImago, Think.Check.Submit
-- Open access journal finders: Enago OA Journal Finder, DOAJ, PLOS
-
-When to direct users there:
-- "What tools can I use to find a journal?"
-- "How do I choose the right journal for my paper?"
-- "Are there other journal recommendation tools besides ResearchBee?"
-- "Which Elsevier/Wiley/Springer journal should I target?"
-- "How do I check if a journal is indexed?"
-- "Is this journal trustworthy / predatory?"
-
-━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
-9. RESEARCH ANALYTICS TOOLS (for citation & performance analysis)
-━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
-For deep citation analysis, journal impact metrics, author h-index,
-research performance, and institutional analytics — direct researchers
-to these tools (all accessible via KU Library proxy):
-
-SCOPUS (Elsevier):
-  Access: http://login.khalifa.idm.oclc.org/login?url=https://www.scopus.com
-  Best for: Journal rankings, citation counts, author profiles, h-index,
-            journal source metrics (CiteScore, SJR, SNIP)
-
-WEB OF SCIENCE (Clarivate):
-  Access: http://login.khalifa.idm.oclc.org/login?url=https://www.webofscience.com
-  Best for: Citation reports, impact factor, Journal Citation Reports (JCR),
-            researcher profiles, highly cited papers
-
-SCIVAL (Elsevier):
-  Access: http://login.khalifa.idm.oclc.org/login?url=https://www.scival.com
-  Best for: Institutional research performance, benchmarking KU against
-            other institutions, field-weighted citation impact (FWCI),
-            collaboration networks, research trends by topic
-
-When to direct users there:
-- "What is my h-index?" → Scopus or Web of Science
-- "How does KU compare to other UAE universities?" → SciVal
-- "What is the impact factor of this journal?" → Web of Science JCR
-- "Show me citation trends for my field" → Scopus or SciVal
-- "Who are the top authors in my field?" → Scopus
-- "Detailed journal metrics beyond what ResearchBee shows" → Scopus Sources
-
-━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
-10. APPROVED LINKS (chatbot may only use these)
-━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
-- KU Journal Finder Guide:  https://library.ku.ac.ae/journalfinder
-- KU OA guide:              https://library.ku.ac.ae/oa
-- KU Library:               https://library.ku.ac.ae/lib
-- Khazna repository:        https://khazna.ku.ac.ae
-- OA policy document:       https://portal.ku.ac.ae/Policy/KU%20Poliy%20and%20Procedure%20Library/LIB%208540%20Open%20Access%20Publishing.pdf
-- Publisher recommendation: https://kustar.libwizard.com/f/oapub
-- Open Policy Finder:       https://openpolicyfinder.jisc.ac.uk
-- re3data (repositories):   https://www.re3data.org
-- FAIRsharing:              https://fairsharing.org
-- Research Librarian:       nikesh.narayanan@ku.ac.ae
-- OA services email:        openaccess@ku.ac.ae
-- Khazna contact:           khazna@ku.ac.ae
-- Library email:            library@ku.ac.ae
-- Scopus (KU proxy):        http://login.khalifa.idm.oclc.org/login?url=https://www.scopus.com
-- Web of Science (KU proxy):http://login.khalifa.idm.oclc.org/login?url=https://www.webofscience.com
-- SciVal (KU proxy):        http://login.khalifa.idm.oclc.org/login?url=https://www.scival.com
-""".strip()
-
-
-def get_knowledge() -> str:
-    return KU_OA_KNOWLEDGE
+"""
+KU Open Access Knowledge Base
+Extracted from: https://library.ku.ac.ae/oa (all tabs)
+Last extracted: May 2026
+Loaded once at startup and injected into CHAT_SYSTEM_PROMPT.
+"""
+
+KU_OA_KNOWLEDGE = """
+═══════════════════════════════════════════════════════════════
+KU LIBRARY — OPEN ACCESS PUBLISHING KNOWLEDGE BASE
+Source: https://library.ku.ac.ae/oa | Extracted May 2026
+═══════════════════════════════════════════════════════════════
+
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+1. WHAT IS OPEN ACCESS?
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+Open Access (OA) publishing means published work can be accessed
+by readers without a subscription or institutional access.
+Because readers do not pay, publishers generate revenue by charging
+the author an Article Processing Charge (APC).
+
+KU supports three OA models:
+
+GOLD OA:
+- Journal is completely Open Access
+- Publisher charges an APC to the author/institution
+- All articles in the journal are freely available
+
+HYBRID:
+- Journal has a mix of OA and closed-access articles
+- OA articles have an APC; traditional articles do not
+- Author chooses at submission whether to go OA
+
+GREEN OA:
+- Author/institution publishes a copy in an institutional repository
+- The repository copy is freely accessible even if the journal version is not
+- KU's institutional repository is Khazna (khazna.ku.ac.ae)
+
+Important: OA does not change the peer-review process. Articles must
+still be accepted and pass peer review before publication.
+
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+2. KU OPEN ACCESS POLICY
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+KU administration is committed to encouraging and supporting faculty
+to use Open Access publishing for their scholarly work.
+
+Full policy document:
+https://portal.ku.ac.ae/Policy/KU%20Poliy%20and%20Procedure%20Library/LIB%208540%20Open%20Access%20Publishing.pdf
+
+OA contact email: openaccess@ku.ac.ae
+OA guide: https://library.ku.ac.ae/oa/
+
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+3. KU OA FUNDING OPTIONS — THREE TIERS
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+
+TIER 1 — TOP 15% JOURNALS (Library pays APC only):
+  Conditions (ALL three must be met):
+  - At least one author's PRIMARY affiliation on the paper must be KU
+  - Paper must be in a journal from one of KU's 6 partner publishers:
+    ACS, Elsevier, IEEE, Springer Nature, Taylor & Francis, Wiley
+  - Journal must be rated in the TOP 15% by Scopus
+  
+  Publisher-specific OA process pages:
+  - ACS:            https://library.ku.ac.ae/oa/acs
+  - IEEE:           https://library.ku.ac.ae/oa/ieee
+  - Elsevier:       https://library.ku.ac.ae/oa/elsevier
+  - Springer Nature:https://library.ku.ac.ae/oa/SpringerNature
+  - Taylor & Francis:https://library.ku.ac.ae/oa/TandF
+  - Wiley:          https://library.ku.ac.ae/oa/wiley
+  - Check ratings:  https://library.ku.ac.ae/oa/ratings
+  
+  What is covered: Full basic APC cost
+  What is NOT covered: Overlength page charges, colour page charges,
+    or any charges beyond the basic APC
+  
+  How to apply: Process is handled automatically through the publisher
+    portal when the author selects KU as their institution.
+  
+  Contact: openaccess@ku.ac.ae
+
+TIER 2 — TOP 10% JOURNALS (APC + EPC via Provost office):
+  Conditions:
+  - Paper accepted in journal rated TOP 10% by Scopus
+  - KU must be author's primary affiliation
+  - Must have received email from library confirming they don't
+    cover these charges (i.e. not in Tier 1 agreement)
+  
+  How to apply:
+  - Submit Payment Request (PMR) through KU Portal:
+    https://kuconnect.ku.ac.ae/group/guest/services#/service-details/831545
+  - Attach: Invoice + Final accepted manuscript + Library rejection email
+  
+  Important notes:
+  - This is NOT a guarantee of approval — Provost office reviews each case
+  - Payments completed by finance team (their SLAs not in Provost control)
+  - Direct PMR only — expense reimbursement is NOT allowed
+
+TIER 3 — Q1 JOURNALS (Department budget):
+  Conditions:
+  - Neither Tier 1 nor Tier 2 applies
+  - Paper published in a Q1 journal (Scopus-determined)
+  
+  How to apply:
+  - Contact your department head to inquire about department budget support
+  - Direct PMR only — expense reimbursement is NOT allowed
+
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+4. HOW KU DETERMINES JOURNAL RATINGS (TOP 15%)
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+KU uses Scopus to determine journal ratings.
+- Ratings page: https://library.ku.ac.ae/oa/ratings
+
+Guidelines history:
+- From 24 October 2023: journals must be in top 20% by Scopus
+- From 1 January 2024 onwards: journals must be in top 15% by Scopus
+  (current requirement)
+
+How to interpret Scopus scores:
+https://service-elsevier-com.libconnect.ku.ac.ae/app/answers/detail/a_id/14834/supporthub/scopus/
+
+━━━━━━━━━━━━━━━━━━━━━━━���━━━━━━━━━━━━━━━━━━━━━━━━━
+5. PUBLISHER-SPECIFIC PROCESSES
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+
+── WILEY ──────────────────────────────────────────
+Agreement: Active. Covers Gold OA and Hybrid journals.
+Covered journals list (top 15%):
+  Excel: https://library.ku.ac.ae/ld.php?content_id=84913015
+  PDF: https://library.ku.ac.ae/ld.php?content_id=84913016
+
+Submission process:
+1. Author submits article via Wiley Author Center
+2. When asked who pays APC, select:
+   "My institution or funder has an agreement with Wiley and may pay"
+3. Select Khalifa University using the institutional lookup tool
+4. APC administrators notified — approved IF journal is top 15% (Scopus)
+5. Paper goes through editing and peer review
+6. If accepted, author acknowledges licence agreement
+7. Paper published
+
+Wiley Author Portal: http://www.wileyauthors.com/
+OA resources: https://www.wiley.com/network/researchers/licensing-and-open-access
+
+── ELSEVIER ──────────────────────────────────────
+Agreement: Active. Covers Gold OA and Hybrid journals.
+CRITICAL: Corresponding author MUST be affiliated with KU.
+  If corresponding author is from another university, that university
+  or the authors must pay the APC — KU cannot cover it.
+
+Covered journals list (top 5%):
+  Excel: https://library.ku.ac.ae/ld.php?content_id=84910855
+  PDF: https://library.ku.ac.ae/ld.php?content_id=84910856
+Note: Elsevier list covers top 5% (stricter than other publishers).
+
+Submission process:
+1. After acceptance, corresponding author receives email with link
+2. Click "Rights and Access" form link
+3. Corresponding author selects KU as affiliation
+4. Add co-authors and their affiliations
+5. Optionally add funder details
+6. Choose Gold Open Access or Subscription
+7. If OA chosen, select CC licence
+8. Accept agreement and submit
+9. KU administrators notified directly to pay the APC
+
+── SPRINGER NATURE ────────────────────────────────
+Agreement: Active from agreement date. Covers HYBRID journals only.
+IMPORTANT EXCLUSIONS:
+- Agreement covers hybrid journals ONLY — NOT pure Gold OA journals
+- Scientific Reports is specifically EXCLUDED from the agreement
+  (KU cannot pay APCs for Scientific Reports)
+- Corresponding author MUST be affiliated with KU
+
+Covered journals list (top 15%):
+  Excel: https://library.ku.ac.ae/ld.php?content_id=84909217
+  PDF: https://library.ku.ac.ae/ld.php?content_id=84909223
+
+Submission process:
+1. After editorial acceptance, corresponding author receives form
+2. Confirm country
+3. Confirm institutional affiliation (select KU)
+4. Publisher checks eligibility — confirmation given if eligible
+5. Author submits
+6. KU staff notified
+7. If journal meets KU guidelines → APC approved and paid
+8. If journal does NOT meet guidelines → request rejected,
+   authors must arrange alternative payment
+
+── TAYLOR & FRANCIS ───────────────────────────────
+Agreement: Active from 1 January 2023.
+Corresponding author MUST be affiliated with KU and use KU email
+address — this allows publisher to automatically request APC from KU.
+If corresponding author not affiliated with KU, KU cannot pay.
+
+Note: Journal list currently being updated to reflect newest Scopus ratings.
+
+Helpful T&F author resources:
+- Why OA: https://authorservices.taylorandfrancis.com/choose-open/publishing-open-access/
+- Writing advice: https://authorservices.taylorandfrancis.com/publishing-your-research/writing-your-paper/
+- Submission info: https://authorservices.taylorandfrancis.com/publishing-your-research/making-your-submission/
+
+── ACS (American Chemical Society) ────────────────
+Agreement: Active. 42 journals meet KU guidelines.
+NOTE: ACS Omega is NOT rated in top 15% and is NOT eligible for KU OA payment.
+
+Covered journals list (top 15%):
+  Excel: https://library.ku.ac.ae/ld.php?content_id=75321482
+  PDF: https://library.ku.ac.ae/ld.php?content_id=75321479
+
+Submission process:
+1. Submit article via ACS Publication Center: https://publish.acs.org/publish/
+2. When asked who pays APC, select that the university may pay
+3. Select Khalifa University using institutional lookup
+4. APC administrators notified
+   - If journal is top 15% (Scopus) → APC approved
+   - If NOT top 15% → funding request rejected
+5. Paper goes through editing and peer review
+6. If accepted, author acknowledges licence agreement
+7. Paper published
+
+ACS tools: https://publish.acs.org/publish/
+ACS OA info: https://acsopenaccess.org/
+
+���─ IEEE ────────────────────────────────────────────
+Agreement: Active. KU has agreement with IEEE.
+Contact openaccess@ku.ac.ae for current IEEE journal list and process details.
+IEEE journal list: https://library.ku.ac.ae/oa/ieee
+
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+6. KHAZNA — KU INSTITUTIONAL REPOSITORY
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+URL: https://khazna.ku.ac.ae
+Contact: khazna@ku.ac.ae
+
+Khazna is KU's institutional repository (Green OA).
+- All KU published articles should be deposited in Khazna
+- Even under embargo, deposit the metadata record immediately
+- Full text deposited once embargo expires
+- Deposit increases visibility of KU research globally
+- Supports KU OA compliance
+
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+7. RESEARCHBEE TOOLS — WHAT EACH DOES
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+ResearchBee has THREE tools available as tabs on the main page:
+
+TOOL 1 — JOURNAL SUBMISSION TAB:
+  Two modes:
+  a) Browse by Subject: Type a discipline → get top 15 ranked journals
+     instantly from 53,000+ indexed titles. No manuscript needed.
+  b) Analyse Manuscript: Paste abstract → AI matches journals with full
+     Green OA policy breakdown, submission strategy, cover letter generation.
+  Use this for: finding where to publish, journal rankings, OA compliance.
+
+TOOL 2 — LICENSE CHECKING TAB:
+  Look up self-archiving rights for a specific known journal.
+  Gives: version-specific permissions (preprint/AAM/VoR), embargo periods,
+  licence conditions, repository restrictions.
+  Use this for: "Can I deposit my accepted manuscript in Khazna?"
+  or "What's the embargo on this journal?"
+
+TOOL 3 — DATA REPOSITORY TAB:
+  Describe a dataset → get matched repositories (domain-specific +
+  generalist like Zenodo, Dryad) + Khazna.
+  Gives: FAIR alignment, licence support, embargo support, cost, DOI provision.
+  Use this for: "Where should I deposit my research data?"
+
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+8. KEY CONTACTS & SUPPORT
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+OA Services:          openaccess@ku.ac.ae
+Khazna Repository:    khazna@ku.ac.ae
+Library general:      library@ku.ac.ae
+KU Library:           https://library.ku.ac.ae/lib
+OA Guide:             https://library.ku.ac.ae/oa
+
+RESEARCH SUPPORT — PRIMARY CONTACT:
+  Nikesh Narayanan, Research Librarian
+  Email: nikesh.narayanan@ku.ac.ae
+  For: journal selection advice, OA publishing support, research data management,
+       data management plans, repository selection, FAIR data guidance,
+       any publishing or research support question not answered by the tools.
+
+PUBLISHER RECOMMENDATION FORM:
+  URL: https://kustar.libwizard.com/f/oapub
+  Purpose: Use this form to suggest a publisher or journal to be added to
+  KU Library's OA APC agreement. If a researcher wants KU to negotiate
+  an APC agreement with a publisher not currently covered, they submit
+  this form. The library reviews all suggestions.
+
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+8b. KU JOURNAL FINDER TOOLS GUIDE
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+KU Library maintains a comprehensive guide to journal finder tools:
+URL: https://library.ku.ac.ae/journalfinder
+Created by: Nikesh Narayanan, Research Librarian
+
+The guide covers:
+- ResearchBee (KU's own AI tool — recommended starting point)
+- Cross-publisher AI matchers: EndNote Find a Journal, Trinka, JournalGuide, Paperpile, JANE, Editage
+- Publisher-specific finders: Elsevier, Springer Nature, Wiley, Taylor & Francis, IEEE, SAGE, Emerald, MDPI
+- Indexing & verification tools: Scopus Sources, WoS Master Journal List, DOAJ, SCImago, Think.Check.Submit
+- Open access journal finders: Enago OA Journal Finder, DOAJ, PLOS
+
+When to direct users there:
+- "What tools can I use to find a journal?"
+- "How do I choose the right journal for my paper?"
+- "Are there other journal recommendation tools besides ResearchBee?"
+- "Which Elsevier/Wiley/Springer journal should I target?"
+- "How do I check if a journal is indexed?"
+- "Is this journal trustworthy / predatory?"
+
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+9. RESEARCH ANALYTICS TOOLS (for citation & performance analysis)
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+For deep citation analysis, journal impact metrics, author h-index,
+research performance, and institutional analytics — direct researchers
+to these tools (all accessible via KU Library proxy):
+
+SCOPUS (Elsevier):
+  Access: http://login.khalifa.idm.oclc.org/login?url=https://www.scopus.com
+  Best for: Journal rankings, citation counts, author profiles, h-index,
+            journal source metrics (CiteScore, SJR, SNIP)
+
+WEB OF SCIENCE (Clarivate):
+  Access: http://login.khalifa.idm.oclc.org/login?url=https://www.webofscience.com
+  Best for: Citation reports, impact factor, Journal Citation Reports (JCR),
+            researcher profiles, highly cited papers
+
+SCIVAL (Elsevier):
+  Access: http://login.khalifa.idm.oclc.org/login?url=https://www.scival.com
+  Best for: Institutional research performance, benchmarking KU against
+            other institutions, field-weighted citation impact (FWCI),
+            collaboration networks, research trends by topic
+
+When to direct users there:
+- "What is my h-index?" → Scopus or Web of Science
+- "How does KU compare to other UAE universities?" → SciVal
+- "What is the impact factor of this journal?" → Web of Science JCR
+- "Show me citation trends for my field" → Scopus or SciVal
+- "Who are the top authors in my field?" → Scopus
+- "Detailed journal metrics beyond what ResearchBee shows" → Scopus Sources
+
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+10. APPROVED LINKS (chatbot may only use these)
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+- KU Journal Finder Guide:  https://library.ku.ac.ae/journalfinder
+- KU OA guide:              https://library.ku.ac.ae/oa
+- KU Library:               https://library.ku.ac.ae/lib
+- Khazna repository:        https://khazna.ku.ac.ae
+- OA policy document:       https://portal.ku.ac.ae/Policy/KU%20Poliy%20and%20Procedure%20Library/LIB%208540%20Open%20Access%20Publishing.pdf
+- Publisher recommendation: https://kustar.libwizard.com/f/oapub
+- Open Policy Finder:       https://openpolicyfinder.jisc.ac.uk
+- re3data (repositories):   https://www.re3data.org
+- FAIRsharing:              https://fairsharing.org
+- Research Librarian:       nikesh.narayanan@ku.ac.ae
+- OA services email:        openaccess@ku.ac.ae
+- Khazna contact:           khazna@ku.ac.ae
+- Library email:            library@ku.ac.ae
+- Scopus (KU proxy):        http://login.khalifa.idm.oclc.org/login?url=https://www.scopus.com
+- Web of Science (KU proxy):http://login.khalifa.idm.oclc.org/login?url=https://www.webofscience.com
+- SciVal (KU proxy):        http://login.khalifa.idm.oclc.org/login?url=https://www.scival.com
+""".strip()
+
+
+def get_knowledge() -> str:
+    return KU_OA_KNOWLEDGE

oaworks.py

@@ -153,46 +153,87 @@ def _licence_of(perm: dict) -> str:
     return ", ".join(types)
 
 
-def _embargo_months(perm: dict) -> Optional[int]:
-    """Raw embargo in months, or None when the record states none."""
-    for k in ("embargo_months", "embargoMonths"):
-        v = perm.get(k)
-        if isinstance(v, (int, float)) and v > 0:
-            return int(v)
-        if isinstance(v, str) and v.strip().isdigit() and int(v) > 0:
-            return int(v)
+def _normalise_embargo_months(value) -> Optional[int]:
+    """Return a non-negative month value only when the source states one."""
+    if isinstance(value, bool):
+        return None
+    if isinstance(value, (int, float)) and value >= 0:
+        return int(value)
+    if isinstance(value, str):
+        raw = value.strip().lower()
+        if raw in {"none", "no embargo", "immediate", "0", "0.0"}:
+            return 0
+        match = _re.fullmatch(r"(\d+)\s*(?:months?|mos?)?", raw)
+        if match:
+            return int(match.group(1))
     return None
 
 
-def _embargo_of(perm: dict) -> tuple[str, Optional[int]]:
+def _embargo_details(perm: dict) -> tuple[str, Optional[int], str]:
     """
-    Returns (display_text, months_or_None).
+    Return ``(display_text, months, status)`` without collapsing unknown to zero.
 
-    Some publishers (notably T&F) record no embargo at all. We say
-    "not stated" rather than "none" — see the module docstring.
+    ``status`` is one of:
+      * ``zero``    — the permission record explicitly states zero/no embargo;
+      * ``stated``  — a positive duration or article-specific end date is stated;
+      * ``unknown`` — the record contains no usable embargo statement.
     """
-    m = _embargo_months(perm)
-    if m is None:
-        return "Not stated in the permission record", None
-    end = perm.get("embargo_end")
-    txt = f"{m} months"
+    for key in ("embargo_months", "embargoMonths"):
+        if key not in perm:
+            continue
+        months = _normalise_embargo_months(perm.get(key))
+        if months is None:
+            continue
+        if months == 0:
+            return "No embargo (0 months stated in the permission record)", 0, "zero"
+        end = perm.get("embargo_end")
+        text = f"{months} months"
+        if end:
+            text += f" (ends {end} for this article)"
+        return text, months, "stated"
+
+    # Some responses may use a generic embargo value rather than the confirmed
+    # month keys. Accept only explicit, machine-readable values.
+    if "embargo" in perm and not isinstance(perm.get("embargo"), dict):
+        months = _normalise_embargo_months(perm.get("embargo"))
+        if months is not None:
+            if months == 0:
+                return "No embargo (0 months stated in the permission record)", 0, "zero"
+            end = perm.get("embargo_end")
+            text = f"{months} months"
+            if end:
+                text += f" (ends {end} for this article)"
+            return text, months, "stated"
+
+    end = str(perm.get("embargo_end") or "").strip()
     if end:
-        txt += f" (ends {end} for this article)"
-    return txt, m
+        return f"Embargo ends {end} for this article", None, "stated"
+
+    return "Not stated in the permission record", None, "unknown"
+
+
+def _embargo_months(perm: dict) -> Optional[int]:
+    """Compatibility helper: 0, positive months, or None when not stated."""
+    return _embargo_details(perm)[1]
+
+
+def _embargo_of(perm: dict) -> tuple[str, Optional[int]]:
+    text, months, _status = _embargo_details(perm)
+    return text, months
+
+
+def _embargo_status(perm: dict) -> str:
+    return _embargo_details(perm)[2]
 
 
 def _permission_rank(perm: dict) -> tuple:
-    """
-    Sort key for choosing between competing permissions.
-
-    CRITICAL: Elsevier publishes DIFFERENT embargoes for UK institutions
-    (12 months) and everyone else (24 months), and both records carry the
-    SAME score. Ranking by score alone can silently select the shorter UK
-    embargo for a KU author. We therefore break score ties by the LONGER
-    embargo — the conservative choice, and the one OA.Works' own
-    best_permission makes.
-    """
-    return (perm.get("score", 0), _embargo_months(perm) or 0)
+    """Conservative sort key for competing permissions of equal source score."""
+    _text, months, status = _embargo_details(perm)
+    # Prefer a known positive/end-dated embargo, then an unknown embargo, then
+    # an explicit zero embargo. This avoids silently selecting "immediate" when
+    # another equally authoritative record leaves timing unknown.
+    status_rank = {"stated": 2, "unknown": 1, "zero": 0}.get(status, 1)
+    return (perm.get("score", 0), status_rank, months or 0)
 
 
 def _covered_versions(perm: dict) -> list[str]:
@@ -220,7 +261,8 @@ def _blank() -> dict:
 
 def _slot_from(perm: dict) -> dict:
     locations = _clean_locations(perm)
-    embargo_txt, _ = _embargo_of(perm)
+    embargo_txt, _months, embargo_status = _embargo_details(perm)
+    can_archive = bool(perm.get("can_archive"))
     conditions = []
     if perm.get("deposit_statement"):
         conditions.append("A specific deposit statement must accompany the file.")
@@ -228,7 +270,9 @@ def _slot_from(perm: dict) -> dict:
     if issuer.get("notes"):
         conditions.append(str(issuer["notes"]))
     return {
-        "allowed": "Yes" if perm.get("can_archive") else "No",
+        "allowed": "Yes" if can_archive else "No",
+        "permission_status": "allowed" if can_archive else "denied",
+        "embargo_status": embargo_status,
         "where": ", ".join(locations),
         "embargo": embargo_txt,
         "licence": _licence_of(perm),
@@ -286,8 +330,10 @@ def parse_permissions(payload: Optional[dict]) -> Optional[dict]:
         elif isinstance(v, list):
             evidence_urls += [str(u) for u in v if str(u).startswith("http")]
 
-    ir_ok = any(INSTITUTIONAL in l.lower()
-                for p in chosen.values() for l in _clean_locations(p))
+    ir_ok = any(
+        p.get("can_archive") is True and INSTITUTIONAL in location.lower()
+        for p in chosen.values() for location in _clean_locations(p)
+    )
 
     updated = meta.get("updated") or meta.get("added") or ""
     evidence = ("Source: ShareYourPaper Permissions (OA.Works) — the dataset behind "
@@ -296,8 +342,14 @@ def parse_permissions(payload: Optional[dict]) -> Optional[dict]:
     if updated:
         evidence += f" Policy record last updated {updated}."
 
+    can_archive = bool(top.get("can_archive"))
+    embargo_text, embargo_months, embargo_status = _embargo_details(top)
+    permission_status = "allowed" if can_archive else "denied"
+
     return {
         "policy_status": "Confirmed",
+        "permission_status": permission_status,
+        "embargo_status": embargo_status,
         **slots,
         "licence_notes": _licence_of(top),
         "repository_action_note": "",       # LLM fills this from these facts only
@@ -318,12 +370,14 @@ def parse_permissions(payload: Optional[dict]) -> Optional[dict]:
         "monitoring":        meta.get("monitoring", ""),
 
         # ---- internal, for deterministic downstream logic ----
-        "_can_archive":      bool(top.get("can_archive")),
+        "_can_archive":      can_archive,
         "_versions":         sorted(chosen.keys()),
         "_best_version":     VERSION_MAP.get(str(top.get("version", "")).lower(), ""),
         "_locations":        _clean_locations(top),
         "_institutional_ok": ir_ok,
-        "_embargo_months":   _embargo_of(top)[1],
+        "_embargo_months":   embargo_months,
+        "_embargo_status":   embargo_status,
+        "_embargo_text":     embargo_text,
         "_licence":          _licence_of(top),
     }
 
@@ -334,6 +388,8 @@ def not_confirmed(journal: str = "", issn: str = "", reason: str = "") -> dict:
         verify += f"search?term={quote(issn)}"
     return {
         "policy_status": "Not confirmed",
+        "permission_status": "unknown",
+        "embargo_status": "unknown",
         "preprint": _blank(), "postprint": _blank(),
         "published_version": _blank(),
         "licence_notes": "",
@@ -343,33 +399,91 @@ def not_confirmed(journal: str = "", issn: str = "", reason: str = "") -> dict:
                           f"{(' — ' + reason) if reason else ''}. Verify at {verify}"),
         "risk_flag": "Policy not verified — confirm before depositing.",
         "deposit_statement": "",
+        "embargo_end": "",
         "evidence_urls": [],
-        "_can_archive": False,
+        "_can_archive": None,
         "_versions": [],
+        "_best_version": "",
+        "_locations": [],
         "_institutional_ok": False,
         "_embargo_months": None,
+        "_embargo_status": "unknown",
+        "_embargo_text": "Not stated in a verified permission record",
+        "_licence": "",
         "_verify_url": verify,
     }
 
 
 # ── Deterministic recommendation — no LLM ──────────────────────────────────
 def recommendation_from(green: dict) -> dict:
-    ok = green.get("_can_archive", False)
-    emb = green.get("_embargo_months")
+    permission = str(green.get("permission_status") or "").lower()
+    if permission not in {"allowed", "denied", "unknown"}:
+        raw = green.get("_can_archive")
+        permission = "allowed" if raw is True else "denied" if raw is False else "unknown"
+
+    embargo_status = str(
+        green.get("embargo_status") or green.get("_embargo_status") or "unknown"
+    ).lower()
+    if embargo_status not in {"zero", "stated", "unknown"}:
+        embargo_status = "unknown"
+
+    months = green.get("_embargo_months")
     best = green.get("_best_version", "")
-    checks = []
-    if green.get("policy_status") != "Confirmed":
+    checks: list[str] = []
+
+    if permission == "unknown":
         checks.append("Verify the policy at Open Policy Finder before depositing.")
-    if ok and emb is None:
-        checks.append("No embargo is stated in the record — confirm against the "
-                      "publisher's policy page before depositing immediately.")
+        return {
+            "permission_status": "unknown",
+            "embargo_status": "unknown",
+            "best_version_to_deposit": "Not determined",
+            "best_timing": "Not determined — verify",
+            "immediate_open_deposit_possible": "Unknown",
+            "embargoed_deposit_needed": "Unknown",
+            "metadata_only_first": "Unknown",
+            "manual_checks_required": checks,
+        }
+
+    if permission == "denied":
+        return {
+            "permission_status": "denied",
+            "embargo_status": embargo_status,
+            "best_version_to_deposit": "No permitted self-archived version identified",
+            "best_timing": "Not applicable",
+            "immediate_open_deposit_possible": "No",
+            "embargoed_deposit_needed": "No",
+            "metadata_only_first": "No",
+            "manual_checks_required": checks,
+        }
+
+    # Permission is allowed from this point onward.
+    if embargo_status == "zero":
+        immediate, embargoed, metadata = "Yes", "No", "No"
+        timing = "Immediate open deposit is permitted by the verified record"
+    elif embargo_status == "stated":
+        immediate, embargoed, metadata = "No", "Yes", "Yes"
+        if isinstance(months, int) and months > 0:
+            timing = f"After a {months}-month embargo"
+        elif green.get("embargo_end"):
+            timing = f"After the stated embargo ends ({green['embargo_end']})"
+        else:
+            timing = "After the stated embargo; verify the release date"
+            checks.append("Confirm the exact embargo release date before opening the file.")
+    else:
+        immediate, embargoed, metadata = "Unknown", "Unknown", "Unknown"
+        timing = "Embargo not stated — verify before opening the file"
+        checks.append(
+            "The verified permission record does not state an embargo; confirm timing before immediate open deposit."
+        )
+
     return {
-        "best_version_to_deposit": best or "Not specified",
-        "best_timing": (f"After a {emb}-month embargo" if emb
-                        else ("Not stated — verify" if ok else "Not applicable")),
-        "immediate_open_deposit_possible": "Yes" if (ok and emb is None) else "No",
-        "embargoed_deposit_needed": "Yes" if emb else "No",
-        "metadata_only_first": "Yes" if emb else "No",
+        "permission_status": "allowed",
+        "embargo_status": embargo_status,
+        "best_version_to_deposit": best or "Permitted version not specified",
+        "best_timing": timing,
+        "immediate_open_deposit_possible": immediate,
+        "embargoed_deposit_needed": embargoed,
+        "metadata_only_first": metadata,
         "manual_checks_required": checks,
     }
 
@@ -377,22 +491,36 @@ def recommendation_from(green: dict) -> dict:
 def next_actions_from(green: dict, doi: str,
                       khazna: dict | None = None) -> list[str]:
     acts: list[str] = []
-    if green.get("_can_archive"):
+    permission = green.get("_can_archive")
+    embargo_status = green.get("_embargo_status") or green.get("embargo_status")
+
+    if permission is True:
         if green.get("_institutional_ok"):
-            acts.append("Deposit the permitted version in the KU repository, Khazna.")
+            if embargo_status == "unknown":
+                acts.append(
+                    "Prepare the permitted version for Khazna, but verify the release timing before making the file open."
+                )
+            else:
+                acts.append("Deposit the permitted version in the KU repository, Khazna, using the verified timing.")
         if green.get("deposit_statement"):
             acts.append("Include the required deposit statement with the file.")
         link = share_link(doi)
         if link:
             acts.append(f"Or deposit via ShareYourPaper: {link}")
+    elif permission is False:
+        acts.append(
+            "The verified permission record does not permit self-archiving for the selected version; check whether a KU APC agreement supports Gold OA."
+        )
     else:
-        acts.append("Self-archiving does not appear to be permitted — check "
-                    "whether a KU APC agreement covers Gold OA for this journal.")
+        acts.append(
+            "ResearchBee could not determine whether self-archiving is permitted. Verify the journal policy before depositing any version."
+        )
+
     if (khazna and khazna.get("checked") and khazna.get("in_khazna")
             and khazna.get("needs_deposit")):
         acts.insert(0, "This work is already recorded in the KU repository, Khazna, "
-                       "but has no open full text attached — adding the file "
-                       "would close the gap.")
+                       "but has no open full text attached — add a file only after "
+                       "the permitted version and timing have been verified.")
     return acts
 
 

openalex.py

@@ -1,145 +1,145 @@
-"""
-OpenAlex enrichment — journal metadata + peer percentile calculation.
-Free API, no key required. Uses polite pool with mailto header.
-
-Changes from v1.0:
-- Shared module-level httpx.AsyncClient (init_client / close_client for lifespan)
-- compute_percentile accepts an optional concept_cache dict to deduplicate
-  identical concept peer-set fetches across journals in one request
-- _norm_issn removed — uses utils.norm_issn
-"""
-import httpx
-import asyncio
-from typing import Optional
-from utils import norm_issn
-
-UA = "ResearchNavigator/1.0 (mailto:library@ku.ac.ae)"
-BASE = "https://api.openalex.org"
-
-_http_client: Optional[httpx.AsyncClient] = None
-
-
-def init_client() -> None:
-    global _http_client
-    _http_client = httpx.AsyncClient(
-        timeout=httpx.Timeout(10.0, connect=5.0),
-        headers={"User-Agent": UA},
-    )
-
-
-async def close_client() -> None:
-    global _http_client
-    if _http_client and not _http_client.is_closed:
-        await _http_client.aclose()
-    _http_client = None
-
-
-def _get_client() -> httpx.AsyncClient:
-    if _http_client is None or _http_client.is_closed:
-        init_client()
-    return _http_client
-
-
-async def enrich_journal(name: str, issn: Optional[str]) -> Optional[dict]:
-    client = _get_client()
-    try:
-        if issn:
-            clean = norm_issn(issn) or issn.strip()
-            r = await client.get(f"{BASE}/sources/issn:{clean}")
-        else:
-            r = await client.get(
-                f"{BASE}/sources", params={"search": name, "per_page": 1}
-            )
-        if not r.is_success:
-            return None
-        d = r.json()
-        s = d if issn else (d.get("results") or [None])[0]
-        if not s:
-            return None
-        return {
-            "openalex_id":           s.get("id"),
-            "display_name":          s.get("display_name"),
-            "issn_l":                s.get("issn_l"),
-            "issns":                 s.get("issn", []),
-            "host_organization":     s.get("host_organization_name"),
-            "is_oa":                 s.get("is_oa"),
-            "is_in_doaj":            s.get("is_in_doaj"),
-            "works_count":           s.get("works_count"),
-            "cited_by_count":        s.get("cited_by_count"),
-            "h_index":               (s.get("summary_stats") or {}).get("h_index"),
-            "two_yr_mean_citedness": (s.get("summary_stats") or {}).get("2yr_mean_citedness"),
-            "homepage_url":          s.get("homepage_url"),
-            "top_concept":           ((s.get("x_concepts") or [{}])[0]).get("display_name"),
-            "top_concept_id":        ((s.get("x_concepts") or [{}])[0]).get("id"),
-        }
-    except Exception as e:
-        print(f"[OpenAlex] enrich error for {name}: {e}")
-        return None
-
-
-async def compute_percentile(
-    openalex: dict,
-    concept_cache: Optional[dict] = None,
-) -> Optional[dict]:
-    try:
-        if not openalex or not openalex.get("top_concept_id"):
-            return None
-        val = openalex.get("two_yr_mean_citedness")
-        if val is None:
-            return None
-
-        concept_id = str(openalex["top_concept_id"]).split("/")[-1]
-
-        if concept_cache is not None and concept_id in concept_cache:
-            peers = concept_cache[concept_id]
-        else:
-            client = _get_client()
-            r = await client.get(
-                f"{BASE}/sources",
-                params={
-                    "filter": f"concepts.id:{concept_id},type:journal",
-                    "per_page": 200,
-                    "sort": "summary_stats.2yr_mean_citedness:desc",
-                },
-            )
-            if not r.is_success:
-                return None
-            peers = [
-                s.get("summary_stats", {}).get("2yr_mean_citedness")
-                for s in r.json().get("results", [])
-            ]
-            peers = [v for v in peers if isinstance(v, (int, float))]
-            if concept_cache is not None:
-                concept_cache[concept_id] = peers
-
-        if len(peers) < 10:
-            return None
-
-        below = sum(1 for v in peers if v < val)
-        pct = round(below / len(peers) * 100)
-        q = "Q1" if pct >= 75 else "Q2" if pct >= 50 else "Q3" if pct >= 25 else "Q4"
-        return {
-            "percentile": pct,
-            "quartile": q,
-            "peer_count": len(peers),
-            "concept": openalex.get("top_concept"),
-        }
-    except Exception as e:
-        print(f"[OpenAlex] percentile error: {e}")
-        return None
-
-
-async def enrich_all(journals: list) -> list:
-    """Enrich a list of journal dicts in parallel, deduplicating concept peer-sets."""
-    concept_cache: dict = {}
-
-    async def _enrich_one(j: dict) -> dict:
-        oa = await enrich_journal(j.get("name", ""), j.get("issn"))
-        pct = await compute_percentile(oa, concept_cache) if oa else None
-        j["openalex"] = oa
-        j["oa_percentile"] = pct
-        if not j.get("issn") and oa:
-            j["issn"] = oa.get("issn_l") or (oa.get("issns") or [None])[0]
-        return j
-
-    return list(await asyncio.gather(*[_enrich_one(j) for j in journals]))
+"""
+OpenAlex enrichment — journal metadata + peer percentile calculation.
+Free API, no key required. Uses polite pool with mailto header.
+
+Changes from v1.0:
+- Shared module-level httpx.AsyncClient (init_client / close_client for lifespan)
+- compute_percentile accepts an optional concept_cache dict to deduplicate
+  identical concept peer-set fetches across journals in one request
+- _norm_issn removed — uses utils.norm_issn
+"""
+import httpx
+import asyncio
+from typing import Optional
+from utils import norm_issn
+
+UA = "ResearchNavigator/1.0 (mailto:library@ku.ac.ae)"
+BASE = "https://api.openalex.org"
+
+_http_client: Optional[httpx.AsyncClient] = None
+
+
+def init_client() -> None:
+    global _http_client
+    _http_client = httpx.AsyncClient(
+        timeout=httpx.Timeout(10.0, connect=5.0),
+        headers={"User-Agent": UA},
+    )
+
+
+async def close_client() -> None:
+    global _http_client
+    if _http_client and not _http_client.is_closed:
+        await _http_client.aclose()
+    _http_client = None
+
+
+def _get_client() -> httpx.AsyncClient:
+    if _http_client is None or _http_client.is_closed:
+        init_client()
+    return _http_client
+
+
+async def enrich_journal(name: str, issn: Optional[str]) -> Optional[dict]:
+    client = _get_client()
+    try:
+        if issn:
+            clean = norm_issn(issn) or issn.strip()
+            r = await client.get(f"{BASE}/sources/issn:{clean}")
+        else:
+            r = await client.get(
+                f"{BASE}/sources", params={"search": name, "per_page": 1}
+            )
+        if not r.is_success:
+            return None
+        d = r.json()
+        s = d if issn else (d.get("results") or [None])[0]
+        if not s:
+            return None
+        return {
+            "openalex_id":           s.get("id"),
+            "display_name":          s.get("display_name"),
+            "issn_l":                s.get("issn_l"),
+            "issns":                 s.get("issn", []),
+            "host_organization":     s.get("host_organization_name"),
+            "is_oa":                 s.get("is_oa"),
+            "is_in_doaj":            s.get("is_in_doaj"),
+            "works_count":           s.get("works_count"),
+            "cited_by_count":        s.get("cited_by_count"),
+            "h_index":               (s.get("summary_stats") or {}).get("h_index"),
+            "two_yr_mean_citedness": (s.get("summary_stats") or {}).get("2yr_mean_citedness"),
+            "homepage_url":          s.get("homepage_url"),
+            "top_concept":           ((s.get("x_concepts") or [{}])[0]).get("display_name"),
+            "top_concept_id":        ((s.get("x_concepts") or [{}])[0]).get("id"),
+        }
+    except Exception as e:
+        print(f"[OpenAlex] enrich error for {name}: {e}")
+        return None
+
+
+async def compute_percentile(
+    openalex: dict,
+    concept_cache: Optional[dict] = None,
+) -> Optional[dict]:
+    try:
+        if not openalex or not openalex.get("top_concept_id"):
+            return None
+        val = openalex.get("two_yr_mean_citedness")
+        if val is None:
+            return None
+
+        concept_id = str(openalex["top_concept_id"]).split("/")[-1]
+
+        if concept_cache is not None and concept_id in concept_cache:
+            peers = concept_cache[concept_id]
+        else:
+            client = _get_client()
+            r = await client.get(
+                f"{BASE}/sources",
+                params={
+                    "filter": f"concepts.id:{concept_id},type:journal",
+                    "per_page": 200,
+                    "sort": "summary_stats.2yr_mean_citedness:desc",
+                },
+            )
+            if not r.is_success:
+                return None
+            peers = [
+                s.get("summary_stats", {}).get("2yr_mean_citedness")
+                for s in r.json().get("results", [])
+            ]
+            peers = [v for v in peers if isinstance(v, (int, float))]
+            if concept_cache is not None:
+                concept_cache[concept_id] = peers
+
+        if len(peers) < 10:
+            return None
+
+        below = sum(1 for v in peers if v < val)
+        pct = round(below / len(peers) * 100)
+        q = "Q1" if pct >= 75 else "Q2" if pct >= 50 else "Q3" if pct >= 25 else "Q4"
+        return {
+            "percentile": pct,
+            "quartile": q,
+            "peer_count": len(peers),
+            "concept": openalex.get("top_concept"),
+        }
+    except Exception as e:
+        print(f"[OpenAlex] percentile error: {e}")
+        return None
+
+
+async def enrich_all(journals: list) -> list:
+    """Enrich a list of journal dicts in parallel, deduplicating concept peer-sets."""
+    concept_cache: dict = {}
+
+    async def _enrich_one(j: dict) -> dict:
+        oa = await enrich_journal(j.get("name", ""), j.get("issn"))
+        pct = await compute_percentile(oa, concept_cache) if oa else None
+        j["openalex"] = oa
+        j["oa_percentile"] = pct
+        if not j.get("issn") and oa:
+            j["issn"] = oa.get("issn_l") or (oa.get("issns") or [None])[0]
+        return j
+
+    return list(await asyncio.gather(*[_enrich_one(j) for j in journals]))

prompts.py

@@ -3,7 +3,7 @@ ResearchBee — system prompts for all API endpoints.
 """
 
 # ── Journal analysis ──────────────────────────────────────────────────────
-JOURNAL_SYSTEM_PROMPT = """You are ResearchBee, an expert academic publishing advisor and repository specialist. Analyse the manuscript details and return a structured JSON response.
+JOURNAL_SYSTEM_PROMPT = """You are ResearchBee, an expert academic publishing advisor. Analyse the manuscript and recommend journals based on scope, audience, article type, methods, and the author's stated preferences.
 
 Return ONLY valid JSON in this exact structure:
 {
@@ -17,45 +17,34 @@ Return ONLY valid JSON in this exact structure:
   "journals": [
     {
       "name": "Journal name",
-      "issn": "ISSN if known",
+      "issn": "ISSN if known, otherwise empty",
       "publisher": "Publisher name",
       "confidence": "High|Medium|Low",
       "fit_reason": "Why this journal fits",
-      "audience_match": "How audience aligns",
-      "submission_strategy": "Specific submission advice",
-      "oa_compliance_note": "OA and funder compliance note",
-      "green_oa": {
-        "policy_status": "Confirmed|Partially confirmed|Not confirmed",
-        "preprint": {"allowed": "Yes|No|Unclear", "where": "", "embargo": "", "licence": "", "conditions": ""},
-        "postprint": {"allowed": "Yes|No|Unclear", "where": "", "embargo": "", "licence": "", "conditions": ""},
-        "published_version": {"allowed": "Yes|No|Unclear", "where": "", "embargo": "", "licence": "", "conditions": ""},
-        "licence_notes": "",
-        "repository_action_note": "",
-        "evidence_note": "",
-        "risk_flag": null
-      },
-
+      "audience_match": "How the journal audience aligns",
+      "submission_strategy": "Specific submission advice that does not claim acceptance probability",
+      "oa_compliance_note": "General OA/funder planning note only; direct the author to ResearchBee License Checking with the article DOI for deposit rights"
     }
   ],
   "extended_list": [
     {"name": "", "issn": "", "publisher": "", "fit_reason": ""}
   ],
-  "repository_recommendation": {
-    "best_version_to_deposit": "",
-    "best_timing": "",
-    "immediate_open_deposit_possible": "Yes|No",
-    "embargoed_deposit_needed": "Yes|No",
-    "metadata_only_first": "Yes|No",
-    "manual_checks_required": []
-  },
-  "next_actions": ["action 1", "action 2"],
+  "next_actions": ["submission-planning action 1", "submission-planning action 2"],
   "global_notes": ""
 }
 
 Return 10 detailed journals in 'journals' and up to 20 quick matches in 'extended_list'.
-Do NOT include submission_checklist in the response — omit it entirely to save tokens.
-Always recommend depositing to Khazna (khazna.ku.ac.ae) for KU authors.
-Always recommend depositing to an appropriate institutional or subject repository."""
+Do NOT include submission_checklist in the response.
+
+POLICY SAFETY — ABSOLUTE RULES:
+- This workflow DOES NOT check publisher or journal self-archiving policies.
+- Do NOT return a green_oa object or repository_recommendation object.
+- Do NOT state or imply a permitted manuscript version, embargo duration, licence, deposit location, or immediate-deposit right for any journal.
+- Do NOT label any policy as confirmed or partially confirmed.
+- The oa_compliance_note may discuss general OA/funder planning only and must tell the author to use ResearchBee's DOI-based License Checking tool before depositing any version.
+- For KU authors, you may mention Khazna only as an institutional repository to consider AFTER the permitted version and timing have been verified.
+- Recommend real journals only; never invent an ISSN. If uncertain, leave ISSN empty.
+"""
 
 # ── Licence checking ──────────────────────────────────────────────────────
 LICENSE_SYSTEM_PROMPT = """You are ResearchBee, a scholarly communication assistant at Khalifa University Library.
@@ -68,12 +57,13 @@ confirmed.
 ABSOLUTE RULES:
 1. NEVER state a specific embargo period, licence, or permitted version as fact.
 2. ALWAYS set "policy_status" to "Not confirmed".
-3. Set every version block to {"allowed": "Unclear", ...} unless the user
+3. ALWAYS set "permission_status" and "embargo_status" to "unknown".
+4. Set every version block to {"allowed": "Unclear", ...} unless the user
    themselves supplied the information, in which case attribute it to them.
-4. NEVER invent an ISSN. If you were not given one, leave it empty.
-5. Your job is to explain WHAT the author should check and WHERE, not to
+5. NEVER invent an ISSN. If you were not given one, leave it empty.
+6. Your job is to explain WHAT the author should check and WHERE, not to
    answer the policy question yourself.
-6. Recommend entering the article DOI for a verified answer.
+7. Recommend entering the article DOI for a verified answer.
 
 Return ONLY valid JSON:
 {
@@ -84,22 +74,26 @@ Return ONLY valid JSON:
       "publisher": "only if supplied by the user, else empty string",
       "green_oa": {
         "policy_status": "Not confirmed",
+        "permission_status": "unknown",
+        "embargo_status": "unknown",
         "preprint": {"allowed": "Unclear", "where": "", "embargo": "", "licence": "", "conditions": ""},
         "postprint": {"allowed": "Unclear", "where": "", "embargo": "", "licence": "", "conditions": ""},
         "published_version": {"allowed": "Unclear", "where": "", "embargo": "", "licence": "", "conditions": ""},
         "licence_notes": "",
         "repository_action_note": "Practical guidance on what to check and where.",
         "evidence_note": "No verified policy record was retrieved for this journal.",
-        "risk_flag": "Policy not verified \u2014 confirm before depositing."
+        "risk_flag": "Policy not verified — confirm before depositing."
       }
     }
   ],
   "repository_recommendation": {
+    "permission_status": "unknown",
+    "embargo_status": "unknown",
     "best_version_to_deposit": "Not determined",
-    "best_timing": "Not determined",
-    "immediate_open_deposit_possible": "No",
-    "embargoed_deposit_needed": "No",
-    "metadata_only_first": "No",
+    "best_timing": "Not determined — verify",
+    "immediate_open_deposit_possible": "Unknown",
+    "embargoed_deposit_needed": "Unknown",
+    "metadata_only_first": "Unknown",
     "manual_checks_required": ["Verify the policy at Open Policy Finder before depositing."]
   },
   "next_actions": ["Enter your article's DOI for a verified, article-level answer."],
@@ -137,20 +131,21 @@ Return ONLY valid JSON: {"note": "..."}"""
 REPO_SYSTEM_PROMPT = """You are ResearchBee, a research data management advisor at Khalifa University Library.
 
 CRITICAL: You will be given a CURATED REGISTRY of verified repositories.
-You must SELECT from that registry by "id". You must NOT invent repositories,
-and you must NOT state factual repository attributes (max file size, licences,
-certification, cost, persistent identifier type) \u2014 those are supplied by the
-server from verified records and will overwrite anything you write.
+You must SELECT from that registry by the exact "id" value. You must NOT invent
+repositories and must NOT state factual repository attributes such as URLs,
+file-size limits, licences, certification, cost, identifiers, access models,
+embargo support, versioning, or sensitive-data capability. The server supplies
+all factual attributes from verified registry records.
 
 Your job is ONLY to:
   * understand the dataset,
-  * choose the 4\u20136 most suitable registry entries by id, in rank order,
-  * write a "fit_reason" for each explaining why it suits THIS dataset.
+  * choose the 4–6 most suitable registry entries by exact id, in rank order,
+  * write a dataset-specific fit_reason and fit_confidence for each choice,
+  * provide general deposit-planning advice without inventing repository facts.
 
 If no registry entry fits well, return fewer entries rather than inventing one.
 
-Return ONLY valid JSON:
-
+Return ONLY valid JSON in this exact structure:
 {
   "dataset_understanding": {
     "summary": "",
@@ -162,32 +157,15 @@ Return ONLY valid JSON:
   },
   "repositories": [
     {
-      "name": "",
-      "type": "Institutional|Domain-specific|Generalist",
-      "url": "",
-      "scope": "",
-      "fit_reason": "",
-      "data_types_accepted": "",
-      "max_file_size": "",
-      "licences_supported": "",
-      "persistent_identifier": "DOI|Handle|ARK|None",
-      "versioning": "Yes|No",
-      "embargo_support": "Yes|No",
-      "access_model": "Open|Controlled|Mixed",
-      "cost": "Free|Paid|Institutional",
-      "certification": "CoreTrust Seal|FAIR|Institutional|None",
-      "fair_alignment": "High|Medium|Low",
-      "funder_compliance_note": "",
-      "sensitive_data_suitable": "Yes|No|Conditional",
-      "confidence": "High|Medium|Low",
-      "verification_status": "",
-      "risk_flag": null
+      "id": "exact registry id",
+      "fit_reason": "Why this registry entry suits this dataset",
+      "fit_confidence": "High|Medium|Low"
     }
   ],
   "deposit_strategy": {
-    "primary_choice": "",
-    "secondary_choice": "",
-    "metadata_standard": "",
+    "primary_repository_id": "exact id from repositories, or empty",
+    "secondary_repository_id": "exact id from repositories, or empty",
+    "metadata_standard": "General recommendation; say 'Check repository guidance' when not established by the input",
     "deposit_workflow": [],
     "manual_checks_required": []
   },
@@ -196,10 +174,13 @@ Return ONLY valid JSON:
 }
 
 Rules:
-- Recommend 4-5 repositories — prefer domain-specific, always include 1-2 generalist options (e.g. Zenodo, Figshare).
-- If institution is UW-Madison, note that Dryad and MINDS@UW are the primary institutional options and are free — these will be prepended by the server automatically.
-- If sensitive or PII data: recommend controlled-access repositories only.
-- If no funder mandate: still recommend open repositories for discoverability."""
+- Every repositories item MUST contain an exact registry id.
+- Return only the three repository fields shown above: id, fit_reason, fit_confidence.
+- Prefer a suitable domain-specific repository and include one or two generalist alternatives when the registry supports that choice.
+- Khazna is prepended by the server automatically; do not select it merely to satisfy the requested count.
+- If the dataset contains identifiable, pseudonymised, health, or otherwise sensitive data, choose only registry entries whose supplied description explicitly supports the required access model; otherwise return fewer choices and flag a manual check.
+- If no funder mandate is supplied, favour discoverability and disciplinary norms, but do not invent compliance claims.
+"""
 
 # ── Subject normalisation ─────────────────────────────────────────────────
 SUBJECT_NORMALISE_PROMPT = """You are an academic subject classification expert. Map the user's subject input onto SCImago's classification.

repositories.py

@@ -302,13 +302,24 @@ REGISTRY: dict[str, dict[str, Any]] = {
 
 
 # ── Helpers ────────────────────────────────────────────────────────────────
-# Fields the LLM is NOT permitted to author. Anything it returns for these is
-# discarded and replaced from the registry.
-SERVER_OWNED = (
-    "name", "type", "url", "scope", "data_types_accepted", "max_file_size",
-    "persistent_identifier", "versioning", "embargo_support", "cost",
-    "curation", "certification", "source_url", "verified", "is_khazna",
-)
+# The model may author ONLY analytical fit fields. Every other repository
+# attribute is supplied from REGISTRY, including the id itself after lookup.
+MODEL_ALLOWED = ("fit_reason", "fit_confidence")
+
+
+def _clean_text(value: Any, limit: int = 1000) -> str:
+    return " ".join(str(value or "").split())[:limit]
+
+
+def _clean_list(value: Any, limit: int = 12, item_limit: int = 500) -> list[str]:
+    if not isinstance(value, list):
+        return []
+    out: list[str] = []
+    for item in value[:limit]:
+        cleaned = _clean_text(item, item_limit)
+        if cleaned:
+            out.append(cleaned)
+    return out
 
 
 def registry_for_prompt(limit: int = 0) -> str:
@@ -326,21 +337,38 @@ def registry_for_prompt(limit: int = 0) -> str:
 
 def hydrate_selection(selected: list[dict]) -> list[dict]:
     """
-    Replace every server-owned field with the verified registry value.
+    Hydrate exact registry ids and discard every model-authored factual field.
 
-    Anything the model invented — an unknown id, a fabricated file-size limit —
-    is dropped. Only `fit_reason` and ranking survive from the model.
+    Only `fit_reason` and `fit_confidence` survive from the model. Unknown ids,
+    duplicate ids, non-object entries, and malformed confidence values are
+    dropped or normalised rather than displayed as repository facts.
     """
     out: list[dict] = []
     seen: set[str] = set()
-    for item in selected or []:
+    if not isinstance(selected, list):
+        return out
+
+    for item in selected:
+        if not isinstance(item, dict):
+            continue
         rid = str(item.get("id", "")).strip().lower()
         entry = REGISTRY.get(rid)
         if not entry or rid in seen:
             continue
         seen.add(rid)
-        merged = {k: v for k, v in item.items() if k not in SERVER_OWNED}
-        merged.update(entry)
+
+        fit_reason = _clean_text(item.get("fit_reason"), 1200)
+        confidence = _clean_text(item.get("fit_confidence"), 20).title()
+        if confidence not in {"High", "Medium", "Low"}:
+            confidence = ""
+
+        merged = dict(entry)
+        merged["fit_reason"] = fit_reason
+        merged["fit_confidence"] = confidence
+        # Backwards-compatible alias for older frontends; it is still an
+        # analytical fit score, never a repository-fact confidence score.
+        merged["confidence"] = confidence
+        merged["verification_status"] = "Curated KU Library registry record"
         merged["verified_note"] = (
             f"Repository details verified {entry['verified']} from {entry['source_url']}"
         )
@@ -348,11 +376,52 @@ def hydrate_selection(selected: list[dict]) -> list[dict]:
     return out
 
 
+def hydrate_deposit_strategy(raw: Any, hydrated: list[dict]) -> dict:
+    """Resolve model-selected ids to server-owned names and bound free text."""
+    raw = raw if isinstance(raw, dict) else {}
+    by_id = {str(r.get("id", "")): r for r in hydrated if r.get("id")}
+    ranked_ids = [
+        str(r.get("id")) for r in hydrated
+        if r.get("id") and not r.get("is_khazna")
+    ]
+
+    def selected_id(key: str, fallback_index: int) -> str:
+        candidate = str(raw.get(key, "")).strip().lower()
+        if candidate in by_id and not by_id[candidate].get("is_khazna"):
+            return candidate
+        return ranked_ids[fallback_index] if len(ranked_ids) > fallback_index else ""
+
+    primary_id = selected_id("primary_repository_id", 0)
+    secondary_id = selected_id("secondary_repository_id", 1)
+    if secondary_id == primary_id:
+        secondary_id = ranked_ids[1] if len(ranked_ids) > 1 else ""
+
+    return {
+        "primary_repository_id": primary_id,
+        "secondary_repository_id": secondary_id,
+        "primary_choice": by_id.get(primary_id, {}).get("name", "Not determined"),
+        "secondary_choice": by_id.get(secondary_id, {}).get("name", "Not determined"),
+        "metadata_standard": _clean_text(
+            raw.get("metadata_standard") or "Check the selected repository's current metadata guidance.",
+            500,
+        ),
+        "deposit_workflow": _clean_list(raw.get("deposit_workflow"), limit=10),
+        "manual_checks_required": _clean_list(raw.get("manual_checks_required"), limit=10),
+        "khazna_metadata_note": (
+            "Register the dataset metadata in Khazna for KU visibility even when "
+            "the files are deposited in a domain or generalist repository."
+        ),
+    }
+
+
 def khazna_entry() -> dict:
     e = dict(REGISTRY["khazna"])
     e["fit_reason"] = ("Khazna is Khalifa University's institutional repository. "
                        "Register your dataset here for KU visibility and compliance, "
                        "even when the data itself lives in a domain repository.")
+    e["fit_confidence"] = "High"
+    e["confidence"] = "High"
+    e["verification_status"] = "Curated KU Library registry record"
     e["verified_note"] = (f"Repository details verified {e['verified']} "
                           f"from {e['source_url']}")
     return e

requirements-dev.txt

@@ -0,0 +1,3 @@
+-r requirements.txt
+pytest>=8.0,<10
+pytest-asyncio>=0.23,<2

requirements.txt

@@ -1,12 +1,12 @@
-fastapi==0.111.0
-uvicorn[standard]==0.29.0
-openai==1.30.0
-httpx==0.27.0
-python-multipart==0.0.9
-tenacity==8.3.0
-huggingface_hub>=0.23
-apscheduler>=3.10
-joblib>=1.3
-numpy>=1.26
-scipy>=1.11
-scikit-learn>=1.4
+fastapi==0.111.0
+uvicorn[standard]==0.29.0
+openai==1.30.0
+httpx==0.27.0
+python-multipart==0.0.9
+tenacity==8.3.0
+huggingface_hub>=0.23
+apscheduler>=3.10
+joblib>=1.3
+numpy>=1.26
+scipy>=1.11
+scikit-learn>=1.4

researchbee_journals.py

@@ -1,606 +1,606 @@
-"""
-ResearchBee journal index + recommender.
-
-    from researchbee_journals import JournalIndex
-
-    idx = JournalIndex("journals.db")
-    idx.lookup(issn="0737-8831")            # -> JournalRecord (Library Hi Tech)
-    idx.recommend(title=..., abstract=...)  # -> [Recommendation, ...]
-
-Architecture
-------------
-Three stages, deliberately separated so the expensive one stays optional:
-
-  1. CANDIDATE GENERATION  — hard ASJC filter (when subject codes are known) plus
-     FTS5/BM25 over title + publisher + subject names. Pure SQLite, no model,
-     sub-10ms, returns a few hundred candidates.
-
-  2. SEMANTIC RERANK       — optional. Cosine similarity between the manuscript
-     embedding and a per-journal centroid. See the honest caveat below.
-
-  3. FIT SCORING           — quartile, indexation state, OA status, coverage
-     recency, language, publisher agreement coverage. Deterministic, explainable,
-     and the part that should drive the text ResearchBee shows the researcher.
-
-The caveat on stage 2
----------------------
-The Scopus Source List carries no aims-and-scope text. Embedding a journal
-*title* against a manuscript *abstract* is a weak signal — it recovers obvious
-lexical matches ("Journal of Hydrology" for a hydrology paper) and little else.
-Real matchers (Jane, Elsevier Journal Finder, WoS Match Manuscript) compare the
-manuscript against *articles the journal has published*. To reach that quality
-you need article-level text, which means building journal centroids from OpenAlex
-works — `build_centroids.py` does this. Until centroids exist, `recommend()`
-runs stages 1 and 3 only and says so in `Recommendation.method`, rather than
-quietly returning title-similarity noise dressed up as semantic matching.
-"""
-from __future__ import annotations
-
-import re
-import sqlite3
-import threading
-from dataclasses import dataclass, field, asdict
-from pathlib import Path
-from typing import Iterable, Sequence
-
-__all__ = ["JournalIndex", "JournalRecord", "Recommendation", "IndexationState"]
-
-
-# ---------------------------------------------------------------------------
-# Indexation state — the distinction the old lookup collapsed
-# ---------------------------------------------------------------------------
-class IndexationState:
-    """
-    Why this exists: 'no quartile' is three materially different answers to a
-    researcher deciding where to submit.
-
-      INDEXED_RANKED   in Scopus, has a current SJR and quartile. (30,405)
-      INDEXED_NEW      in Scopus, coverage began 2024+, SJR not yet computed.
-                       SJR needs a ~3-year citation window, so the absence is a
-                       timing artefact, not a quality signal. Safe to submit to.
-                       (326)
-      INDEXED_UNRANKED SCImago has a record but computes no SJR — typically below
-                       its citable-document threshold. Thin but real. (279)
-      INDEXED_NO_SJR   in Scopus, absent from SCImago entirely. Overwhelmingly
-                       book series (1,052), which SJR does not rank at all. Only
-                       ~70 are long-running journals, mostly regional titles.
-                       (1,366)
-      NOT_INDEXED      no Scopus source record at all, active or inactive.
-
-    And three states for titles Scopus no longer indexes. Keeping them apart is
-    the whole point: of 16,838 inactive titles only 967 were removed for cause.
-    Telling a researcher that a journal which merely changed its name is a
-    quality risk is worse than saying nothing.
-
-      DISCONTINUED     removed by Scopus for cause. The real red flag. (967)
-      RENAMED          ceased under this title but the work continues elsewhere;
-                       `successor_title` names where. Benign — redirect, don't
-                       warn. (~7,000 with a resolvable successor)
-      CEASED           stopped being indexed, no successor recorded and no
-                       discontinuation flag. Usually a journal that folded.
-    """
-    INDEXED_RANKED = "indexed_ranked"
-    INDEXED_NEW = "indexed_new"
-    INDEXED_UNRANKED = "indexed_unranked"
-    INDEXED_NO_SJR = "indexed_no_sjr"
-    NOT_INDEXED = "not_indexed"
-    DISCONTINUED = "discontinued"
-    RENAMED = "renamed"
-    CEASED = "ceased"
-
-    #: States a prospective author should be actively warned about.
-    WARN = frozenset({DISCONTINUED})
-    #: States meaning "Scopus does not index new articles here".
-    NOT_CURRENTLY_INDEXED = frozenset({DISCONTINUED, RENAMED, CEASED, NOT_INDEXED})
-
-    MESSAGES = {
-        INDEXED_RANKED: "Indexed in Scopus with a current SJR quartile.",
-        INDEXED_NEW: (
-            "Indexed in Scopus, but coverage is too recent for an SJR quartile. "
-            "SJR requires roughly three years of citation data; expect a first "
-            "quartile assignment once that window closes. The missing quartile "
-            "is not a quality signal."
-        ),
-        INDEXED_UNRANKED: (
-            "Indexed in Scopus and listed by SCImago, but no SJR is computed — "
-            "usually because the title publishes too few citable documents to "
-            "meet SCImago's threshold."
-        ),
-        INDEXED_NO_SJR: (
-            "Indexed in Scopus but absent from SCImago. Normal for book series "
-            "and trade journals, which SJR does not rank. Check the source type "
-            "before treating this as a concern."
-        ),
-        NOT_INDEXED: (
-            "No Scopus source record found — this title is not indexed in "
-            "Scopus, and has no record of having been."
-        ),
-        DISCONTINUED: (
-            "Scopus has discontinued this title — it was removed from the index "
-            "rather than ceasing publication. Articles published here will not "
-            "be indexed in Scopus. Scopus does not publish the reason for an "
-            "individual discontinuation; grounds range from publication-ethics "
-            "concerns to metrics such as self-citation, and discontinued titles "
-            "include some from major publishers. Verify independently before "
-            "submitting, and do not assume the journal is predatory."
-        ),
-        RENAMED: (
-            "This title is no longer indexed under this name, but the journal "
-            "continues. This is a rename or merger, not a quality concern."
-        ),
-        CEASED: (
-            "Scopus no longer indexes this title and no successor is recorded — "
-            "the journal appears to have ceased publication. Not flagged as a "
-            "discontinuation for cause."
-        ),
-    }
-
-
-_ISSN_RE = re.compile(r"[^0-9X]")
-
-
-def norm_issn(v: str | None) -> str | None:
-    if not v:
-        return None
-    s = _ISSN_RE.sub("", str(v).strip().upper())
-    if len(s) < 8:
-        s = s.zfill(8)
-    return s if len(s) == 8 else None
-
-
-# ---------------------------------------------------------------------------
-# Records
-# ---------------------------------------------------------------------------
-@dataclass
-class JournalRecord:
-    scopus_id: str
-    title: str
-    issn_print: str | None
-    issn_e: str | None
-    source_type: str | None
-    publisher: str | None
-    publisher_group: str | None
-    coverage_start: int | None
-    coverage_end: int | None
-    coverage_has_gap: bool
-    oa_unpaywall: bool
-    medline: bool
-    has_english: bool
-    languages: str | None
-    sjr: float | None
-    sjr_rank: int | None
-    sjr_best_quartile: str | None
-    h_index: int | None
-    cites_per_doc_2y: float | None
-    total_docs_year: int | None
-    oa_diamond: bool
-    scimago_oa: bool
-    overton: int | None
-    country: str | None
-    scimago_id: str | None = None
-    lifecycle: str = "active"
-    successor_title: str | None = None
-    successor_scopus_id: str | None = None
-    successor_is_active: bool = False
-    subjects: list[dict] = field(default_factory=list)
-    state: str = IndexationState.NOT_INDEXED
-    state_message: str = ""
-
-    @property
-    def warn(self) -> bool:
-        """True when a prospective author should be actively cautioned."""
-        return self.state in IndexationState.WARN
-
-    @property
-    def currently_indexed(self) -> bool:
-        return self.state not in IndexationState.NOT_CURRENTLY_INDEXED
-
-    def quartile_for(self, asjc: str) -> str | None:
-        """Quartile in one specific subject area, not the journal's best."""
-        for s in self.subjects:
-            if s["asjc"] == asjc:
-                return s.get("quartile")
-        return None
-
-    def to_dict(self) -> dict:
-        return asdict(self)
-
-
-@dataclass
-class Recommendation:
-    journal: JournalRecord
-    score: float
-    reasons: list[str]
-    method: str
-
-
-# ---------------------------------------------------------------------------
-# Index
-# ---------------------------------------------------------------------------
-_STOP = set("""
-a an the and or but if of in on at to for with by from as is are was were be been
-being this that these those it its we our they their he she his her you your not
-no can may might will would should could have has had do does did than then so
-such other others new novel study studies paper article research results method
-methods approach based using used use show shows shown propose proposed present
-presented provide provides between among during through both each more most many
-much also however therefore thus while when where which who whom what how why
-""".split())
-
-
-class JournalIndex:
-    """Thread-safe read-only wrapper over journals.db."""
-
-    def __init__(self, db_path: str | Path):
-        self.db_path = str(db_path)
-        if not Path(self.db_path).exists():
-            raise FileNotFoundError(self.db_path)
-        self._local = threading.local()
-
-    @property
-    def con(self) -> sqlite3.Connection:
-        con = getattr(self._local, "con", None)
-        if con is None:
-            con = sqlite3.connect(f"file:{self.db_path}?mode=ro", uri=True,
-                                  check_same_thread=False)
-            con.row_factory = sqlite3.Row
-            con.execute("PRAGMA mmap_size = 268435456")
-            self._local.con = con
-        return con
-
-    # -- meta ---------------------------------------------------------------
-    def meta(self) -> dict[str, str]:
-        return {r["key"]: r["value"] for r in self.con.execute("SELECT * FROM meta")}
-
-    # -- lookup -------------------------------------------------------------
-    def lookup(self, issn: str | None = None, title: str | None = None,
-               scopus_id: str | None = None) -> JournalRecord | None:
-        row = None
-        if scopus_id:
-            row = self.con.execute(
-                "SELECT * FROM journals WHERE scopus_id = ?", (scopus_id,)
-            ).fetchone()
-        if row is None and issn:
-            key = norm_issn(issn)
-            if key:
-                row = self.con.execute(
-                    "SELECT * FROM journals WHERE issn_e = ? OR issn_print = ? LIMIT 1",
-                    (key, key),
-                ).fetchone()
-        if row is None and title:
-            row = self.con.execute(
-                "SELECT * FROM journals WHERE lower(title) = lower(?) LIMIT 1",
-                (title.strip(),),
-            ).fetchone()
-            if row is None:
-                hits = self._fts(title, limit=1, weights=self._W_LOOKUP)
-                if hits:
-                    row = self.con.execute(
-                        "SELECT * FROM journals WHERE scopus_id = ?", (hits[0][0],)
-                    ).fetchone()
-        return self._to_record(row) if row else None
-
-    def _to_record(self, row: sqlite3.Row) -> JournalRecord:
-        subs = [
-            {"asjc": r["asjc"], "name": r["name"], "parent": r["parent_name"],
-             "quartile": r["quartile"], "is_primary": bool(r["is_primary"])}
-            for r in self.con.execute(
-                """SELECT ja.asjc, ja.is_primary, ja.quartile, a.name, a.parent_name
-                   FROM journal_asjc ja JOIN asjc a ON a.code = ja.asjc
-                   WHERE ja.scopus_id = ? ORDER BY ja.is_primary DESC, ja.asjc""",
-                (row["scopus_id"],),
-            )
-        ]
-        rec = JournalRecord(
-            scopus_id=row["scopus_id"], title=row["title"],
-            issn_print=self._fmt(row["issn_print"]), issn_e=self._fmt(row["issn_e"]),
-            source_type=row["source_type"], publisher=row["publisher"],
-            publisher_group=row["publisher_group"],
-            coverage_start=row["coverage_start"], coverage_end=row["coverage_end"],
-            coverage_has_gap=bool(row["coverage_has_gap"]),
-            oa_unpaywall=bool(row["oa_unpaywall"]), medline=bool(row["medline"]),
-            has_english=bool(row["has_english"]), languages=row["languages"],
-            sjr=row["sjr"], sjr_rank=row["sjr_rank"],
-            sjr_best_quartile=row["sjr_best_quartile"], h_index=row["h_index"],
-            cites_per_doc_2y=row["cites_per_doc_2y"],
-            total_docs_year=row["total_docs_year"],
-            oa_diamond=bool(row["oa_diamond"]),
-            scimago_oa=bool(row["scimago_oa"]), overton=row["overton"],
-            country=row["country"], scimago_id=row["scimago_id"],
-            lifecycle=row["lifecycle"],
-            successor_title=row["successor_title"],
-            successor_scopus_id=row["successor_scopus_id"],
-            successor_is_active=bool(row["successor_is_active"]),
-            subjects=subs,
-        )
-        rec.state = self._state(row)
-        rec.state_message = self._message(rec)
-        return rec
-
-    @staticmethod
-    def _message(rec: "JournalRecord") -> str:
-        """Base message, extended with the successor when we resolved one."""
-        msg = IndexationState.MESSAGES[rec.state]
-        if rec.successor_title:
-            where = (
-                "which is actively indexed"
-                if rec.successor_is_active
-                else "which is itself no longer indexed"
-            )
-            if rec.state == IndexationState.RENAMED:
-                msg += f" It continues as {rec.successor_title!r}, {where}."
-            elif rec.state in (IndexationState.DISCONTINUED, IndexationState.CEASED):
-                # A successor on a discontinued title does not soften the
-                # warning — it just tells the researcher where to look next.
-                msg += f" Scopus records a successor title, {rec.successor_title!r}, {where}."
-        return msg
-
-    @staticmethod
-    def _fmt(s: str | None) -> str | None:
-        return f"{s[:4]}-{s[4:]}" if s and len(s) == 8 else None
-
-    _STATE_BY_STATUS = {
-        "matched": IndexationState.INDEXED_RANKED,
-        "no_sjr_new_title": IndexationState.INDEXED_NEW,
-        "no_sjr_listed_unranked": IndexationState.INDEXED_UNRANKED,
-        "no_sjr_unmatched": IndexationState.INDEXED_NO_SJR,
-    }
-
-    _STATE_BY_LIFECYCLE = {
-        "discontinued": IndexationState.DISCONTINUED,
-        "renamed": IndexationState.RENAMED,
-        "ceased": IndexationState.CEASED,
-    }
-
-    @classmethod
-    def _state(cls, row: sqlite3.Row) -> str:
-        # Lifecycle wins over SJR status. A discontinued title may still carry a
-        # quartile from when it was indexed, and reporting that quartile as the
-        # headline would bury the thing the researcher actually needs to know.
-        lc = cls._STATE_BY_LIFECYCLE.get(row["lifecycle"])
-        if lc:
-            return lc
-        return cls._STATE_BY_STATUS.get(
-            row["sjr_status"], IndexationState.INDEXED_NO_SJR
-        )
-
-    # -- candidate generation ----------------------------------------------
-    # bm25 column weights: (scopus_id, title, publisher_group, subjects).
-    # Publisher is weighted to zero for topical work — "University Library
-    # System" is not evidence that a cinema journal wants your library paper,
-    # and leaving it at 1.0 lets one publisher flood the candidate pool. It
-    # stays queryable for name lookups via _W_LOOKUP.
-    _W_TOPIC = (0.0, 4.0, 0.0, 3.0)
-    _W_LOOKUP = (0.0, 8.0, 1.0, 0.5)
-
-    def _fts(self, text: str, limit: int = 400,
-             asjc: Sequence[str] | None = None,
-             weights: tuple[float, ...] | None = None,
-             active_only: bool = False) -> list[tuple[str, float]]:
-        terms = self._terms(text)
-        if not terms:
-            return []
-        w = weights or self._W_TOPIC
-        query = " OR ".join(f'"{t}"' for t in terms)
-        sql = f"""
-            SELECT f.scopus_id AS sid,
-                   bm25(journals_fts, {w[0]}, {w[1]}, {w[2]}, {w[3]}) AS rank
-            FROM journals_fts f WHERE journals_fts MATCH ?
-        """
-        params: list = [query]
-        if active_only:
-            # Applied inside candidate generation, not after: the index holds
-            # 16,838 inactive titles, and letting them consume the candidate
-            # pool would silently shrink the number of real options considered.
-            sql += " AND f.scopus_id IN (SELECT scopus_id FROM journals WHERE active = 1)"
-        if asjc:
-            marks = ",".join("?" * len(asjc))
-            sql += f""" AND f.scopus_id IN (
-                        SELECT scopus_id FROM journal_asjc WHERE asjc IN ({marks}))"""
-            params += list(asjc)
-        sql += " ORDER BY rank LIMIT ?"
-        params.append(limit)
-        return [(r["sid"], r["rank"]) for r in self.con.execute(sql, params)]
-
-    @staticmethod
-    def _terms(text: str, max_terms: int = 24) -> list[str]:
-        """
-        FTS5 treats bare punctuation as syntax, so terms are stripped to word
-        characters and quoted. Frequency-ordered, stopword-filtered.
-        """
-        from collections import Counter
-        words = re.findall(r"[A-Za-z][A-Za-z\-]{2,}", (text or "").lower())
-        words = [w.strip("-") for w in words if w not in _STOP and len(w) > 3]
-        return [w for w, _ in Counter(words).most_common(max_terms)]
-
-    # -- recommend ----------------------------------------------------------
-    def recommend(
-        self,
-        title: str = "",
-        abstract: str = "",
-        asjc: Sequence[str] | None = None,
-        *,
-        limit: int = 20,
-        candidate_pool: int = 400,
-        require_english: bool = True,
-        require_oa: bool = False,
-        exclude_book_series: bool = True,
-        min_quartile: str | None = None,
-        publisher_agreements: Iterable[str] = (),
-        embedder=None,
-        centroids=None,
-        asjc_classifier=None,
-    ) -> list[Recommendation]:
-        """
-        `asjc`  — subject codes to hard-filter on. Supply these when you know the
-                  manuscript's field; recall from free text alone is mediocre.
-        `min_quartile` — 'Q1' or 'Q2'. Applied against the quartile in the
-                  filtered subject area when `asjc` is given, otherwise against
-                  the journal's best quartile.
-        `publisher_agreements` — publisher_group names the institution holds a
-                  read-and-publish deal with; matches get a scoring bonus and an
-                  explicit reason string.
-        `embedder`/`centroids` — optional stage 2. See module docstring.
-        `asjc_classifier` — an AsjcClassifier. When `asjc` is not supplied, the
-                  subject codes are inferred from the manuscript rather than
-                  left empty. This matters more than any other option here: the
-                  subject filter is what makes the results usable, and an
-                  unfiltered free-text search returns unrelated journals.
-        """
-        inferred = None
-        if not asjc and asjc_classifier is not None:
-            inferred = asjc_classifier.classify(title=title, abstract=abstract)
-            asjc = inferred.codes or None
-
-        text = f"{title} {title} {abstract}".strip()
-        cands = self._fts(text, limit=candidate_pool, asjc=asjc, active_only=True)
-        if not cands and asjc:
-            marks = ",".join("?" * len(asjc))
-            cands = [
-                (r["scopus_id"], 0.0)
-                for r in self.con.execute(
-                    f"""SELECT DISTINCT j.scopus_id FROM journals j
-                        JOIN journal_asjc ja ON ja.scopus_id = j.scopus_id
-                        WHERE ja.asjc IN ({marks}) AND j.active = 1
-                        ORDER BY j.sjr DESC LIMIT ?""",
-                    (*asjc, candidate_pool),
-                )
-            ]
-        if not cands:
-            return []
-
-        ranks = [r for _, r in cands]
-        lo, hi = min(ranks), max(ranks)
-        span = (hi - lo) or 1.0
-
-        method = "lexical+fit"
-        sem: dict[str, float] = {}
-        if embedder is not None and centroids is not None:
-            sem = self._semantic(text, [c for c, _ in cands], embedder, centroids)
-            if sem:
-                method = "lexical+semantic+fit"
-
-        out: list[Recommendation] = []
-        agreements = {p.lower() for p in publisher_agreements}
-        for sid, rank in cands:
-            row = self.con.execute(
-                "SELECT * FROM journals WHERE scopus_id = ?", (sid,)
-            ).fetchone()
-            if row is None:
-                continue
-            if exclude_book_series and row["source_type"] != "Journal":
-                continue
-            if require_english and not row["has_english"]:
-                continue
-            if require_oa and not row["oa_unpaywall"]:
-                continue
-            # Never recommend somewhere Scopus no longer indexes — discontinued,
-            # renamed or ceased. Lookup still returns these (a researcher asking
-            # about a specific title needs the answer); recommendation must not.
-            if not row["active"]:
-                continue
-
-            rec = self._to_record(row)
-            q = rec.quartile_for(asjc[0]) if asjc else rec.sjr_best_quartile
-            q = q or rec.sjr_best_quartile
-            if min_quartile and (q is None or q > min_quartile):
-                continue
-
-            # bm25 returns lower-is-better; invert to 0..1
-            lex = 1.0 - ((rank - lo) / span)
-            score = 0.45 * lex
-            reasons: list[str] = []
-
-            if sem:
-                score = 0.30 * lex + 0.30 * sem.get(sid, 0.0)
-
-            qbonus = {"Q1": 0.25, "Q2": 0.18, "Q3": 0.10, "Q4": 0.04}.get(q or "", 0.0)
-            score += qbonus
-            if q:
-                label = f"{q} in {self._asjc_name(asjc[0])}" if asjc else f"{q} (best quartile)"
-                reasons.append(label)
-            elif rec.state == IndexationState.INDEXED_NEW:
-                score += 0.06
-                reasons.append("Newly indexed — no quartile yet, not a quality signal")
-
-            if rec.coverage_end and rec.coverage_end >= 2026:
-                score += 0.05
-            else:
-                reasons.append(f"Coverage appears to end {rec.coverage_end} — verify status")
-
-            if rec.oa_diamond:
-                # Diamond = no APC and no subscription. For an author without
-                # grant funding this outranks quartile as a practical criterion,
-                # so it is weighted above the generic OA flag.
-                score += 0.10
-                reasons.append("Diamond open access — no APC")
-            elif rec.oa_unpaywall:
-                score += 0.06
-                reasons.append("Open access")
-
-            if rec.publisher_group and rec.publisher_group.lower() in agreements:
-                score += 0.12
-                reasons.append(f"Covered by your {rec.publisher_group} agreement — APC likely waived")
-
-            if rec.coverage_has_gap:
-                score -= 0.04
-                reasons.append("Discontinuous Scopus coverage")
-
-            if rec.medline:
-                reasons.append("Medline-sourced")
-
-            out.append(Recommendation(rec, round(score, 4), reasons, method))
-
-        out.sort(key=lambda r: r.score, reverse=True)
-        out = out[:limit]
-
-        # An inferred subject filter silently determines the entire result set,
-        # so it is never left implicit: every result says which subject was
-        # inferred and how, giving the user something concrete to correct when
-        # the classifier picked the wrong field.
-        if inferred is not None:
-            note = f"Subject inferred as {', '.join(inferred.names)} ({inferred.method})"
-            for r in out:
-                r.reasons.append(note)
-                r.method = f"{r.method}+asjc:{inferred.method}"
-        return out
-
-    def _asjc_name(self, code: str) -> str:
-        r = self.con.execute("SELECT name, parent_name FROM asjc WHERE code = ?",
-                             (code,)).fetchone()
-        return (r["name"] or r["parent_name"]) if r else code
-
-    @staticmethod
-    def _semantic(text, sids, embedder, centroids) -> dict[str, float]:
-        import numpy as np
-        v = np.asarray(embedder(text), dtype="float32").ravel()
-        v /= (np.linalg.norm(v) or 1.0)
-        got = {s: centroids[s] for s in sids if s in centroids}
-        if not got:
-            return {}
-        keys = list(got)
-        M = np.vstack([got[k] for k in keys]).astype("float32")
-        M /= (np.linalg.norm(M, axis=1, keepdims=True) + 1e-9)
-        sims = M @ v
-        lo, hi = float(sims.min()), float(sims.max())
-        span = (hi - lo) or 1.0
-        return {k: float((s - lo) / span) for k, s in zip(keys, sims)}
-
-    # -- subject helpers ----------------------------------------------------
-    def find_asjc(self, query: str, limit: int = 10) -> list[dict]:
-        """Resolve a free-text subject to ASJC codes, for populating the filter."""
-        like = f"%{query.strip().lower()}%"
-        return [
-            dict(r) for r in self.con.execute(
-                """SELECT code, name, parent_code, parent_name, n_journals FROM asjc
-                   WHERE lower(name) LIKE ? OR lower(parent_name) LIKE ?
-                   ORDER BY n_journals DESC LIMIT ?""",
-                (like, like, limit),
-            )
-        ]
-
-    def publisher_titles(self, publisher_group: str) -> int:
-        return self.con.execute(
-            "SELECT count(*) FROM journals WHERE publisher_group = ?",
-            (publisher_group,),
-        ).fetchone()[0]
+"""
+ResearchBee journal index + recommender.
+
+    from researchbee_journals import JournalIndex
+
+    idx = JournalIndex("journals.db")
+    idx.lookup(issn="0737-8831")            # -> JournalRecord (Library Hi Tech)
+    idx.recommend(title=..., abstract=...)  # -> [Recommendation, ...]
+
+Architecture
+------------
+Three stages, deliberately separated so the expensive one stays optional:
+
+  1. CANDIDATE GENERATION  — hard ASJC filter (when subject codes are known) plus
+     FTS5/BM25 over title + publisher + subject names. Pure SQLite, no model,
+     sub-10ms, returns a few hundred candidates.
+
+  2. SEMANTIC RERANK       — optional. Cosine similarity between the manuscript
+     embedding and a per-journal centroid. See the honest caveat below.
+
+  3. FIT SCORING           — quartile, indexation state, OA status, coverage
+     recency, language, publisher agreement coverage. Deterministic, explainable,
+     and the part that should drive the text ResearchBee shows the researcher.
+
+The caveat on stage 2
+---------------------
+The Scopus Source List carries no aims-and-scope text. Embedding a journal
+*title* against a manuscript *abstract* is a weak signal — it recovers obvious
+lexical matches ("Journal of Hydrology" for a hydrology paper) and little else.
+Real matchers (Jane, Elsevier Journal Finder, WoS Match Manuscript) compare the
+manuscript against *articles the journal has published*. To reach that quality
+you need article-level text, which means building journal centroids from OpenAlex
+works — `build_centroids.py` does this. Until centroids exist, `recommend()`
+runs stages 1 and 3 only and says so in `Recommendation.method`, rather than
+quietly returning title-similarity noise dressed up as semantic matching.
+"""
+from __future__ import annotations
+
+import re
+import sqlite3
+import threading
+from dataclasses import dataclass, field, asdict
+from pathlib import Path
+from typing import Iterable, Sequence
+
+__all__ = ["JournalIndex", "JournalRecord", "Recommendation", "IndexationState"]
+
+
+# ---------------------------------------------------------------------------
+# Indexation state — the distinction the old lookup collapsed
+# ---------------------------------------------------------------------------
+class IndexationState:
+    """
+    Why this exists: 'no quartile' is three materially different answers to a
+    researcher deciding where to submit.
+
+      INDEXED_RANKED   in Scopus, has a current SJR and quartile. (30,405)
+      INDEXED_NEW      in Scopus, coverage began 2024+, SJR not yet computed.
+                       SJR needs a ~3-year citation window, so the absence is a
+                       timing artefact, not a quality signal. Safe to submit to.
+                       (326)
+      INDEXED_UNRANKED SCImago has a record but computes no SJR — typically below
+                       its citable-document threshold. Thin but real. (279)
+      INDEXED_NO_SJR   in Scopus, absent from SCImago entirely. Overwhelmingly
+                       book series (1,052), which SJR does not rank at all. Only
+                       ~70 are long-running journals, mostly regional titles.
+                       (1,366)
+      NOT_INDEXED      no Scopus source record at all, active or inactive.
+
+    And three states for titles Scopus no longer indexes. Keeping them apart is
+    the whole point: of 16,838 inactive titles only 967 were removed for cause.
+    Telling a researcher that a journal which merely changed its name is a
+    quality risk is worse than saying nothing.
+
+      DISCONTINUED     removed by Scopus for cause. The real red flag. (967)
+      RENAMED          ceased under this title but the work continues elsewhere;
+                       `successor_title` names where. Benign — redirect, don't
+                       warn. (~7,000 with a resolvable successor)
+      CEASED           stopped being indexed, no successor recorded and no
+                       discontinuation flag. Usually a journal that folded.
+    """
+    INDEXED_RANKED = "indexed_ranked"
+    INDEXED_NEW = "indexed_new"
+    INDEXED_UNRANKED = "indexed_unranked"
+    INDEXED_NO_SJR = "indexed_no_sjr"
+    NOT_INDEXED = "not_indexed"
+    DISCONTINUED = "discontinued"
+    RENAMED = "renamed"
+    CEASED = "ceased"
+
+    #: States a prospective author should be actively warned about.
+    WARN = frozenset({DISCONTINUED})
+    #: States meaning "Scopus does not index new articles here".
+    NOT_CURRENTLY_INDEXED = frozenset({DISCONTINUED, RENAMED, CEASED, NOT_INDEXED})
+
+    MESSAGES = {
+        INDEXED_RANKED: "Indexed in Scopus with a current SJR quartile.",
+        INDEXED_NEW: (
+            "Indexed in Scopus, but coverage is too recent for an SJR quartile. "
+            "SJR requires roughly three years of citation data; expect a first "
+            "quartile assignment once that window closes. The missing quartile "
+            "is not a quality signal."
+        ),
+        INDEXED_UNRANKED: (
+            "Indexed in Scopus and listed by SCImago, but no SJR is computed — "
+            "usually because the title publishes too few citable documents to "
+            "meet SCImago's threshold."
+        ),
+        INDEXED_NO_SJR: (
+            "Indexed in Scopus but absent from SCImago. Normal for book series "
+            "and trade journals, which SJR does not rank. Check the source type "
+            "before treating this as a concern."
+        ),
+        NOT_INDEXED: (
+            "No Scopus source record found — this title is not indexed in "
+            "Scopus, and has no record of having been."
+        ),
+        DISCONTINUED: (
+            "Scopus has discontinued this title — it was removed from the index "
+            "rather than ceasing publication. Articles published here will not "
+            "be indexed in Scopus. Scopus does not publish the reason for an "
+            "individual discontinuation; grounds range from publication-ethics "
+            "concerns to metrics such as self-citation, and discontinued titles "
+            "include some from major publishers. Verify independently before "
+            "submitting, and do not assume the journal is predatory."
+        ),
+        RENAMED: (
+            "This title is no longer indexed under this name, but the journal "
+            "continues. This is a rename or merger, not a quality concern."
+        ),
+        CEASED: (
+            "Scopus no longer indexes this title and no successor is recorded — "
+            "the journal appears to have ceased publication. Not flagged as a "
+            "discontinuation for cause."
+        ),
+    }
+
+
+_ISSN_RE = re.compile(r"[^0-9X]")
+
+
+def norm_issn(v: str | None) -> str | None:
+    if not v:
+        return None
+    s = _ISSN_RE.sub("", str(v).strip().upper())
+    if len(s) < 8:
+        s = s.zfill(8)
+    return s if len(s) == 8 else None
+
+
+# ---------------------------------------------------------------------------
+# Records
+# ---------------------------------------------------------------------------
+@dataclass
+class JournalRecord:
+    scopus_id: str
+    title: str
+    issn_print: str | None
+    issn_e: str | None
+    source_type: str | None
+    publisher: str | None
+    publisher_group: str | None
+    coverage_start: int | None
+    coverage_end: int | None
+    coverage_has_gap: bool
+    oa_unpaywall: bool
+    medline: bool
+    has_english: bool
+    languages: str | None
+    sjr: float | None
+    sjr_rank: int | None
+    sjr_best_quartile: str | None
+    h_index: int | None
+    cites_per_doc_2y: float | None
+    total_docs_year: int | None
+    oa_diamond: bool
+    scimago_oa: bool
+    overton: int | None
+    country: str | None
+    scimago_id: str | None = None
+    lifecycle: str = "active"
+    successor_title: str | None = None
+    successor_scopus_id: str | None = None
+    successor_is_active: bool = False
+    subjects: list[dict] = field(default_factory=list)
+    state: str = IndexationState.NOT_INDEXED
+    state_message: str = ""
+
+    @property
+    def warn(self) -> bool:
+        """True when a prospective author should be actively cautioned."""
+        return self.state in IndexationState.WARN
+
+    @property
+    def currently_indexed(self) -> bool:
+        return self.state not in IndexationState.NOT_CURRENTLY_INDEXED
+
+    def quartile_for(self, asjc: str) -> str | None:
+        """Quartile in one specific subject area, not the journal's best."""
+        for s in self.subjects:
+            if s["asjc"] == asjc:
+                return s.get("quartile")
+        return None
+
+    def to_dict(self) -> dict:
+        return asdict(self)
+
+
+@dataclass
+class Recommendation:
+    journal: JournalRecord
+    score: float
+    reasons: list[str]
+    method: str
+
+
+# ---------------------------------------------------------------------------
+# Index
+# ---------------------------------------------------------------------------
+_STOP = set("""
+a an the and or but if of in on at to for with by from as is are was were be been
+being this that these those it its we our they their he she his her you your not
+no can may might will would should could have has had do does did than then so
+such other others new novel study studies paper article research results method
+methods approach based using used use show shows shown propose proposed present
+presented provide provides between among during through both each more most many
+much also however therefore thus while when where which who whom what how why
+""".split())
+
+
+class JournalIndex:
+    """Thread-safe read-only wrapper over journals.db."""
+
+    def __init__(self, db_path: str | Path):
+        self.db_path = str(db_path)
+        if not Path(self.db_path).exists():
+            raise FileNotFoundError(self.db_path)
+        self._local = threading.local()
+
+    @property
+    def con(self) -> sqlite3.Connection:
+        con = getattr(self._local, "con", None)
+        if con is None:
+            con = sqlite3.connect(f"file:{self.db_path}?mode=ro", uri=True,
+                                  check_same_thread=False)
+            con.row_factory = sqlite3.Row
+            con.execute("PRAGMA mmap_size = 268435456")
+            self._local.con = con
+        return con
+
+    # -- meta ---------------------------------------------------------------
+    def meta(self) -> dict[str, str]:
+        return {r["key"]: r["value"] for r in self.con.execute("SELECT * FROM meta")}
+
+    # -- lookup -------------------------------------------------------------
+    def lookup(self, issn: str | None = None, title: str | None = None,
+               scopus_id: str | None = None) -> JournalRecord | None:
+        row = None
+        if scopus_id:
+            row = self.con.execute(
+                "SELECT * FROM journals WHERE scopus_id = ?", (scopus_id,)
+            ).fetchone()
+        if row is None and issn:
+            key = norm_issn(issn)
+            if key:
+                row = self.con.execute(
+                    "SELECT * FROM journals WHERE issn_e = ? OR issn_print = ? LIMIT 1",
+                    (key, key),
+                ).fetchone()
+        if row is None and title:
+            row = self.con.execute(
+                "SELECT * FROM journals WHERE lower(title) = lower(?) LIMIT 1",
+                (title.strip(),),
+            ).fetchone()
+            if row is None:
+                hits = self._fts(title, limit=1, weights=self._W_LOOKUP)
+                if hits:
+                    row = self.con.execute(
+                        "SELECT * FROM journals WHERE scopus_id = ?", (hits[0][0],)
+                    ).fetchone()
+        return self._to_record(row) if row else None
+
+    def _to_record(self, row: sqlite3.Row) -> JournalRecord:
+        subs = [
+            {"asjc": r["asjc"], "name": r["name"], "parent": r["parent_name"],
+             "quartile": r["quartile"], "is_primary": bool(r["is_primary"])}
+            for r in self.con.execute(
+                """SELECT ja.asjc, ja.is_primary, ja.quartile, a.name, a.parent_name
+                   FROM journal_asjc ja JOIN asjc a ON a.code = ja.asjc
+                   WHERE ja.scopus_id = ? ORDER BY ja.is_primary DESC, ja.asjc""",
+                (row["scopus_id"],),
+            )
+        ]
+        rec = JournalRecord(
+            scopus_id=row["scopus_id"], title=row["title"],
+            issn_print=self._fmt(row["issn_print"]), issn_e=self._fmt(row["issn_e"]),
+            source_type=row["source_type"], publisher=row["publisher"],
+            publisher_group=row["publisher_group"],
+            coverage_start=row["coverage_start"], coverage_end=row["coverage_end"],
+            coverage_has_gap=bool(row["coverage_has_gap"]),
+            oa_unpaywall=bool(row["oa_unpaywall"]), medline=bool(row["medline"]),
+            has_english=bool(row["has_english"]), languages=row["languages"],
+            sjr=row["sjr"], sjr_rank=row["sjr_rank"],
+            sjr_best_quartile=row["sjr_best_quartile"], h_index=row["h_index"],
+            cites_per_doc_2y=row["cites_per_doc_2y"],
+            total_docs_year=row["total_docs_year"],
+            oa_diamond=bool(row["oa_diamond"]),
+            scimago_oa=bool(row["scimago_oa"]), overton=row["overton"],
+            country=row["country"], scimago_id=row["scimago_id"],
+            lifecycle=row["lifecycle"],
+            successor_title=row["successor_title"],
+            successor_scopus_id=row["successor_scopus_id"],
+            successor_is_active=bool(row["successor_is_active"]),
+            subjects=subs,
+        )
+        rec.state = self._state(row)
+        rec.state_message = self._message(rec)
+        return rec
+
+    @staticmethod
+    def _message(rec: "JournalRecord") -> str:
+        """Base message, extended with the successor when we resolved one."""
+        msg = IndexationState.MESSAGES[rec.state]
+        if rec.successor_title:
+            where = (
+                "which is actively indexed"
+                if rec.successor_is_active
+                else "which is itself no longer indexed"
+            )
+            if rec.state == IndexationState.RENAMED:
+                msg += f" It continues as {rec.successor_title!r}, {where}."
+            elif rec.state in (IndexationState.DISCONTINUED, IndexationState.CEASED):
+                # A successor on a discontinued title does not soften the
+                # warning — it just tells the researcher where to look next.
+                msg += f" Scopus records a successor title, {rec.successor_title!r}, {where}."
+        return msg
+
+    @staticmethod
+    def _fmt(s: str | None) -> str | None:
+        return f"{s[:4]}-{s[4:]}" if s and len(s) == 8 else None
+
+    _STATE_BY_STATUS = {
+        "matched": IndexationState.INDEXED_RANKED,
+        "no_sjr_new_title": IndexationState.INDEXED_NEW,
+        "no_sjr_listed_unranked": IndexationState.INDEXED_UNRANKED,
+        "no_sjr_unmatched": IndexationState.INDEXED_NO_SJR,
+    }
+
+    _STATE_BY_LIFECYCLE = {
+        "discontinued": IndexationState.DISCONTINUED,
+        "renamed": IndexationState.RENAMED,
+        "ceased": IndexationState.CEASED,
+    }
+
+    @classmethod
+    def _state(cls, row: sqlite3.Row) -> str:
+        # Lifecycle wins over SJR status. A discontinued title may still carry a
+        # quartile from when it was indexed, and reporting that quartile as the
+        # headline would bury the thing the researcher actually needs to know.
+        lc = cls._STATE_BY_LIFECYCLE.get(row["lifecycle"])
+        if lc:
+            return lc
+        return cls._STATE_BY_STATUS.get(
+            row["sjr_status"], IndexationState.INDEXED_NO_SJR
+        )
+
+    # -- candidate generation ----------------------------------------------
+    # bm25 column weights: (scopus_id, title, publisher_group, subjects).
+    # Publisher is weighted to zero for topical work — "University Library
+    # System" is not evidence that a cinema journal wants your library paper,
+    # and leaving it at 1.0 lets one publisher flood the candidate pool. It
+    # stays queryable for name lookups via _W_LOOKUP.
+    _W_TOPIC = (0.0, 4.0, 0.0, 3.0)
+    _W_LOOKUP = (0.0, 8.0, 1.0, 0.5)
+
+    def _fts(self, text: str, limit: int = 400,
+             asjc: Sequence[str] | None = None,
+             weights: tuple[float, ...] | None = None,
+             active_only: bool = False) -> list[tuple[str, float]]:
+        terms = self._terms(text)
+        if not terms:
+            return []
+        w = weights or self._W_TOPIC
+        query = " OR ".join(f'"{t}"' for t in terms)
+        sql = f"""
+            SELECT f.scopus_id AS sid,
+                   bm25(journals_fts, {w[0]}, {w[1]}, {w[2]}, {w[3]}) AS rank
+            FROM journals_fts f WHERE journals_fts MATCH ?
+        """
+        params: list = [query]
+        if active_only:
+            # Applied inside candidate generation, not after: the index holds
+            # 16,838 inactive titles, and letting them consume the candidate
+            # pool would silently shrink the number of real options considered.
+            sql += " AND f.scopus_id IN (SELECT scopus_id FROM journals WHERE active = 1)"
+        if asjc:
+            marks = ",".join("?" * len(asjc))
+            sql += f""" AND f.scopus_id IN (
+                        SELECT scopus_id FROM journal_asjc WHERE asjc IN ({marks}))"""
+            params += list(asjc)
+        sql += " ORDER BY rank LIMIT ?"
+        params.append(limit)
+        return [(r["sid"], r["rank"]) for r in self.con.execute(sql, params)]
+
+    @staticmethod
+    def _terms(text: str, max_terms: int = 24) -> list[str]:
+        """
+        FTS5 treats bare punctuation as syntax, so terms are stripped to word
+        characters and quoted. Frequency-ordered, stopword-filtered.
+        """
+        from collections import Counter
+        words = re.findall(r"[A-Za-z][A-Za-z\-]{2,}", (text or "").lower())
+        words = [w.strip("-") for w in words if w not in _STOP and len(w) > 3]
+        return [w for w, _ in Counter(words).most_common(max_terms)]
+
+    # -- recommend ----------------------------------------------------------
+    def recommend(
+        self,
+        title: str = "",
+        abstract: str = "",
+        asjc: Sequence[str] | None = None,
+        *,
+        limit: int = 20,
+        candidate_pool: int = 400,
+        require_english: bool = True,
+        require_oa: bool = False,
+        exclude_book_series: bool = True,
+        min_quartile: str | None = None,
+        publisher_agreements: Iterable[str] = (),
+        embedder=None,
+        centroids=None,
+        asjc_classifier=None,
+    ) -> list[Recommendation]:
+        """
+        `asjc`  — subject codes to hard-filter on. Supply these when you know the
+                  manuscript's field; recall from free text alone is mediocre.
+        `min_quartile` — 'Q1' or 'Q2'. Applied against the quartile in the
+                  filtered subject area when `asjc` is given, otherwise against
+                  the journal's best quartile.
+        `publisher_agreements` — publisher_group names the institution holds a
+                  read-and-publish deal with; matches get a scoring bonus and an
+                  explicit reason string.
+        `embedder`/`centroids` — optional stage 2. See module docstring.
+        `asjc_classifier` — an AsjcClassifier. When `asjc` is not supplied, the
+                  subject codes are inferred from the manuscript rather than
+                  left empty. This matters more than any other option here: the
+                  subject filter is what makes the results usable, and an
+                  unfiltered free-text search returns unrelated journals.
+        """
+        inferred = None
+        if not asjc and asjc_classifier is not None:
+            inferred = asjc_classifier.classify(title=title, abstract=abstract)
+            asjc = inferred.codes or None
+
+        text = f"{title} {title} {abstract}".strip()
+        cands = self._fts(text, limit=candidate_pool, asjc=asjc, active_only=True)
+        if not cands and asjc:
+            marks = ",".join("?" * len(asjc))
+            cands = [
+                (r["scopus_id"], 0.0)
+                for r in self.con.execute(
+                    f"""SELECT DISTINCT j.scopus_id FROM journals j
+                        JOIN journal_asjc ja ON ja.scopus_id = j.scopus_id
+                        WHERE ja.asjc IN ({marks}) AND j.active = 1
+                        ORDER BY j.sjr DESC LIMIT ?""",
+                    (*asjc, candidate_pool),
+                )
+            ]
+        if not cands:
+            return []
+
+        ranks = [r for _, r in cands]
+        lo, hi = min(ranks), max(ranks)
+        span = (hi - lo) or 1.0
+
+        method = "lexical+fit"
+        sem: dict[str, float] = {}
+        if embedder is not None and centroids is not None:
+            sem = self._semantic(text, [c for c, _ in cands], embedder, centroids)
+            if sem:
+                method = "lexical+semantic+fit"
+
+        out: list[Recommendation] = []
+        agreements = {p.lower() for p in publisher_agreements}
+        for sid, rank in cands:
+            row = self.con.execute(
+                "SELECT * FROM journals WHERE scopus_id = ?", (sid,)
+            ).fetchone()
+            if row is None:
+                continue
+            if exclude_book_series and row["source_type"] != "Journal":
+                continue
+            if require_english and not row["has_english"]:
+                continue
+            if require_oa and not row["oa_unpaywall"]:
+                continue
+            # Never recommend somewhere Scopus no longer indexes — discontinued,
+            # renamed or ceased. Lookup still returns these (a researcher asking
+            # about a specific title needs the answer); recommendation must not.
+            if not row["active"]:
+                continue
+
+            rec = self._to_record(row)
+            q = rec.quartile_for(asjc[0]) if asjc else rec.sjr_best_quartile
+            q = q or rec.sjr_best_quartile
+            if min_quartile and (q is None or q > min_quartile):
+                continue
+
+            # bm25 returns lower-is-better; invert to 0..1
+            lex = 1.0 - ((rank - lo) / span)
+            score = 0.45 * lex
+            reasons: list[str] = []
+
+            if sem:
+                score = 0.30 * lex + 0.30 * sem.get(sid, 0.0)
+
+            qbonus = {"Q1": 0.25, "Q2": 0.18, "Q3": 0.10, "Q4": 0.04}.get(q or "", 0.0)
+            score += qbonus
+            if q:
+                label = f"{q} in {self._asjc_name(asjc[0])}" if asjc else f"{q} (best quartile)"
+                reasons.append(label)
+            elif rec.state == IndexationState.INDEXED_NEW:
+                score += 0.06
+                reasons.append("Newly indexed — no quartile yet, not a quality signal")
+
+            if rec.coverage_end and rec.coverage_end >= 2026:
+                score += 0.05
+            else:
+                reasons.append(f"Coverage appears to end {rec.coverage_end} — verify status")
+
+            if rec.oa_diamond:
+                # Diamond = no APC and no subscription. For an author without
+                # grant funding this outranks quartile as a practical criterion,
+                # so it is weighted above the generic OA flag.
+                score += 0.10
+                reasons.append("Diamond open access — no APC")
+            elif rec.oa_unpaywall:
+                score += 0.06
+                reasons.append("Open access")
+
+            if rec.publisher_group and rec.publisher_group.lower() in agreements:
+                score += 0.12
+                reasons.append(f"Covered by your {rec.publisher_group} agreement — APC likely waived")
+
+            if rec.coverage_has_gap:
+                score -= 0.04
+                reasons.append("Discontinuous Scopus coverage")
+
+            if rec.medline:
+                reasons.append("Medline-sourced")
+
+            out.append(Recommendation(rec, round(score, 4), reasons, method))
+
+        out.sort(key=lambda r: r.score, reverse=True)
+        out = out[:limit]
+
+        # An inferred subject filter silently determines the entire result set,
+        # so it is never left implicit: every result says which subject was
+        # inferred and how, giving the user something concrete to correct when
+        # the classifier picked the wrong field.
+        if inferred is not None:
+            note = f"Subject inferred as {', '.join(inferred.names)} ({inferred.method})"
+            for r in out:
+                r.reasons.append(note)
+                r.method = f"{r.method}+asjc:{inferred.method}"
+        return out
+
+    def _asjc_name(self, code: str) -> str:
+        r = self.con.execute("SELECT name, parent_name FROM asjc WHERE code = ?",
+                             (code,)).fetchone()
+        return (r["name"] or r["parent_name"]) if r else code
+
+    @staticmethod
+    def _semantic(text, sids, embedder, centroids) -> dict[str, float]:
+        import numpy as np
+        v = np.asarray(embedder(text), dtype="float32").ravel()
+        v /= (np.linalg.norm(v) or 1.0)
+        got = {s: centroids[s] for s in sids if s in centroids}
+        if not got:
+            return {}
+        keys = list(got)
+        M = np.vstack([got[k] for k in keys]).astype("float32")
+        M /= (np.linalg.norm(M, axis=1, keepdims=True) + 1e-9)
+        sims = M @ v
+        lo, hi = float(sims.min()), float(sims.max())
+        span = (hi - lo) or 1.0
+        return {k: float((s - lo) / span) for k, s in zip(keys, sims)}
+
+    # -- subject helpers ----------------------------------------------------
+    def find_asjc(self, query: str, limit: int = 10) -> list[dict]:
+        """Resolve a free-text subject to ASJC codes, for populating the filter."""
+        like = f"%{query.strip().lower()}%"
+        return [
+            dict(r) for r in self.con.execute(
+                """SELECT code, name, parent_code, parent_name, n_journals FROM asjc
+                   WHERE lower(name) LIKE ? OR lower(parent_name) LIKE ?
+                   ORDER BY n_journals DESC LIMIT ?""",
+                (like, like, limit),
+            )
+        ]
+
+    def publisher_titles(self, publisher_group: str) -> int:
+        return self.con.execute(
+            "SELECT count(*) FROM journals WHERE publisher_group = ?",
+            (publisher_group,),
+        ).fetchone()[0]

scimago_index.py

@@ -1,299 +1,299 @@
-"""
-SCImago CSV loader — builds in-memory ISSN → journal dict at startup.
-Used PRIVATELY for quartile lookup only. Never exposed as branded data source in UI.
-
-Changes from v1.0:
-- _norm_issn removed — uses utils.norm_issn
-- sourceid now included in ISSN index entries (was missing — broke verify_links)
-"""
-import csv
-import re
-import os
-import difflib
-from typing import Optional, List, Dict, Set, Tuple
-from utils import norm_issn
-
-_scimago_cache: dict = {}
-
-
-# ── Closed-vocabulary parsing ─────────────────────────────────────────────
-#
-# SCImago's Categories column looks like
-#   "astronomy and astrophysics (q1); space and planetary science (q1)"
-# and Areas like
-#   "earth and planetary sciences; physics and astronomy"
-#
-# Both are ";"-delimited lists of names from a CLOSED vocabulary: 337
-# categories and 27 areas. Treating either as free text is what produced
-# the "crime" bug — "sciences" is a substring of 41% of all area strings,
-# so an area word-match returned 13,154 journals, every one scoring
-# identically, leaving h-index to sort the whole database.
-
-_STOPWORDS = {"and", "of", "the", "in", "for", "or", "a", "an", "&"}
-
-_QUARTILE_SUFFIX = re.compile(r"\s*\(q[1-4]\)\s*$")
-
-
-def _split_categories(raw: str) -> Set[str]:
-    """"a (q1); b (q2)" -> {"a", "b"}. Quartile suffix stripped."""
-    out = set()
-    for part in raw.split(";"):
-        name = _QUARTILE_SUFFIX.sub("", part.strip()).strip()
-        if name:
-            out.add(name)
-    return out
-
-
-def _split_areas(raw: str) -> Set[str]:
-    return {p.strip() for p in raw.split(";") if p.strip()}
-
-
-def _tokens(s: str) -> Set[str]:
-    """Significant whole words. Used for containment tests, never substrings."""
-    return {t for t in re.split(r"[^a-z0-9]+", (s or "").lower())
-            if t and t not in _STOPWORDS}
-
-
-# ── Vocabulary accessors ─────────────────────────────────────────────────
-_vocab_cache: Optional[Dict[str, Dict[str, int]]] = None
-
-
-def subject_vocabulary() -> Dict[str, Dict[str, int]]:
-    """
-    The real, closed vocabulary with journal counts.
-
-        {"categories": {"astronomy and astrophysics": 84, ...},
-         "areas":      {"physics and astronomy": 3011, ...}}
-
-    Counts matter: they let the caller drop categories too small to be worth
-    offering, and they are the evidence that a name is real rather than
-    plausible.
-    """
-    global _vocab_cache
-    if _vocab_cache is not None:
-        return _vocab_cache
-    cats: Dict[str, int] = {}
-    areas: Dict[str, int] = {}
-    for r in load_subject_index():
-        for c in r["cat_set"]:
-            cats[c] = cats.get(c, 0) + 1
-        for a in r["area_set"]:
-            areas[a] = areas.get(a, 0) + 1
-    _vocab_cache = {"categories": cats, "areas": areas}
-    return _vocab_cache
-
-
-def is_real_category(name: str) -> bool:
-    return bool(name) and name.strip().lower() in subject_vocabulary()["categories"]
-
-
-def is_real_area(name: str) -> bool:
-    return bool(name) and name.strip().lower() in subject_vocabulary()["areas"]
-
-
-def suggest_categories(text: str, limit: int = 12) -> List[str]:
-    """
-    Deterministic shortlist of REAL category names for a free-text subject.
-    Exact -> whole-word containment -> difflib. Never invents a name.
-
-    This exists so the LLM chooses from a closed list rather than generating
-    one: "Crime and Society" is a plausible SCImago category name and is not
-    a SCImago category at all.
-    """
-    vocab = list(subject_vocabulary()["categories"].keys())
-    q = (text or "").strip().lower()
-    if not q:
-        return []
-    if q in vocab:
-        return [q]
-
-    qt = _tokens(q)
-    ranked: List[Tuple[int, str]] = []
-    for name in vocab:
-        nt = _tokens(name)
-        if not nt:
-            continue
-        if qt and qt <= nt:
-            ranked.append((3, name))
-        elif qt & nt:
-            ranked.append((2, name))
-    ranked.sort(key=lambda x: (-x[0], len(x[1])))
-    out = [n for _, n in ranked][:limit]
-
-    # difflib only as a typo repair, and only at a cutoff high enough to be
-    # about spelling rather than about vibes. At 0.72 it offered "immunology"
-    # for "criminology" (ratio 0.76) — lexically close, semantically absurd.
-    # 0.90 still catches "engineerng" -> "engineering" (0.95).
-    if len(out) < limit:
-        for n in difflib.get_close_matches(q, vocab, n=limit - len(out), cutoff=0.90):
-            if n not in out:
-                out.append(n)
-    return out[:limit]
-
-
-def load_scimago(csv_path: str = None) -> dict:
-    global _scimago_cache
-    if _scimago_cache:
-        return _scimago_cache
-    if csv_path is None:
-        csv_path = os.path.join(os.path.dirname(__file__), "scimagojr_2025.csv")
-    index = {}
-    try:
-        with open(csv_path, encoding="latin-1", newline="") as f:
-            reader = csv.DictReader(f, delimiter=",")
-            for row in reader:
-                issn_raw = row.get("Issn", "")
-                issn_parts = re.split(r"[,;\s]+", issn_raw)
-                entry = {
-                    "title":      row.get("Title", "").strip(),
-                    "rank":       row.get("Rank", "").strip(),
-                    "quartile":   row.get("SJR Best Quartile", "").strip(),
-                    "h_index":    row.get("H index", "").strip(),
-                    "categories": row.get("Categories", "").strip(),
-                    "areas":      row.get("Areas", "").strip(),
-                    "publisher":  row.get("Publisher", "").strip(),
-                    "open_access":row.get("Open Access", "No").strip(),
-                    "country":    row.get("Country", "").strip(),
-                    "sourceid":   row.get("Sourceid", "").strip(),  # FIX: was missing
-                }
-                for part in issn_parts:
-                    normed = norm_issn(part)
-                    if normed:
-                        index[normed] = entry
-        _scimago_cache = index
-        print(f"[SCImago] Loaded {len(index):,} ISSN entries.")
-    except Exception as e:
-        print(f"[SCImago] Failed: {e}")
-        _scimago_cache = {}
-    return _scimago_cache
-
-
-def lookup_any(issns: list) -> Optional[dict]:
-    index = load_scimago()
-    for issn in issns:
-        normed = norm_issn(str(issn)) if issn else None
-        if normed and normed in index:
-            result = dict(index[normed])
-            result["matched_issn"] = normed
-            return result
-    return None
-
-
-# ── Subject search index (separate from ISSN index) ──────────────────────
-_subject_cache: list = []
-
-
-def load_subject_index(csv_path: str = None) -> list:
-    """Load full journal list for subject browsing — separate from ISSN lookup."""
-    global _subject_cache
-    if _subject_cache:
-        return _subject_cache
-    if csv_path is None:
-        csv_path = os.path.join(os.path.dirname(__file__), "scimagojr_2025.csv")
-    rows = []
-    seen_titles = set()
-    try:
-        with open(csv_path, encoding="latin-1", newline="") as f:
-            reader = csv.DictReader(f, delimiter=",")
-            for row in reader:
-                title = row.get("Title", "").strip()
-                if not title or title in seen_titles:
-                    continue
-                seen_titles.add(title)
-                issn_raw = row.get("Issn", "")
-                issn_parts = re.split(r"[,;\s]+", issn_raw)
-                issns = [norm_issn(p) for p in issn_parts if norm_issn(p)]
-                primary_issn = issns[0] if issns else None
-                try:
-                    h = int(row.get("H index", "0").strip() or 0)
-                except Exception:
-                    h = 0
-                cats_raw  = row.get("Categories", "").strip().lower()
-                areas_raw = row.get("Areas", "").strip().lower()
-                rows.append({
-                    "title":      title,
-                    "issn":       primary_issn,
-                    "quartile":   row.get("SJR Best Quartile", "").strip(),
-                    "h_index":    h,
-                    "categories": cats_raw,
-                    "areas":      areas_raw,
-                    # Parsed once at load. Matching against these SETS instead
-                    # of against the joined string is what stops "sciences"
-                    # matching "earth and planetary sciences".
-                    "cat_set":    _split_categories(cats_raw),
-                    "area_set":   _split_areas(areas_raw),
-                    "publisher":  row.get("Publisher", "").strip(),
-                    "open_access":row.get("Open Access", "No").strip(),
-                    "country":    row.get("Country", "").strip(),
-                    "rank":       row.get("Rank", "").strip(),
-                    "sourceid":   row.get("Sourceid", "").strip(),
-                })
-        _subject_cache = rows
-        print(f"[SCImago] Subject index loaded: {len(rows):,} unique journals.")
-    except Exception as e:
-        print(f"[SCImago] Subject index failed: {e}")
-        _subject_cache = []
-    return _subject_cache
-
-
-def search_by_subject(category: str, area: str, top_n: int = 15,
-                      with_level: bool = False):
-    """
-    Journals for a SCImago category (and/or area), best match first.
-
-    Match tiers, highest first:
-        3  category name matches EXACTLY one of the journal's categories
-        2  every significant word of the query is a whole word in one of them
-        1  area name matches EXACTLY one of the journal's areas
-
-    All comparisons are against parsed name SETS. There is deliberately no
-    substring test anywhere: the previous implementation asked whether any
-    word of the area appeared anywhere in the joined area string, and
-    "sciences" is a substring of 13,154 journals' areas. Every one scored 1,
-    so the h-index tie-break silently returned the global top-30 — an
-    astrophysics list for a criminology query.
-
-    An area-only hit (tier 1) is a genuinely weaker answer than a category
-    hit and the caller must be able to say so. Pass with_level=True to get
-    (journals, level) where level is 3, 2, 1 or 0.
-    """
-    rows = load_subject_index()
-    cat_lower  = (category or "").strip().lower()
-    area_lower = (area or "").strip().lower()
-    cat_tokens = _tokens(cat_lower)
-
-    def score(r) -> int:
-        cs = r.get("cat_set") or set()
-        if cat_lower and cat_lower in cs:
-            return 3
-        if cat_tokens and any(cat_tokens <= _tokens(c) for c in cs):
-            return 2
-        if area_lower and area_lower in (r.get("area_set") or set()):
-            return 1
-        return 0
-
-    matched = []
-    for r in rows:
-        s = score(r)
-        if s:
-            matched.append((s, r))
-
-    if not matched:
-        return ([], 0) if with_level else []
-
-    matched.sort(key=lambda x: (-x[0], -x[1]["h_index"]))
-    best = matched[0][0]
-    out = [r for _, r in matched[:top_n]]
-    return (out, best) if with_level else out
-
-
-def get_all_areas() -> list:
-    """Return sorted unique area names for frontend hints."""
-    rows = load_subject_index()
-    areas = set()
-    for r in rows:
-        for a in r["areas"].split(";"):
-            a = a.strip().title()
-            if a:
-                areas.add(a)
+"""
+SCImago CSV loader — builds in-memory ISSN → journal dict at startup.
+Used PRIVATELY for quartile lookup only. Never exposed as branded data source in UI.
+
+Changes from v1.0:
+- _norm_issn removed — uses utils.norm_issn
+- sourceid now included in ISSN index entries (was missing — broke verify_links)
+"""
+import csv
+import re
+import os
+import difflib
+from typing import Optional, List, Dict, Set, Tuple
+from utils import norm_issn
+
+_scimago_cache: dict = {}
+
+
+# ── Closed-vocabulary parsing ─────────────────────────────────────────────
+#
+# SCImago's Categories column looks like
+#   "astronomy and astrophysics (q1); space and planetary science (q1)"
+# and Areas like
+#   "earth and planetary sciences; physics and astronomy"
+#
+# Both are ";"-delimited lists of names from a CLOSED vocabulary: 337
+# categories and 27 areas. Treating either as free text is what produced
+# the "crime" bug — "sciences" is a substring of 41% of all area strings,
+# so an area word-match returned 13,154 journals, every one scoring
+# identically, leaving h-index to sort the whole database.
+
+_STOPWORDS = {"and", "of", "the", "in", "for", "or", "a", "an", "&"}
+
+_QUARTILE_SUFFIX = re.compile(r"\s*\(q[1-4]\)\s*$")
+
+
+def _split_categories(raw: str) -> Set[str]:
+    """"a (q1); b (q2)" -> {"a", "b"}. Quartile suffix stripped."""
+    out = set()
+    for part in raw.split(";"):
+        name = _QUARTILE_SUFFIX.sub("", part.strip()).strip()
+        if name:
+            out.add(name)
+    return out
+
+
+def _split_areas(raw: str) -> Set[str]:
+    return {p.strip() for p in raw.split(";") if p.strip()}
+
+
+def _tokens(s: str) -> Set[str]:
+    """Significant whole words. Used for containment tests, never substrings."""
+    return {t for t in re.split(r"[^a-z0-9]+", (s or "").lower())
+            if t and t not in _STOPWORDS}
+
+
+# ── Vocabulary accessors ─────────────────────────────────────────────────
+_vocab_cache: Optional[Dict[str, Dict[str, int]]] = None
+
+
+def subject_vocabulary() -> Dict[str, Dict[str, int]]:
+    """
+    The real, closed vocabulary with journal counts.
+
+        {"categories": {"astronomy and astrophysics": 84, ...},
+         "areas":      {"physics and astronomy": 3011, ...}}
+
+    Counts matter: they let the caller drop categories too small to be worth
+    offering, and they are the evidence that a name is real rather than
+    plausible.
+    """
+    global _vocab_cache
+    if _vocab_cache is not None:
+        return _vocab_cache
+    cats: Dict[str, int] = {}
+    areas: Dict[str, int] = {}
+    for r in load_subject_index():
+        for c in r["cat_set"]:
+            cats[c] = cats.get(c, 0) + 1
+        for a in r["area_set"]:
+            areas[a] = areas.get(a, 0) + 1
+    _vocab_cache = {"categories": cats, "areas": areas}
+    return _vocab_cache
+
+
+def is_real_category(name: str) -> bool:
+    return bool(name) and name.strip().lower() in subject_vocabulary()["categories"]
+
+
+def is_real_area(name: str) -> bool:
+    return bool(name) and name.strip().lower() in subject_vocabulary()["areas"]
+
+
+def suggest_categories(text: str, limit: int = 12) -> List[str]:
+    """
+    Deterministic shortlist of REAL category names for a free-text subject.
+    Exact -> whole-word containment -> difflib. Never invents a name.
+
+    This exists so the LLM chooses from a closed list rather than generating
+    one: "Crime and Society" is a plausible SCImago category name and is not
+    a SCImago category at all.
+    """
+    vocab = list(subject_vocabulary()["categories"].keys())
+    q = (text or "").strip().lower()
+    if not q:
+        return []
+    if q in vocab:
+        return [q]
+
+    qt = _tokens(q)
+    ranked: List[Tuple[int, str]] = []
+    for name in vocab:
+        nt = _tokens(name)
+        if not nt:
+            continue
+        if qt and qt <= nt:
+            ranked.append((3, name))
+        elif qt & nt:
+            ranked.append((2, name))
+    ranked.sort(key=lambda x: (-x[0], len(x[1])))
+    out = [n for _, n in ranked][:limit]
+
+    # difflib only as a typo repair, and only at a cutoff high enough to be
+    # about spelling rather than about vibes. At 0.72 it offered "immunology"
+    # for "criminology" (ratio 0.76) — lexically close, semantically absurd.
+    # 0.90 still catches "engineerng" -> "engineering" (0.95).
+    if len(out) < limit:
+        for n in difflib.get_close_matches(q, vocab, n=limit - len(out), cutoff=0.90):
+            if n not in out:
+                out.append(n)
+    return out[:limit]
+
+
+def load_scimago(csv_path: str = None) -> dict:
+    global _scimago_cache
+    if _scimago_cache:
+        return _scimago_cache
+    if csv_path is None:
+        csv_path = os.path.join(os.path.dirname(__file__), "scimagojr_2025.csv")
+    index = {}
+    try:
+        with open(csv_path, encoding="latin-1", newline="") as f:
+            reader = csv.DictReader(f, delimiter=",")
+            for row in reader:
+                issn_raw = row.get("Issn", "")
+                issn_parts = re.split(r"[,;\s]+", issn_raw)
+                entry = {
+                    "title":      row.get("Title", "").strip(),
+                    "rank":       row.get("Rank", "").strip(),
+                    "quartile":   row.get("SJR Best Quartile", "").strip(),
+                    "h_index":    row.get("H index", "").strip(),
+                    "categories": row.get("Categories", "").strip(),
+                    "areas":      row.get("Areas", "").strip(),
+                    "publisher":  row.get("Publisher", "").strip(),
+                    "open_access":row.get("Open Access", "No").strip(),
+                    "country":    row.get("Country", "").strip(),
+                    "sourceid":   row.get("Sourceid", "").strip(),  # FIX: was missing
+                }
+                for part in issn_parts:
+                    normed = norm_issn(part)
+                    if normed:
+                        index[normed] = entry
+        _scimago_cache = index
+        print(f"[SCImago] Loaded {len(index):,} ISSN entries.")
+    except Exception as e:
+        print(f"[SCImago] Failed: {e}")
+        _scimago_cache = {}
+    return _scimago_cache
+
+
+def lookup_any(issns: list) -> Optional[dict]:
+    index = load_scimago()
+    for issn in issns:
+        normed = norm_issn(str(issn)) if issn else None
+        if normed and normed in index:
+            result = dict(index[normed])
+            result["matched_issn"] = normed
+            return result
+    return None
+
+
+# ── Subject search index (separate from ISSN index) ──────────────────────
+_subject_cache: list = []
+
+
+def load_subject_index(csv_path: str = None) -> list:
+    """Load full journal list for subject browsing — separate from ISSN lookup."""
+    global _subject_cache
+    if _subject_cache:
+        return _subject_cache
+    if csv_path is None:
+        csv_path = os.path.join(os.path.dirname(__file__), "scimagojr_2025.csv")
+    rows = []
+    seen_titles = set()
+    try:
+        with open(csv_path, encoding="latin-1", newline="") as f:
+            reader = csv.DictReader(f, delimiter=",")
+            for row in reader:
+                title = row.get("Title", "").strip()
+                if not title or title in seen_titles:
+                    continue
+                seen_titles.add(title)
+                issn_raw = row.get("Issn", "")
+                issn_parts = re.split(r"[,;\s]+", issn_raw)
+                issns = [norm_issn(p) for p in issn_parts if norm_issn(p)]
+                primary_issn = issns[0] if issns else None
+                try:
+                    h = int(row.get("H index", "0").strip() or 0)
+                except Exception:
+                    h = 0
+                cats_raw  = row.get("Categories", "").strip().lower()
+                areas_raw = row.get("Areas", "").strip().lower()
+                rows.append({
+                    "title":      title,
+                    "issn":       primary_issn,
+                    "quartile":   row.get("SJR Best Quartile", "").strip(),
+                    "h_index":    h,
+                    "categories": cats_raw,
+                    "areas":      areas_raw,
+                    # Parsed once at load. Matching against these SETS instead
+                    # of against the joined string is what stops "sciences"
+                    # matching "earth and planetary sciences".
+                    "cat_set":    _split_categories(cats_raw),
+                    "area_set":   _split_areas(areas_raw),
+                    "publisher":  row.get("Publisher", "").strip(),
+                    "open_access":row.get("Open Access", "No").strip(),
+                    "country":    row.get("Country", "").strip(),
+                    "rank":       row.get("Rank", "").strip(),
+                    "sourceid":   row.get("Sourceid", "").strip(),
+                })
+        _subject_cache = rows
+        print(f"[SCImago] Subject index loaded: {len(rows):,} unique journals.")
+    except Exception as e:
+        print(f"[SCImago] Subject index failed: {e}")
+        _subject_cache = []
+    return _subject_cache
+
+
+def search_by_subject(category: str, area: str, top_n: int = 15,
+                      with_level: bool = False):
+    """
+    Journals for a SCImago category (and/or area), best match first.
+
+    Match tiers, highest first:
+        3  category name matches EXACTLY one of the journal's categories
+        2  every significant word of the query is a whole word in one of them
+        1  area name matches EXACTLY one of the journal's areas
+
+    All comparisons are against parsed name SETS. There is deliberately no
+    substring test anywhere: the previous implementation asked whether any
+    word of the area appeared anywhere in the joined area string, and
+    "sciences" is a substring of 13,154 journals' areas. Every one scored 1,
+    so the h-index tie-break silently returned the global top-30 — an
+    astrophysics list for a criminology query.
+
+    An area-only hit (tier 1) is a genuinely weaker answer than a category
+    hit and the caller must be able to say so. Pass with_level=True to get
+    (journals, level) where level is 3, 2, 1 or 0.
+    """
+    rows = load_subject_index()
+    cat_lower  = (category or "").strip().lower()
+    area_lower = (area or "").strip().lower()
+    cat_tokens = _tokens(cat_lower)
+
+    def score(r) -> int:
+        cs = r.get("cat_set") or set()
+        if cat_lower and cat_lower in cs:
+            return 3
+        if cat_tokens and any(cat_tokens <= _tokens(c) for c in cs):
+            return 2
+        if area_lower and area_lower in (r.get("area_set") or set()):
+            return 1
+        return 0
+
+    matched = []
+    for r in rows:
+        s = score(r)
+        if s:
+            matched.append((s, r))
+
+    if not matched:
+        return ([], 0) if with_level else []
+
+    matched.sort(key=lambda x: (-x[0], -x[1]["h_index"]))
+    best = matched[0][0]
+    out = [r for _, r in matched[:top_n]]
+    return (out, best) if with_level else out
+
+
+def get_all_areas() -> list:
+    """Return sorted unique area names for frontend hints."""
+    rows = load_subject_index()
+    areas = set()
+    for r in rows:
+        for a in r["areas"].split(";"):
+            a = a.strip().title()
+            if a:
+                areas.add(a)
     return sorted(areas)

style.css

@@ -1,28 +1,28 @@
-body {
-	padding: 2rem;
-	font-family: -apple-system, BlinkMacSystemFont, "Arial", sans-serif;
-}
-
-h1 {
-	font-size: 16px;
-	margin-top: 0;
-}
-
-p {
-	color: rgb(107, 114, 128);
-	font-size: 15px;
-	margin-bottom: 10px;
-	margin-top: 5px;
-}
-
-.card {
-	max-width: 620px;
-	margin: 0 auto;
-	padding: 16px;
-	border: 1px solid lightgray;
-	border-radius: 16px;
-}
-
-.card p:last-child {
-	margin-bottom: 0;
-}
+body {
+	padding: 2rem;
+	font-family: -apple-system, BlinkMacSystemFont, "Arial", sans-serif;
+}
+
+h1 {
+	font-size: 16px;
+	margin-top: 0;
+}
+
+p {
+	color: rgb(107, 114, 128);
+	font-size: 15px;
+	margin-bottom: 10px;
+	margin-top: 5px;
+}
+
+.card {
+	max-width: 620px;
+	margin: 0 auto;
+	padding: 16px;
+	border: 1px solid lightgray;
+	border-radius: 16px;
+}
+
+.card p:last-child {
+	margin-bottom: 0;
+}

train_asjc.py

@@ -1,329 +1,329 @@
-"""
-Train the abstract -> ASJC multi-label classifier.
-
-    # interim model, no network needed — trains on journal titles in the index
-    python train_asjc.py --corpus titles --db journals.db --out asjc_model.joblib
-
-    # full model, after harvest_openalex.py has produced labelled abstracts
-    python train_asjc.py --corpus abstracts --train-jsonl asjc_train.jsonl \
-                         --db journals.db --out asjc_model.joblib
-
-Why TF-IDF + linear and not a transformer
------------------------------------------
-The task is topical multi-label text classification over 334 labels with a large
-vocabulary signal — exactly where a linear model over character and word n-grams
-is hard to beat per unit of cost. It trains in seconds on CPU, the artifact is
-tens of MB, and inference is sub-millisecond with no GPU and no model download at
-Space startup. A fine-tuned transformer might add a few points of F1 for ~400MB
-and a cold-start penalty on every Space restart. Take the linear model; spend the
-complexity budget on the LLM fallback for the cases it actually gets wrong.
-
-Two corpora, deliberately
--------------------------
-`titles`    trains on journal titles (48,888 labelled examples, already in the
-            index, zero network). Ships immediately. But it trains on titles and
-            is applied to abstracts, and that domain gap is real — treat its
-            numbers as an upper bound on what it will do in production.
-
-`abstracts` trains on article abstracts labelled with their journal's ASJC codes
-            (weak supervision). Far better matched to inference, and the only
-            version whose held-out score means what it appears to mean.
-
-Label noise in the `abstracts` corpus
--------------------------------------
-Giving every article its journal's codes is weak supervision: a physics paper in
-a multidisciplinary journal inherits 'Medicine' too. `--max-labels` drops journals
-carrying more than N ASJC codes from training, which removes the worst offenders
-(megajournals and general-science titles) at the cost of some coverage. Their
-articles are still *classifiable* at inference — they are just poor teachers.
-"""
-from __future__ import annotations
-
-import argparse
-import json
-import re
-import sqlite3
-import sys
-from pathlib import Path
-
-import numpy as np
-from scipy import sparse
-from sklearn.feature_extraction.text import TfidfVectorizer
-from sklearn.linear_model import SGDClassifier
-from sklearn.model_selection import train_test_split
-from sklearn.multiclass import OneVsRestClassifier
-from sklearn.preprocessing import MultiLabelBinarizer
-
-import joblib
-
-
-# ---------------------------------------------------------------------------
-# Corpora
-# ---------------------------------------------------------------------------
-def load_titles(db: str, max_labels: int) -> tuple[list[str], list[list[str]]]:
-    """Journal title + subject names -> its ASJC codes."""
-    con = sqlite3.connect(db)
-    con.row_factory = sqlite3.Row
-    rows = con.execute(
-        """SELECT j.scopus_id, j.title, j.publisher_group,
-                  group_concat(ja.asjc) codes, count(*) n
-           FROM journals j JOIN journal_asjc ja ON ja.scopus_id = j.scopus_id
-           WHERE j.title IS NOT NULL
-           GROUP BY j.scopus_id"""
-    ).fetchall()
-    X, y = [], []
-    for r in rows:
-        if r["n"] > max_labels:
-            continue
-        X.append(r["title"])
-        y.append(r["codes"].split(","))
-    con.close()
-    return X, y
-
-
-def load_abstracts(path: str, db: str, max_labels: int) -> tuple[list[str], list[list[str]]]:
-    """
-    JSONL from harvest_openalex.py: {"text": ..., "scopus_id": ...}.
-    Labels come from the index, so the harvest never has to carry them.
-    """
-    con = sqlite3.connect(db)
-    codes: dict[str, list[str]] = {}
-    for sid, c in con.execute(
-        "SELECT scopus_id, group_concat(asjc) FROM journal_asjc GROUP BY scopus_id"
-    ):
-        codes[sid] = c.split(",")
-    con.close()
-
-    X, y = [], []
-    with open(path, encoding="utf-8") as f:
-        for line in f:
-            rec = json.loads(line)
-            c = codes.get(rec.get("scopus_id"))
-            if not c or len(c) > max_labels:
-                continue
-            t = (rec.get("text") or "").strip()
-            if len(t) < 120:          # a bare title is not an abstract
-                continue
-            X.append(t)
-            y.append(c)
-    return X, y
-
-
-# ---------------------------------------------------------------------------
-# Evaluation
-# ---------------------------------------------------------------------------
-def evaluate(clf, Xv, Y_true, classes: np.ndarray, parent_of: dict[str, str]) -> dict:
-    """
-    Ranking metrics, not threshold metrics.
-
-    The classifier feeds a *filter*, and the filter takes the top few codes — so
-    what matters is whether a correct code is ranked highly, not whether some
-    probability cleared 0.5. Reporting only micro-F1 at a fixed threshold would
-    flatter a model that is well-calibrated but badly ordered, and vice versa.
-
-    Parent accuracy (2-digit ASJC) is reported alongside because confusing
-    'Information Systems' with 'Computer Science Applications' barely harms the
-    candidate set, while confusing Computer Science with Dentistry destroys it.
-    """
-    P = clf.predict_proba(Xv)
-    order = np.argsort(-P, axis=1)
-    res: dict[str, float] = {}
-
-    for k in (1, 3, 5):
-        topk = order[:, :k]
-        hit = sum(
-            bool(set(classes[topk[i]]) & set(Y_true[i])) for i in range(len(Y_true))
-        )
-        res[f"top{k}_hit"] = hit / len(Y_true)
-
-    # Recall of the full true label set within the top 5 predictions.
-    top5 = order[:, :5]
-    res["top5_label_recall"] = float(np.mean([
-        len(set(classes[top5[i]]) & set(Y_true[i])) / len(Y_true[i])
-        for i in range(len(Y_true))
-    ]))
-
-    top1_parent = [parent_of.get(classes[order[i, 0]]) for i in range(len(Y_true))]
-    res["top1_parent_hit"] = float(np.mean([
-        top1_parent[i] in {parent_of.get(c) for c in Y_true[i]}
-        for i in range(len(Y_true))
-    ]))
-    return res
-
-
-def main() -> None:
-    ap = argparse.ArgumentParser()
-    ap.add_argument("--corpus", choices=["titles", "abstracts"], default="titles")
-    ap.add_argument("--db", default="journals.db")
-    ap.add_argument("--train-jsonl", default="asjc_train.jsonl")
-    ap.add_argument("--out", default="asjc_model.joblib")
-    ap.add_argument("--max-labels", type=int, default=6,
-                    help="drop journals carrying more ASJC codes than this from training")
-    ap.add_argument("--min-per-label", type=int, default=8,
-                    help="drop labels with fewer training examples than this")
-    ap.add_argument("--eval-jsonl", default="asjc_eval.jsonl")
-    ap.add_argument("--max-features", type=int, default=120_000)
-    ap.add_argument("--n-jobs", type=int, default=2,
-                    help="parallel per-label fits. 334 estimators over a large sparse "
-                         "matrix can exhaust memory at -1; 4 is safe on a 16GB Space.")
-    args = ap.parse_args()
-
-    if args.corpus == "titles":
-        X, y = load_titles(args.db, args.max_labels)
-    else:
-        if not Path(args.train_jsonl).exists():
-            sys.exit(f"{args.train_jsonl} not found — run harvest_openalex.py first")
-        X, y = load_abstracts(args.train_jsonl, args.db, args.max_labels)
-
-    print(f"corpus={args.corpus}  examples={len(X)}  "
-          f"mean labels/example={np.mean([len(c) for c in y]):.2f}")
-
-    # Drop labels too rare to learn. They stay resolvable through the LLM
-    # fallback, which does not need training examples.
-    from collections import Counter
-    freq = Counter(c for cs in y for c in cs)
-    keep = {c for c, n in freq.items() if n >= args.min_per_label}
-    y = [[c for c in cs if c in keep] for cs in y]
-    mask = [i for i, cs in enumerate(y) if cs]
-    X, y = [X[i] for i in mask], [y[i] for i in mask]
-    print(f"labels kept: {len(keep)} of {len(freq)}   examples after filtering: {len(X)}")
-
-    mlb = MultiLabelBinarizer()
-    Y = mlb.fit_transform(y)
-
-    Xtr, Xte, Ytr, Yte, ytr_raw, yte_raw = train_test_split(
-        X, Y, y, test_size=0.2, random_state=42
-    )
-
-    # Word n-grams carry the topical signal; character n-grams cover
-    # morphology, non-English titles and the domain shift between titles and
-    # abstracts. Union of both, sublinear tf to damp long abstracts.
-    word = TfidfVectorizer(
-        sublinear_tf=True, strip_accents="unicode", lowercase=True,
-        ngram_range=(1, 2), min_df=2, max_features=args.max_features,
-        stop_words="english", dtype=np.float32,
-    )
-    char = TfidfVectorizer(
-        sublinear_tf=True, strip_accents="unicode", lowercase=True,
-        analyzer="char_wb", ngram_range=(3, 5), min_df=3,
-        max_features=args.max_features // 3, dtype=np.float32,
-    )
-    Xtr_v = sparse.hstack([word.fit_transform(Xtr), char.fit_transform(Xtr)]).tocsr()
-    Xte_v = sparse.hstack([word.transform(Xte), char.transform(Xte)]).tocsr()
-    print(f"features: {Xtr_v.shape[1]}")
-
-    # SGD with modified_huber rather than liblinear logistic regression.
-    # 334 one-vs-rest fits over ~450k sparse features takes liblinear many
-    # minutes and it trips joblib's read-only memmap under parallel fan-out;
-    # SGD fits the same problem in seconds, scales to the ~1M-abstract corpus
-    # the harvest will produce, and modified_huber still exposes predict_proba,
-    # which the LLM fallback needs to decide when to escalate.
-    # log_loss, not modified_huber. modified_huber's predict_proba clips the
-    # margin and normalises, so it returns a saturated 1.0/0.0 with nothing in
-    # between — fine for ranking, useless for deciding when to escalate to the
-    # LLM. log_loss gives genuine logistic probabilities that a threshold can
-    # act on.
-    #
-    # class_weight is left unbalanced on purpose. Balancing inflates scores for
-    # rare labels, which is right for per-label F1 and wrong here: it pushed
-    # 'Rehabilitation' to the top of a cardiology abstract. The output feeds a
-    # ranked filter, so ordering matters more than per-label parity.
-    clf = OneVsRestClassifier(
-        SGDClassifier(
-            loss="log_loss", alpha=1e-6, max_iter=30, tol=1e-4,
-            random_state=42,
-        ),
-        n_jobs=args.n_jobs,
-    )
-    with joblib.parallel_config(backend="loky", max_nbytes=None):
-        clf.fit(Xtr_v, Ytr)
-
-    con = sqlite3.connect(args.db)
-    parent_of = {c: p for c, p in con.execute("SELECT code, parent_code FROM asjc")}
-    names = {c: (n or pn) for c, n, pn in
-             con.execute("SELECT code, name, parent_name FROM asjc")}
-    con.close()
-
-    res = evaluate(clf, Xte_v, yte_raw, mlb.classes_, parent_of)
-    print(f"\nheld-out performance (same distribution as training: {args.corpus})")
-    for k, v in res.items():
-        print(f"  {k:<20} {v:.3f}")
-
-    # The number that actually matters. When training on `titles`, the held-out
-    # score above is measured on 5-word journal titles while production input is
-    # a 200-word abstract — a different and much easier problem. This second
-    # evaluation runs on hand-labelled abstracts across 20 fields, which is the
-    # distribution the classifier is really deployed against.
-    if Path(args.eval_jsonl).exists():
-        ev = [json.loads(l) for l in open(args.eval_jsonl, encoding="utf-8")]
-        Xe = sparse.hstack([word.transform([e["text"] for e in ev]),
-                            char.transform([e["text"] for e in ev])]).tocsr()
-        Pe = clf.predict_proba(Xe)
-        order = np.argsort(-Pe, axis=1)
-        # Two standards, reported separately and deliberately.
-        #
-        # LENIENT counts a hit when *any* acceptable code appears. It overstates
-        # usefulness: for a seismic-engineering abstract, 'Engineering
-        # (miscellaneous)' is in `accept` and scores a hit, but filtering on it
-        # returns nonlinear-analysis maths journals, while the preferred code
-        # 2205 returns concrete-structures journals. Same "hit", different
-        # recommendation quality.
-        #
-        # STRICT requires the single best code — accept[0], the one a subject
-        # librarian would choose. This is the number to watch when retraining on
-        # the abstract corpus, because it is the one that tracks whether the
-        # recommendations are actually good.
-        n = len(ev)
-        hits1 = hits3 = parent1 = 0
-        s5 = s3 = 0
-        print(f"\nabstract eval ({n} hand-labelled abstracts)")
-        for i, e in enumerate(ev):
-            top5 = [mlb.classes_[j] for j in order[i, :5]]
-            top3, top1 = top5[:3], top5[0]
-            pref = e["accept"][0]
-            ok1 = top1 in e["accept"]
-            ok3 = bool(set(top3) & set(e["accept"]))
-            okp = parent_of.get(top1) in {parent_of.get(c) for c in e["accept"]}
-            hits1 += ok1; hits3 += ok3; parent1 += okp
-            s5 += pref in top5
-            s3 += pref in top3
-            flag = "OK  " if pref in top5 else ("weak" if ok3 else "MISS")
-            print(f"  {flag} {e['field'][:28]:<28} pref {pref} "
-                  f"{'in top5' if pref in top5 else 'ABSENT':<8} -> "
-                  f"{', '.join(f'{c}' for c in top3)}")
-        print(f"  lenient: top1 {hits1/n:.2f}  top3 {hits3/n:.2f}  parent {parent1/n:.2f}")
-        print(f"  STRICT : preferred-in-top3 {s3/n:.2f}   preferred-in-top5 {s5/n:.2f}")
-        print("  strict preferred-in-top5 is the number that gates shipping — it "
-              "bounds what the LLM stage can possibly pick.")
-        res.update({"abstract_top1": hits1 / n, "abstract_top3": hits3 / n,
-                    "abstract_top1_parent": parent1 / n,
-                    "abstract_preferred_top3": s3 / n,
-                    "abstract_preferred_top5": s5 / n})
-
-    # Compact the artifact before saving. The bulk is not the vocabulary — it is
-    # 334 dense float64 coefficient vectors (334 x ~160k x 8 bytes ≈ 427 MB
-    # before compression). Casting to float32 halves that with no measurable
-    # effect on ranking, and sparsify() drops the near-zero majority. This
-    # matters operationally: the Space pulls this file on every cold start.
-    for est in clf.estimators_:
-        est.coef_ = est.coef_.astype("float32")
-        est.sparsify()
-    joblib.dump(
-        {"word": word, "char": char, "clf": clf, "classes": mlb.classes_,
-         "names": names, "parent_of": parent_of, "corpus": args.corpus,
-         "metrics": res},
-        args.out, compress=3,
-    )
-    print(f"\nwrote {args.out}  ({Path(args.out).stat().st_size/1e6:.1f} MB)")
-
-    # Sparsified coefficients change the predict path, so re-check the eval
-    # rather than assuming the compaction was lossless.
-    chk = joblib.load(args.out)
-    Xc = sparse.hstack([chk["word"].transform([X[0]]),
-                        chk["char"].transform([X[0]])]).tocsr()
-    assert chk["clf"].predict_proba(Xc).shape[1] == len(mlb.classes_)
-    print("compacted model reloads and predicts correctly")
-
-
-if __name__ == "__main__":
-    main()
+"""
+Train the abstract -> ASJC multi-label classifier.
+
+    # interim model, no network needed — trains on journal titles in the index
+    python train_asjc.py --corpus titles --db journals.db --out asjc_model.joblib
+
+    # full model, after harvest_openalex.py has produced labelled abstracts
+    python train_asjc.py --corpus abstracts --train-jsonl asjc_train.jsonl \
+                         --db journals.db --out asjc_model.joblib
+
+Why TF-IDF + linear and not a transformer
+-----------------------------------------
+The task is topical multi-label text classification over 334 labels with a large
+vocabulary signal — exactly where a linear model over character and word n-grams
+is hard to beat per unit of cost. It trains in seconds on CPU, the artifact is
+tens of MB, and inference is sub-millisecond with no GPU and no model download at
+Space startup. A fine-tuned transformer might add a few points of F1 for ~400MB
+and a cold-start penalty on every Space restart. Take the linear model; spend the
+complexity budget on the LLM fallback for the cases it actually gets wrong.
+
+Two corpora, deliberately
+-------------------------
+`titles`    trains on journal titles (48,888 labelled examples, already in the
+            index, zero network). Ships immediately. But it trains on titles and
+            is applied to abstracts, and that domain gap is real — treat its
+            numbers as an upper bound on what it will do in production.
+
+`abstracts` trains on article abstracts labelled with their journal's ASJC codes
+            (weak supervision). Far better matched to inference, and the only
+            version whose held-out score means what it appears to mean.
+
+Label noise in the `abstracts` corpus
+-------------------------------------
+Giving every article its journal's codes is weak supervision: a physics paper in
+a multidisciplinary journal inherits 'Medicine' too. `--max-labels` drops journals
+carrying more than N ASJC codes from training, which removes the worst offenders
+(megajournals and general-science titles) at the cost of some coverage. Their
+articles are still *classifiable* at inference — they are just poor teachers.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import sqlite3
+import sys
+from pathlib import Path
+
+import numpy as np
+from scipy import sparse
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.linear_model import SGDClassifier
+from sklearn.model_selection import train_test_split
+from sklearn.multiclass import OneVsRestClassifier
+from sklearn.preprocessing import MultiLabelBinarizer
+
+import joblib
+
+
+# ---------------------------------------------------------------------------
+# Corpora
+# ---------------------------------------------------------------------------
+def load_titles(db: str, max_labels: int) -> tuple[list[str], list[list[str]]]:
+    """Journal title + subject names -> its ASJC codes."""
+    con = sqlite3.connect(db)
+    con.row_factory = sqlite3.Row
+    rows = con.execute(
+        """SELECT j.scopus_id, j.title, j.publisher_group,
+                  group_concat(ja.asjc) codes, count(*) n
+           FROM journals j JOIN journal_asjc ja ON ja.scopus_id = j.scopus_id
+           WHERE j.title IS NOT NULL
+           GROUP BY j.scopus_id"""
+    ).fetchall()
+    X, y = [], []
+    for r in rows:
+        if r["n"] > max_labels:
+            continue
+        X.append(r["title"])
+        y.append(r["codes"].split(","))
+    con.close()
+    return X, y
+
+
+def load_abstracts(path: str, db: str, max_labels: int) -> tuple[list[str], list[list[str]]]:
+    """
+    JSONL from harvest_openalex.py: {"text": ..., "scopus_id": ...}.
+    Labels come from the index, so the harvest never has to carry them.
+    """
+    con = sqlite3.connect(db)
+    codes: dict[str, list[str]] = {}
+    for sid, c in con.execute(
+        "SELECT scopus_id, group_concat(asjc) FROM journal_asjc GROUP BY scopus_id"
+    ):
+        codes[sid] = c.split(",")
+    con.close()
+
+    X, y = [], []
+    with open(path, encoding="utf-8") as f:
+        for line in f:
+            rec = json.loads(line)
+            c = codes.get(rec.get("scopus_id"))
+            if not c or len(c) > max_labels:
+                continue
+            t = (rec.get("text") or "").strip()
+            if len(t) < 120:          # a bare title is not an abstract
+                continue
+            X.append(t)
+            y.append(c)
+    return X, y
+
+
+# ---------------------------------------------------------------------------
+# Evaluation
+# ---------------------------------------------------------------------------
+def evaluate(clf, Xv, Y_true, classes: np.ndarray, parent_of: dict[str, str]) -> dict:
+    """
+    Ranking metrics, not threshold metrics.
+
+    The classifier feeds a *filter*, and the filter takes the top few codes — so
+    what matters is whether a correct code is ranked highly, not whether some
+    probability cleared 0.5. Reporting only micro-F1 at a fixed threshold would
+    flatter a model that is well-calibrated but badly ordered, and vice versa.
+
+    Parent accuracy (2-digit ASJC) is reported alongside because confusing
+    'Information Systems' with 'Computer Science Applications' barely harms the
+    candidate set, while confusing Computer Science with Dentistry destroys it.
+    """
+    P = clf.predict_proba(Xv)
+    order = np.argsort(-P, axis=1)
+    res: dict[str, float] = {}
+
+    for k in (1, 3, 5):
+        topk = order[:, :k]
+        hit = sum(
+            bool(set(classes[topk[i]]) & set(Y_true[i])) for i in range(len(Y_true))
+        )
+        res[f"top{k}_hit"] = hit / len(Y_true)
+
+    # Recall of the full true label set within the top 5 predictions.
+    top5 = order[:, :5]
+    res["top5_label_recall"] = float(np.mean([
+        len(set(classes[top5[i]]) & set(Y_true[i])) / len(Y_true[i])
+        for i in range(len(Y_true))
+    ]))
+
+    top1_parent = [parent_of.get(classes[order[i, 0]]) for i in range(len(Y_true))]
+    res["top1_parent_hit"] = float(np.mean([
+        top1_parent[i] in {parent_of.get(c) for c in Y_true[i]}
+        for i in range(len(Y_true))
+    ]))
+    return res
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--corpus", choices=["titles", "abstracts"], default="titles")
+    ap.add_argument("--db", default="journals.db")
+    ap.add_argument("--train-jsonl", default="asjc_train.jsonl")
+    ap.add_argument("--out", default="asjc_model.joblib")
+    ap.add_argument("--max-labels", type=int, default=6,
+                    help="drop journals carrying more ASJC codes than this from training")
+    ap.add_argument("--min-per-label", type=int, default=8,
+                    help="drop labels with fewer training examples than this")
+    ap.add_argument("--eval-jsonl", default="asjc_eval.jsonl")
+    ap.add_argument("--max-features", type=int, default=120_000)
+    ap.add_argument("--n-jobs", type=int, default=2,
+                    help="parallel per-label fits. 334 estimators over a large sparse "
+                         "matrix can exhaust memory at -1; 4 is safe on a 16GB Space.")
+    args = ap.parse_args()
+
+    if args.corpus == "titles":
+        X, y = load_titles(args.db, args.max_labels)
+    else:
+        if not Path(args.train_jsonl).exists():
+            sys.exit(f"{args.train_jsonl} not found — run harvest_openalex.py first")
+        X, y = load_abstracts(args.train_jsonl, args.db, args.max_labels)
+
+    print(f"corpus={args.corpus}  examples={len(X)}  "
+          f"mean labels/example={np.mean([len(c) for c in y]):.2f}")
+
+    # Drop labels too rare to learn. They stay resolvable through the LLM
+    # fallback, which does not need training examples.
+    from collections import Counter
+    freq = Counter(c for cs in y for c in cs)
+    keep = {c for c, n in freq.items() if n >= args.min_per_label}
+    y = [[c for c in cs if c in keep] for cs in y]
+    mask = [i for i, cs in enumerate(y) if cs]
+    X, y = [X[i] for i in mask], [y[i] for i in mask]
+    print(f"labels kept: {len(keep)} of {len(freq)}   examples after filtering: {len(X)}")
+
+    mlb = MultiLabelBinarizer()
+    Y = mlb.fit_transform(y)
+
+    Xtr, Xte, Ytr, Yte, ytr_raw, yte_raw = train_test_split(
+        X, Y, y, test_size=0.2, random_state=42
+    )
+
+    # Word n-grams carry the topical signal; character n-grams cover
+    # morphology, non-English titles and the domain shift between titles and
+    # abstracts. Union of both, sublinear tf to damp long abstracts.
+    word = TfidfVectorizer(
+        sublinear_tf=True, strip_accents="unicode", lowercase=True,
+        ngram_range=(1, 2), min_df=2, max_features=args.max_features,
+        stop_words="english", dtype=np.float32,
+    )
+    char = TfidfVectorizer(
+        sublinear_tf=True, strip_accents="unicode", lowercase=True,
+        analyzer="char_wb", ngram_range=(3, 5), min_df=3,
+        max_features=args.max_features // 3, dtype=np.float32,
+    )
+    Xtr_v = sparse.hstack([word.fit_transform(Xtr), char.fit_transform(Xtr)]).tocsr()
+    Xte_v = sparse.hstack([word.transform(Xte), char.transform(Xte)]).tocsr()
+    print(f"features: {Xtr_v.shape[1]}")
+
+    # SGD with modified_huber rather than liblinear logistic regression.
+    # 334 one-vs-rest fits over ~450k sparse features takes liblinear many
+    # minutes and it trips joblib's read-only memmap under parallel fan-out;
+    # SGD fits the same problem in seconds, scales to the ~1M-abstract corpus
+    # the harvest will produce, and modified_huber still exposes predict_proba,
+    # which the LLM fallback needs to decide when to escalate.
+    # log_loss, not modified_huber. modified_huber's predict_proba clips the
+    # margin and normalises, so it returns a saturated 1.0/0.0 with nothing in
+    # between — fine for ranking, useless for deciding when to escalate to the
+    # LLM. log_loss gives genuine logistic probabilities that a threshold can
+    # act on.
+    #
+    # class_weight is left unbalanced on purpose. Balancing inflates scores for
+    # rare labels, which is right for per-label F1 and wrong here: it pushed
+    # 'Rehabilitation' to the top of a cardiology abstract. The output feeds a
+    # ranked filter, so ordering matters more than per-label parity.
+    clf = OneVsRestClassifier(
+        SGDClassifier(
+            loss="log_loss", alpha=1e-6, max_iter=30, tol=1e-4,
+            random_state=42,
+        ),
+        n_jobs=args.n_jobs,
+    )
+    with joblib.parallel_config(backend="loky", max_nbytes=None):
+        clf.fit(Xtr_v, Ytr)
+
+    con = sqlite3.connect(args.db)
+    parent_of = {c: p for c, p in con.execute("SELECT code, parent_code FROM asjc")}
+    names = {c: (n or pn) for c, n, pn in
+             con.execute("SELECT code, name, parent_name FROM asjc")}
+    con.close()
+
+    res = evaluate(clf, Xte_v, yte_raw, mlb.classes_, parent_of)
+    print(f"\nheld-out performance (same distribution as training: {args.corpus})")
+    for k, v in res.items():
+        print(f"  {k:<20} {v:.3f}")
+
+    # The number that actually matters. When training on `titles`, the held-out
+    # score above is measured on 5-word journal titles while production input is
+    # a 200-word abstract — a different and much easier problem. This second
+    # evaluation runs on hand-labelled abstracts across 20 fields, which is the
+    # distribution the classifier is really deployed against.
+    if Path(args.eval_jsonl).exists():
+        ev = [json.loads(l) for l in open(args.eval_jsonl, encoding="utf-8")]
+        Xe = sparse.hstack([word.transform([e["text"] for e in ev]),
+                            char.transform([e["text"] for e in ev])]).tocsr()
+        Pe = clf.predict_proba(Xe)
+        order = np.argsort(-Pe, axis=1)
+        # Two standards, reported separately and deliberately.
+        #
+        # LENIENT counts a hit when *any* acceptable code appears. It overstates
+        # usefulness: for a seismic-engineering abstract, 'Engineering
+        # (miscellaneous)' is in `accept` and scores a hit, but filtering on it
+        # returns nonlinear-analysis maths journals, while the preferred code
+        # 2205 returns concrete-structures journals. Same "hit", different
+        # recommendation quality.
+        #
+        # STRICT requires the single best code — accept[0], the one a subject
+        # librarian would choose. This is the number to watch when retraining on
+        # the abstract corpus, because it is the one that tracks whether the
+        # recommendations are actually good.
+        n = len(ev)
+        hits1 = hits3 = parent1 = 0
+        s5 = s3 = 0
+        print(f"\nabstract eval ({n} hand-labelled abstracts)")
+        for i, e in enumerate(ev):
+            top5 = [mlb.classes_[j] for j in order[i, :5]]
+            top3, top1 = top5[:3], top5[0]
+            pref = e["accept"][0]
+            ok1 = top1 in e["accept"]
+            ok3 = bool(set(top3) & set(e["accept"]))
+            okp = parent_of.get(top1) in {parent_of.get(c) for c in e["accept"]}
+            hits1 += ok1; hits3 += ok3; parent1 += okp
+            s5 += pref in top5
+            s3 += pref in top3
+            flag = "OK  " if pref in top5 else ("weak" if ok3 else "MISS")
+            print(f"  {flag} {e['field'][:28]:<28} pref {pref} "
+                  f"{'in top5' if pref in top5 else 'ABSENT':<8} -> "
+                  f"{', '.join(f'{c}' for c in top3)}")
+        print(f"  lenient: top1 {hits1/n:.2f}  top3 {hits3/n:.2f}  parent {parent1/n:.2f}")
+        print(f"  STRICT : preferred-in-top3 {s3/n:.2f}   preferred-in-top5 {s5/n:.2f}")
+        print("  strict preferred-in-top5 is the number that gates shipping — it "
+              "bounds what the LLM stage can possibly pick.")
+        res.update({"abstract_top1": hits1 / n, "abstract_top3": hits3 / n,
+                    "abstract_top1_parent": parent1 / n,
+                    "abstract_preferred_top3": s3 / n,
+                    "abstract_preferred_top5": s5 / n})
+
+    # Compact the artifact before saving. The bulk is not the vocabulary — it is
+    # 334 dense float64 coefficient vectors (334 x ~160k x 8 bytes ≈ 427 MB
+    # before compression). Casting to float32 halves that with no measurable
+    # effect on ranking, and sparsify() drops the near-zero majority. This
+    # matters operationally: the Space pulls this file on every cold start.
+    for est in clf.estimators_:
+        est.coef_ = est.coef_.astype("float32")
+        est.sparsify()
+    joblib.dump(
+        {"word": word, "char": char, "clf": clf, "classes": mlb.classes_,
+         "names": names, "parent_of": parent_of, "corpus": args.corpus,
+         "metrics": res},
+        args.out, compress=3,
+    )
+    print(f"\nwrote {args.out}  ({Path(args.out).stat().st_size/1e6:.1f} MB)")
+
+    # Sparsified coefficients change the predict path, so re-check the eval
+    # rather than assuming the compaction was lossless.
+    chk = joblib.load(args.out)
+    Xc = sparse.hstack([chk["word"].transform([X[0]]),
+                        chk["char"].transform([X[0]])]).tocsr()
+    assert chk["clf"].predict_proba(Xc).shape[1] == len(mlb.classes_)
+    print("compacted model reloads and predicts correctly")
+
+
+if __name__ == "__main__":
+    main()

usage.py

@@ -1,355 +1,355 @@
-"""
-usage.py — ResearchBee usage counting
-
-Counts only. There is no query-text path through this module and no
-column to put one in: the manuscript tab receives unpublished abstracts,
-and the cheapest guarantee that those are never persisted is to have
-nowhere to persist them.
-
-Design notes
-------------
-1.  Counting happens in memory and costs a dict update. Nothing touches
-    the network on the request path.
-
-2.  Deltas are flushed to the Cloudflare Worker every FLUSH_SECONDS.
-    A failed flush is NOT lost: the counts are folded back into the
-    pending dict and go out with the next attempt. Because we send
-    accumulated deltas rather than one row per request, an outage costs
-    a delay in the dashboard, not data. This is the main reason the
-    aggregate design is worth having — LibBee's per-row logging had to
-    grow a 200-item retry buffer to get the same property, and still
-    drops rows past that bound.
-
-3.  HF Spaces egress to workers.dev is known to fail intermittently
-    (LibBee v3.9.2 moved its logging into the browser for exactly this
-    reason). Retry-with-accumulation handles that without needing a
-    client-side path, so nothing has to change in the LibGuides fragment.
-
-4.  Counters do NOT survive a container restart if a flush has not
-    happened yet — at most FLUSH_SECONDS of activity is lost on a hard
-    restart. The 6-hour canary keeps the Space warm, so restarts should
-    be rare.
-"""
-
-from __future__ import annotations
-
-import asyncio
-import logging
-import os
-import time
-from collections import deque
-from datetime import datetime
-from typing import Dict, List, Tuple
-
-logger = logging.getLogger(__name__)
-
-WORKER_URL = (os.environ.get("CLOUDFLARE_WORKER_URL") or "").strip().strip('"').strip("'").rstrip("/")
-WORKER_TOKEN = (os.environ.get("CLOUDFLARE_WORKER_TOKEN") or "").strip()
-
-FLUSH_SECONDS = 60
-MAX_ATTEMPTS_LOG = 15
-
-# ── Pull mode ─────────────────────────────────────────────────────────
-# This Space cannot open a connection to workers.dev: three probes failed
-# identically at 15s with ConnectError, while OpenAI, OpenAlex and
-# huggingface.co all work from the same container. Cloudflare throttles
-# shared workers.dev addresses from datacentre IPs. LibBee hit this too,
-# and forcing IPv4 made it worse, so that route is closed.
-#
-# Inbound still works — Cloudflare reaches this Space fine, which is how
-# the canary runs. So the Worker PULLS counts on its 6-hour cron instead
-# of this process pushing them. Pushing is left available but off by
-# default: set USAGE_PUSH_ENABLED=true if egress is ever restored.
-PUSH_ENABLED = (os.environ.get("USAGE_PUSH_ENABLED") or "").lower() in ("1", "true", "yes")
-
-# Two-phase handover. A snapshot moves out of _pending into _in_flight and
-# is only discarded once the Worker confirms it reached D1. If the Worker
-# dies between reading and acknowledging, the snapshot is folded back in
-# rather than lost.
-IN_FLIGHT_TIMEOUT = 900  # 15 minutes
-
-# Endpoint → surface. Adding an endpoint here is the only step needed to
-# start counting it; the Worker validates the name and rejects unknowns,
-# so a typo surfaces as a visible 400 rather than a phantom column.
-SURFACE_BY_PATH: Dict[str, str] = {
-    "/api/browse-subject":  "journal_subject",
-    "/api/analyze-journal": "journal_manuscript",
-    "/api/check-license":   "licence_deposit",
-    "/api/find-repository": "data_repository",
-    "/api/cover-letter":    "cover_letter",
-    "/api/chat":            "chat",
-}
-
-# Key: (surface, outcome, gap_code)
-_Key = Tuple[str, str, str]
-
-
-class UsageCounters:
-    def __init__(self) -> None:
-        self._pending: Dict[_Key, int] = {}
-        self._lifetime: Dict[_Key, int] = {}
-        self._lock = asyncio.Lock()
-        self._attempts: deque = deque(maxlen=MAX_ATTEMPTS_LOG)
-        self._task: asyncio.Task | None = None
-        self._started_at = datetime.utcnow()
-        self._in_flight: Dict[str, tuple] = {}   # id -> (unix_ts, {key: n})
-
-    # ── Recording (sync, no I/O) ──────────────────────────────────────
-
-    def record(self, surface: str, outcome: str = "ok", gap_code: str = "") -> None:
-        """Count one event. Never raises — instrumentation must not be
-        able to fail a user request."""
-        try:
-            if surface not in SURFACE_BY_PATH.values():
-                return
-            if outcome not in ("ok", "miss", "error"):
-                outcome = "error"
-            key = (surface, outcome, (gap_code or "")[:40])
-            self._pending[key] = self._pending.get(key, 0) + 1
-            self._lifetime[key] = self._lifetime.get(key, 0) + 1
-        except Exception:
-            pass
-
-    def record_gap(self, surface: str, gap_code: str) -> None:
-        """A request that completed but could not be answered from the
-        available sources — a SCImago subject with no matches, an
-        OA.Works 404, a DOI that resolves to nothing. Recorded as its own
-        outcome so 'served a page' and 'answered the question' stay
-        distinguishable in the dashboard."""
-        self.record(surface, outcome="miss", gap_code=gap_code)
-
-    # ── Local snapshot (works with no Worker configured) ──────────────
-
-    def snapshot(self) -> dict:
-        by_surface: Dict[str, Dict[str, int]] = {}
-        gaps: Dict[str, int] = {}
-        for (surface, outcome, gap), n in self._lifetime.items():
-            s = by_surface.setdefault(
-                surface, {"total": 0, "ok": 0, "miss": 0, "error": 0}
-            )
-            s["total"] += n
-            s[outcome] = s.get(outcome, 0) + n
-            if gap:
-                gaps[gap] = gaps.get(gap, 0) + n
-        return {
-            "since": self._started_at.strftime("%Y-%m-%d %H:%M:%S") + " UTC",
-            "note": "Counts since this container last started. "
-                    "Durable totals come from the Worker.",
-            "by_surface": by_surface,
-            "gaps": dict(sorted(gaps.items(), key=lambda kv: -kv[1])),
-            "pending_deltas": len(self._pending),
-            "in_flight_snapshots": len(self._in_flight),
-            "mode": "push" if PUSH_ENABLED else "pull (Worker fetches on its cron)",
-            "worker_configured": bool(WORKER_URL),
-            "recent_flushes": list(self._attempts)[::-1],
-        }
-
-    # ── Pull handover ─────────────────────────────────────────────────
-
-    def export_snapshot(self) -> dict:
-        """Hand the accumulated deltas to a caller that will store them.
-
-        The counts move to _in_flight rather than being deleted, so a
-        caller that reads them and then fails does not silently destroy
-        them. Call ack_snapshot() once they are safely written.
-        """
-        self._requeue_stale()
-        if not self._pending:
-            return {"snapshot_id": None, "deltas": [], "pending": 0}
-
-        import uuid
-        snapshot_id = uuid.uuid4().hex[:16]
-        batch = self._pending
-        self._pending = {}
-        self._in_flight[snapshot_id] = (time.time(), batch)
-
-        return {
-            "snapshot_id": snapshot_id,
-            "deltas": [
-                {"surface": s, "outcome": o, "gap_code": g, "n": n}
-                for (s, o, g), n in batch.items()
-            ],
-            "pending": 0,
-            "in_flight": len(self._in_flight),
-        }
-
-    def ack_snapshot(self, snapshot_id: str) -> bool:
-        """Confirm a snapshot was stored. Only now is it discarded."""
-        existed = snapshot_id in self._in_flight
-        self._in_flight.pop(snapshot_id, None)
-        if existed:
-            self._record_attempt("pulled", f"snapshot {snapshot_id} acknowledged")
-        return existed
-
-    def _requeue_stale(self) -> None:
-        """Fold back snapshots nobody acknowledged. The cost of a caller
-        crashing mid-handover is a delay and a possible double count on
-        the same hour bucket, never a loss."""
-        now = time.time()
-        for sid in [k for k, (ts, _) in self._in_flight.items()
-                    if now - ts > IN_FLIGHT_TIMEOUT]:
-            _ts, batch = self._in_flight.pop(sid)
-            self._requeue(batch)
-            self._record_attempt("requeued", f"snapshot {sid} was never acknowledged")
-
-    # ── Flushing ──────────────────────────────────────────────────────
-
-    def _record_attempt(self, outcome: str, detail: str = "") -> None:
-        self._attempts.append({
-            "at": datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"),
-            "outcome": outcome,
-            "detail": detail[:200],
-        })
-
-    async def flush(self) -> bool:
-        """Send accumulated deltas. On failure the deltas are folded back
-        in and retried on the next pass, so nothing is dropped."""
-        if not WORKER_URL or not self._pending:
-            return True
-
-        async with self._lock:
-            batch = self._pending
-            self._pending = {}
-
-        deltas: List[dict] = [
-            {"surface": s, "outcome": o, "gap_code": g, "n": n}
-            for (s, o, g), n in batch.items()
-        ]
-
-        try:
-            import httpx
-            headers = {"Authorization": f"Bearer {WORKER_TOKEN}"} if WORKER_TOKEN else {}
-            async with httpx.AsyncClient(timeout=10) as client:
-                r = await client.post(
-                    f"{WORKER_URL}/rb/log",
-                    json={"deltas": deltas},
-                    headers=headers,
-                )
-            body = {}
-            try:
-                body = r.json()
-            except Exception:
-                pass
-
-            # The Worker answers 200 with logged=false when the D1 write
-            # itself failed, so the status code alone cannot tell a stored
-            # batch from a dropped one. Check the verdict, not the code.
-            if r.status_code < 400 and body.get("logged"):
-                self._record_attempt(
-                    "ok", f"{body.get('accepted', len(deltas))} delta(s) stored"
-                )
-                return True
-
-            self._requeue(batch)
-            if r.status_code == 401:
-                self._record_attempt(
-                    "rejected_401",
-                    "CLOUDFLARE_WORKER_TOKEN does not match the Worker's ANALYTICS_TOKEN",
-                )
-            else:
-                self._record_attempt(
-                    "refused",
-                    f"HTTP {r.status_code} {body.get('warning', '')}",
-                )
-            return False
-
-        except Exception as exc:
-            self._requeue(batch)
-            self._record_attempt("unreachable", f"{type(exc).__name__}: {exc}")
-            return False
-
-    def _requeue(self, batch: Dict[_Key, int]) -> None:
-        for key, n in batch.items():
-            self._pending[key] = self._pending.get(key, 0) + n
-
-    async def _loop(self) -> None:
-        while True:
-            await asyncio.sleep(FLUSH_SECONDS)
-            try:
-                await self.flush()
-            except Exception as exc:
-                logger.warning("usage flush loop error: %s", exc)
-
-    def start(self) -> None:
-        """Start the background flusher, if pushing is enabled at all."""
-        if not PUSH_ENABLED:
-            logger.info("Usage counting active in PULL mode — the Worker "
-                        "collects counts on its cron; no outbound calls made.")
-            return
-        if self._task is not None:
-            return
-        try:
-            loop = asyncio.get_running_loop()
-        except RuntimeError:
-            return
-        # A strong reference is held on the instance: asyncio keeps only
-        # weak references to tasks, and a dropped task can be collected
-        # mid-flight, silently cancelling the flush.
-        self._task = loop.create_task(self._loop())
-        logger.info("Usage flusher started (%ss interval, worker=%s)",
-                    FLUSH_SECONDS, bool(WORKER_URL))
-
-    async def stop(self) -> None:
-        """Final flush on shutdown so the last minute of activity lands.
-        In pull mode there is nowhere to flush to, so this is a no-op and
-        up to 6 hours of counts can be lost on an unlucky restart — the
-        dashboard's "Send counts to Cloudflare now" button exists to make
-        that recoverable before a planned redeploy."""
-        if self._task:
-            self._task.cancel()
-            self._task = None
-        if PUSH_ENABLED:
-            await self.flush()
-
-
-usage = UsageCounters()
-
-
-# ── FastAPI middleware ────────────────────────────────────────────────
-
-def install(app) -> None:
-    """Wire counting into the app. No frontend change is required: the
-    tab a person used is implied by the endpoint their browser called,
-    which is why this survives the move to LibGuides and any future
-    frontend rewrite."""
-    from starlette.requests import Request
-
-    @app.middleware("http")
-    async def _count(request: Request, call_next):
-        surface = SURFACE_BY_PATH.get(request.url.path)
-        # Canary traffic exists to keep the Space awake and must never
-        # appear in usage figures — the same exclusion LibBee applies.
-        counted = surface and request.headers.get("X-ResearchBee-Canary") != "1"
-
-        try:
-            response = await call_next(request)
-        except Exception:
-            # An unhandled exception propagates out of call_next, so
-            # anything written after the await would never run. Without
-            # this branch the most important failures — the ones that
-            # crash a handler — would be the only ones missing from the
-            # dashboard.
-            if counted:
-                usage.record(surface, "error", "unhandled_exception")
-            raise
-
-        try:
-            if counted:
-                if response.status_code < 400:
-                    # A handler that recorded its own miss has already
-                    # counted this request; the flag stops it counting twice.
-                    if not getattr(request.state, "usage_recorded", False):
-                        usage.record(surface, "ok")
-                else:
-                    usage.record(surface, "error")
-        except Exception:
-            pass
-        return response
-
-    @app.on_event("startup")
-    async def _start_usage():
-        usage.start()
-
-    @app.on_event("shutdown")
-    async def _stop_usage():
+"""
+usage.py — ResearchBee usage counting
+
+Counts only. There is no query-text path through this module and no
+column to put one in: the manuscript tab receives unpublished abstracts,
+and the cheapest guarantee that those are never persisted is to have
+nowhere to persist them.
+
+Design notes
+------------
+1.  Counting happens in memory and costs a dict update. Nothing touches
+    the network on the request path.
+
+2.  Deltas are flushed to the Cloudflare Worker every FLUSH_SECONDS.
+    A failed flush is NOT lost: the counts are folded back into the
+    pending dict and go out with the next attempt. Because we send
+    accumulated deltas rather than one row per request, an outage costs
+    a delay in the dashboard, not data. This is the main reason the
+    aggregate design is worth having — LibBee's per-row logging had to
+    grow a 200-item retry buffer to get the same property, and still
+    drops rows past that bound.
+
+3.  HF Spaces egress to workers.dev is known to fail intermittently
+    (LibBee v3.9.2 moved its logging into the browser for exactly this
+    reason). Retry-with-accumulation handles that without needing a
+    client-side path, so nothing has to change in the LibGuides fragment.
+
+4.  Counters do NOT survive a container restart if a flush has not
+    happened yet — at most FLUSH_SECONDS of activity is lost on a hard
+    restart. The 6-hour canary keeps the Space warm, so restarts should
+    be rare.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import os
+import time
+from collections import deque
+from datetime import datetime
+from typing import Dict, List, Tuple
+
+logger = logging.getLogger(__name__)
+
+WORKER_URL = (os.environ.get("CLOUDFLARE_WORKER_URL") or "").strip().strip('"').strip("'").rstrip("/")
+WORKER_TOKEN = (os.environ.get("CLOUDFLARE_WORKER_TOKEN") or "").strip()
+
+FLUSH_SECONDS = 60
+MAX_ATTEMPTS_LOG = 15
+
+# ── Pull mode ─────────────────────────────────────────────────────────
+# This Space cannot open a connection to workers.dev: three probes failed
+# identically at 15s with ConnectError, while OpenAI, OpenAlex and
+# huggingface.co all work from the same container. Cloudflare throttles
+# shared workers.dev addresses from datacentre IPs. LibBee hit this too,
+# and forcing IPv4 made it worse, so that route is closed.
+#
+# Inbound still works — Cloudflare reaches this Space fine, which is how
+# the canary runs. So the Worker PULLS counts on its 6-hour cron instead
+# of this process pushing them. Pushing is left available but off by
+# default: set USAGE_PUSH_ENABLED=true if egress is ever restored.
+PUSH_ENABLED = (os.environ.get("USAGE_PUSH_ENABLED") or "").lower() in ("1", "true", "yes")
+
+# Two-phase handover. A snapshot moves out of _pending into _in_flight and
+# is only discarded once the Worker confirms it reached D1. If the Worker
+# dies between reading and acknowledging, the snapshot is folded back in
+# rather than lost.
+IN_FLIGHT_TIMEOUT = 900  # 15 minutes
+
+# Endpoint → surface. Adding an endpoint here is the only step needed to
+# start counting it; the Worker validates the name and rejects unknowns,
+# so a typo surfaces as a visible 400 rather than a phantom column.
+SURFACE_BY_PATH: Dict[str, str] = {
+    "/api/browse-subject":  "journal_subject",
+    "/api/analyze-journal": "journal_manuscript",
+    "/api/check-license":   "licence_deposit",
+    "/api/find-repository": "data_repository",
+    "/api/cover-letter":    "cover_letter",
+    "/api/chat":            "chat",
+}
+
+# Key: (surface, outcome, gap_code)
+_Key = Tuple[str, str, str]
+
+
+class UsageCounters:
+    def __init__(self) -> None:
+        self._pending: Dict[_Key, int] = {}
+        self._lifetime: Dict[_Key, int] = {}
+        self._lock = asyncio.Lock()
+        self._attempts: deque = deque(maxlen=MAX_ATTEMPTS_LOG)
+        self._task: asyncio.Task | None = None
+        self._started_at = datetime.utcnow()
+        self._in_flight: Dict[str, tuple] = {}   # id -> (unix_ts, {key: n})
+
+    # ── Recording (sync, no I/O) ──────────────────────────────────────
+
+    def record(self, surface: str, outcome: str = "ok", gap_code: str = "") -> None:
+        """Count one event. Never raises — instrumentation must not be
+        able to fail a user request."""
+        try:
+            if surface not in SURFACE_BY_PATH.values():
+                return
+            if outcome not in ("ok", "miss", "error"):
+                outcome = "error"
+            key = (surface, outcome, (gap_code or "")[:40])
+            self._pending[key] = self._pending.get(key, 0) + 1
+            self._lifetime[key] = self._lifetime.get(key, 0) + 1
+        except Exception:
+            pass
+
+    def record_gap(self, surface: str, gap_code: str) -> None:
+        """A request that completed but could not be answered from the
+        available sources — a SCImago subject with no matches, an
+        OA.Works 404, a DOI that resolves to nothing. Recorded as its own
+        outcome so 'served a page' and 'answered the question' stay
+        distinguishable in the dashboard."""
+        self.record(surface, outcome="miss", gap_code=gap_code)
+
+    # ── Local snapshot (works with no Worker configured) ──────────────
+
+    def snapshot(self) -> dict:
+        by_surface: Dict[str, Dict[str, int]] = {}
+        gaps: Dict[str, int] = {}
+        for (surface, outcome, gap), n in self._lifetime.items():
+            s = by_surface.setdefault(
+                surface, {"total": 0, "ok": 0, "miss": 0, "error": 0}
+            )
+            s["total"] += n
+            s[outcome] = s.get(outcome, 0) + n
+            if gap:
+                gaps[gap] = gaps.get(gap, 0) + n
+        return {
+            "since": self._started_at.strftime("%Y-%m-%d %H:%M:%S") + " UTC",
+            "note": "Counts since this container last started. "
+                    "Durable totals come from the Worker.",
+            "by_surface": by_surface,
+            "gaps": dict(sorted(gaps.items(), key=lambda kv: -kv[1])),
+            "pending_deltas": len(self._pending),
+            "in_flight_snapshots": len(self._in_flight),
+            "mode": "push" if PUSH_ENABLED else "pull (Worker fetches on its cron)",
+            "worker_configured": bool(WORKER_URL),
+            "recent_flushes": list(self._attempts)[::-1],
+        }
+
+    # ── Pull handover ─────────────────────────────────────────────────
+
+    def export_snapshot(self) -> dict:
+        """Hand the accumulated deltas to a caller that will store them.
+
+        The counts move to _in_flight rather than being deleted, so a
+        caller that reads them and then fails does not silently destroy
+        them. Call ack_snapshot() once they are safely written.
+        """
+        self._requeue_stale()
+        if not self._pending:
+            return {"snapshot_id": None, "deltas": [], "pending": 0}
+
+        import uuid
+        snapshot_id = uuid.uuid4().hex[:16]
+        batch = self._pending
+        self._pending = {}
+        self._in_flight[snapshot_id] = (time.time(), batch)
+
+        return {
+            "snapshot_id": snapshot_id,
+            "deltas": [
+                {"surface": s, "outcome": o, "gap_code": g, "n": n}
+                for (s, o, g), n in batch.items()
+            ],
+            "pending": 0,
+            "in_flight": len(self._in_flight),
+        }
+
+    def ack_snapshot(self, snapshot_id: str) -> bool:
+        """Confirm a snapshot was stored. Only now is it discarded."""
+        existed = snapshot_id in self._in_flight
+        self._in_flight.pop(snapshot_id, None)
+        if existed:
+            self._record_attempt("pulled", f"snapshot {snapshot_id} acknowledged")
+        return existed
+
+    def _requeue_stale(self) -> None:
+        """Fold back snapshots nobody acknowledged. The cost of a caller
+        crashing mid-handover is a delay and a possible double count on
+        the same hour bucket, never a loss."""
+        now = time.time()
+        for sid in [k for k, (ts, _) in self._in_flight.items()
+                    if now - ts > IN_FLIGHT_TIMEOUT]:
+            _ts, batch = self._in_flight.pop(sid)
+            self._requeue(batch)
+            self._record_attempt("requeued", f"snapshot {sid} was never acknowledged")
+
+    # ── Flushing ──────────────────────────────────────────────────────
+
+    def _record_attempt(self, outcome: str, detail: str = "") -> None:
+        self._attempts.append({
+            "at": datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"),
+            "outcome": outcome,
+            "detail": detail[:200],
+        })
+
+    async def flush(self) -> bool:
+        """Send accumulated deltas. On failure the deltas are folded back
+        in and retried on the next pass, so nothing is dropped."""
+        if not WORKER_URL or not self._pending:
+            return True
+
+        async with self._lock:
+            batch = self._pending
+            self._pending = {}
+
+        deltas: List[dict] = [
+            {"surface": s, "outcome": o, "gap_code": g, "n": n}
+            for (s, o, g), n in batch.items()
+        ]
+
+        try:
+            import httpx
+            headers = {"Authorization": f"Bearer {WORKER_TOKEN}"} if WORKER_TOKEN else {}
+            async with httpx.AsyncClient(timeout=10) as client:
+                r = await client.post(
+                    f"{WORKER_URL}/rb/log",
+                    json={"deltas": deltas},
+                    headers=headers,
+                )
+            body = {}
+            try:
+                body = r.json()
+            except Exception:
+                pass
+
+            # The Worker answers 200 with logged=false when the D1 write
+            # itself failed, so the status code alone cannot tell a stored
+            # batch from a dropped one. Check the verdict, not the code.
+            if r.status_code < 400 and body.get("logged"):
+                self._record_attempt(
+                    "ok", f"{body.get('accepted', len(deltas))} delta(s) stored"
+                )
+                return True
+
+            self._requeue(batch)
+            if r.status_code == 401:
+                self._record_attempt(
+                    "rejected_401",
+                    "CLOUDFLARE_WORKER_TOKEN does not match the Worker's ANALYTICS_TOKEN",
+                )
+            else:
+                self._record_attempt(
+                    "refused",
+                    f"HTTP {r.status_code} {body.get('warning', '')}",
+                )
+            return False
+
+        except Exception as exc:
+            self._requeue(batch)
+            self._record_attempt("unreachable", f"{type(exc).__name__}: {exc}")
+            return False
+
+    def _requeue(self, batch: Dict[_Key, int]) -> None:
+        for key, n in batch.items():
+            self._pending[key] = self._pending.get(key, 0) + n
+
+    async def _loop(self) -> None:
+        while True:
+            await asyncio.sleep(FLUSH_SECONDS)
+            try:
+                await self.flush()
+            except Exception as exc:
+                logger.warning("usage flush loop error: %s", exc)
+
+    def start(self) -> None:
+        """Start the background flusher, if pushing is enabled at all."""
+        if not PUSH_ENABLED:
+            logger.info("Usage counting active in PULL mode — the Worker "
+                        "collects counts on its cron; no outbound calls made.")
+            return
+        if self._task is not None:
+            return
+        try:
+            loop = asyncio.get_running_loop()
+        except RuntimeError:
+            return
+        # A strong reference is held on the instance: asyncio keeps only
+        # weak references to tasks, and a dropped task can be collected
+        # mid-flight, silently cancelling the flush.
+        self._task = loop.create_task(self._loop())
+        logger.info("Usage flusher started (%ss interval, worker=%s)",
+                    FLUSH_SECONDS, bool(WORKER_URL))
+
+    async def stop(self) -> None:
+        """Final flush on shutdown so the last minute of activity lands.
+        In pull mode there is nowhere to flush to, so this is a no-op and
+        up to 6 hours of counts can be lost on an unlucky restart — the
+        dashboard's "Send counts to Cloudflare now" button exists to make
+        that recoverable before a planned redeploy."""
+        if self._task:
+            self._task.cancel()
+            self._task = None
+        if PUSH_ENABLED:
+            await self.flush()
+
+
+usage = UsageCounters()
+
+
+# ── FastAPI middleware ────────────────────────────────────────────────
+
+def install(app) -> None:
+    """Wire counting into the app. No frontend change is required: the
+    tab a person used is implied by the endpoint their browser called,
+    which is why this survives the move to LibGuides and any future
+    frontend rewrite."""
+    from starlette.requests import Request
+
+    @app.middleware("http")
+    async def _count(request: Request, call_next):
+        surface = SURFACE_BY_PATH.get(request.url.path)
+        # Canary traffic exists to keep the Space awake and must never
+        # appear in usage figures — the same exclusion LibBee applies.
+        counted = surface and request.headers.get("X-ResearchBee-Canary") != "1"
+
+        try:
+            response = await call_next(request)
+        except Exception:
+            # An unhandled exception propagates out of call_next, so
+            # anything written after the await would never run. Without
+            # this branch the most important failures — the ones that
+            # crash a handler — would be the only ones missing from the
+            # dashboard.
+            if counted:
+                usage.record(surface, "error", "unhandled_exception")
+            raise
+
+        try:
+            if counted:
+                if response.status_code < 400:
+                    # A handler that recorded its own miss has already
+                    # counted this request; the flag stops it counting twice.
+                    if not getattr(request.state, "usage_recorded", False):
+                        usage.record(surface, "ok")
+                else:
+                    usage.record(surface, "error")
+        except Exception:
+            pass
+        return response
+
+    @app.on_event("startup")
+    async def _start_usage():
+        usage.start()
+
+    @app.on_event("shutdown")
+    async def _stop_usage():
         await usage.stop()

utils.py

@@ -1,13 +1,13 @@
-"""Shared utilities for ResearchBee backend."""
-from typing import Optional
-
-
-def norm_issn(raw: str | None) -> Optional[str]:
-    """
-    Normalise any ISSN-like string to XXXX-XXXX format.
-    Returns None if input is absent or not 8 digits after stripping.
-    """
-    if not raw:
-        return None
-    digits = str(raw).strip().upper().replace(" ", "").replace("-", "")
-    return f"{digits[:4]}-{digits[4:]}" if len(digits) == 8 else None
+"""Shared utilities for ResearchBee backend."""
+from typing import Optional
+
+
+def norm_issn(raw: str | None) -> Optional[str]:
+    """
+    Normalise any ISSN-like string to XXXX-XXXX format.
+    Returns None if input is absent or not 8 digits after stripping.
+    """
+    if not raw:
+        return None
+    digits = str(raw).strip().upper().replace(" ", "").replace("-", "")
+    return f"{digits[:4]}-{digits[4:]}" if len(digits) == 8 else None