Journal Finder: SCIE / SSCI / AHCI / ESCI lists from Wikidata (open, partial) until Clarivate's lists are added

devtools/fetch_wos_wikidata.py

@@ -0,0 +1,43 @@
+"""Web of Science collection lists (SCIE / SSCI / AHCI / ESCI) from Wikidata ("indexed in bibliographic review",
+P8875), for when Clarivate's Master Journal List downloads are not available. Wikidata is CC0 but incomplete
+(roughly 60-90 % of each collection), so a journal missing here may still be indexed.
+
+    python devtools/fetch_wos_wikidata.py      -> finder/data/wos/wikidata_<COLL>.csv
+"""
+import csv
+import sys
+from pathlib import Path
+
+import requests
+
+COLLECTIONS = {"SCIE": "Q104047209", "SSCI": "Q1090953", "AHCI": "Q713927", "ESCI": "Q22908122"}
+OUT = Path(__file__).resolve().parent.parent / "finder" / "data" / "wos"
+QUERY = """SELECT ?j ?title ?issn WHERE {
+  ?j p:P8875 ?st . ?st ps:P8875 wd:%s .
+  FILTER NOT EXISTS { ?st pq:P582 ?end }            # no longer indexed
+  ?j wdt:P236 ?issn .
+  OPTIONAL { ?j wdt:P1476 ?t } OPTIONAL { ?j rdfs:label ?l FILTER(LANG(?l) = "en") }
+  BIND(COALESCE(?t, ?l) AS ?title) }"""
+
+
+def fetch(qid: str) -> list[tuple[str, str]]:
+    r = requests.get("https://query.wikidata.org/sparql", params={"query": QUERY % qid, "format": "json"},
+                     headers={"User-Agent": "CS-AI-Writing-Suite-Finder/1.0"}, timeout=300)
+    r.raise_for_status()
+    return [(b.get("title", {}).get("value", ""), b["issn"]["value"]) for b in r.json()["results"]["bindings"]]
+
+
+def main() -> int:
+    OUT.mkdir(parents=True, exist_ok=True)
+    for coll, qid in COLLECTIONS.items():
+        rows = sorted(set(fetch(qid)))
+        with open(OUT / f"wikidata_{coll}.csv", "w", newline="", encoding="utf-8") as f:
+            w = csv.writer(f)
+            w.writerow(["Journal title", "ISSN"])
+            w.writerows(rows)
+        print(f"{coll}: {len({i for _, i in rows}):,} ISSNs")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

finder_tab.py

@@ -30,6 +30,9 @@ MAX_PER_TERM = int(os.getenv("FINDER_MAX", "100"))
 AVAILABLE = (FINDER_DIR / "run_job.py").exists()
 # the Web of Science lists (Clarivate Master Journal List downloads) in finder/data/wos/
 WOS_READY = any((FINDER_DIR / "data" / "wos").glob("*.csv")) or any((FINDER_DIR / "data" / "wos").glob("*.xlsx"))
+# only the Wikidata copy (devtools/fetch_wos_wikidata.py): good but incomplete, so say so
+WOS_PARTIAL = WOS_READY and all(f.name.startswith("wikidata_") for f in (FINDER_DIR / "data" / "wos").iterdir()
+                                if f.suffix.lower() in (".csv", ".xlsx"))
 
 def freshen_data_caches():
     """The pipeline keeps a parsed copy (.pkl) of each big list (Scopus 20 MB, SCImago, PubMed, ABDC) and trusts
@@ -126,7 +129,8 @@ def _table(csv_path):
         if c in df.columns:
             df[c] = [("—" if pd.isna(v) else round(float(v), 1 if c == "opportunity_score" else 0)) for v in df[c]]
     if "wos" in df.columns:
-        df["wos"] = ["—" if pd.isna(v) or not str(v).strip() else str(v) for v in df["wos"]]
+        df["wos"] = [("not listed" if WOS_PARTIAL else "—") if pd.isna(v) or not str(v).strip() else str(v)
+                     for v in df["wos"]]
     if "in_pubmed" in df.columns:
         # (a column with gaps is read as text: "False" must not count as yes)
         df["in_pubmed"] = ["yes" if str(v).strip().lower() in ("true", "1", "yes") else "—" for v in df["in_pubmed"]]
@@ -282,8 +286,11 @@ def build(ui=None):
                                         "ESCI (Web of Science)", "PubMed", "ABDC"], value=["Scopus"],
                                        label="Must be listed in",
                                        info="Scopus, PubMed and ABDC: every one you tick. Web of Science: any of the "
-                                            "ticked collections." + ("" if WOS_READY else
-                                                                     " (Web of Science lists not installed yet.)"))
+                                            "ticked collections." + (
+                                           " (Web of Science lists not installed yet.)" if not WOS_READY else
+                                           " Web of Science status comes from Wikidata's open list, which misses "
+                                           "some indexed journals: confirm on mjl.clarivate.com before submitting."
+                                           if WOS_PARTIAL else ""))
             quartiles = gr.CheckboxGroup(["Q1", "Q2", "Q3", "Q4"], value=[], label="SJR quartile (empty = any)")
             sort_by = gr.Radio(["Best fit", "Fastest acceptance"], value="Best fit", label="Rank by")
     with gr.Accordion("⚙️ More options (not needed)", open=False):