nikeshn commited on
Commit
888deb2
·
verified ·
1 Parent(s): 6a66b53

Delete openpolicyfinder_web.py

Browse files
Files changed (1) hide show
  1. openpolicyfinder_web.py +0 -883
openpolicyfinder_web.py DELETED
@@ -1,883 +0,0 @@
1
- """
2
- Temporary, read-only Open Policy Finder public-page lookup for ResearchBee.
3
-
4
- Why this module exists
5
- ----------------------
6
- The official Open Policy Finder API requires an API key. Until Khalifa
7
- University's key is issued, ResearchBee may perform a small, user-triggered
8
- lookup against the public journal pages. This is deliberately *not* a bulk
9
- harvester and it does not treat a journal-level website summary as an
10
- article-level permission decision.
11
-
12
- Safety rules
13
- ------------
14
- * Search by ISSN first, then exact title.
15
- * Accept a result only after an exact ISSN match or a very strong title match.
16
- * Parse only text visibly present in the public page HTML.
17
- * Never convert "no policy in place" into "deposit forbidden".
18
- * Never infer a missing embargo as zero months.
19
- * Cache successful lookups to minimise requests to Jisc.
20
- * Keep a direct link to the live record beside every result.
21
-
22
- Once OPF_API_KEY is available, this module should be replaced by the official
23
- API adapter while preserving the same output contract.
24
- """
25
-
26
- from __future__ import annotations
27
-
28
- import asyncio
29
- import logging
30
- import os
31
- import re
32
- import time
33
- import unicodedata
34
- from difflib import SequenceMatcher
35
- from typing import Any, Optional
36
- from urllib.parse import quote_plus, urljoin, urlparse
37
-
38
- import httpx
39
- from bs4 import BeautifulSoup
40
-
41
- from utils import norm_issn
42
-
43
- logger = logging.getLogger(__name__)
44
-
45
- BASE_URL = "https://openpolicyfinder.jisc.ac.uk"
46
- SEARCH_URL = f"{BASE_URL}/search"
47
- CONTACT = "nikesh.narayanan@ku.ac.ae"
48
- USER_AGENT = (
49
- "ResearchBee/1.5.2 (Khalifa University Library; "
50
- f"mailto:{CONTACT}; temporary public-page policy lookup)"
51
- )
52
-
53
- _TIMEOUT = httpx.Timeout(15.0, connect=5.0)
54
- _MAX_HTML_BYTES = 2_000_000
55
-
56
-
57
- def _env_int(name: str, default: int, minimum: int, maximum: int) -> int:
58
- """Read a bounded integer setting without making a bad env value fatal."""
59
- try:
60
- value = int(os.getenv(name, str(default)))
61
- except (TypeError, ValueError):
62
- logger.warning("[OPF web] invalid %s; using %s", name, default)
63
- value = default
64
- return min(max(value, minimum), maximum)
65
-
66
-
67
- _SUCCESS_TTL = _env_int("OPF_WEB_SUCCESS_TTL", 24 * 60 * 60, 300, 7 * 24 * 60 * 60)
68
- _MISS_TTL = _env_int("OPF_WEB_MISS_TTL", 60 * 60, 60, 24 * 60 * 60)
69
- _ERROR_TTL = _env_int("OPF_WEB_ERROR_TTL", 5 * 60, 30, 60 * 60)
70
- _CACHE_MAX = _env_int("OPF_WEB_CACHE_MAX", 512, 32, 5000)
71
- _ENABLED = os.getenv("OPF_WEB_FALLBACK_ENABLED", "true").strip().lower() not in {
72
- "0", "false", "no", "off"
73
- }
74
-
75
- # key -> (expires_at, value). ``value`` is either a parsed record or None.
76
- _CACHE: dict[str, tuple[float, Optional[dict[str, Any]]]] = {}
77
- _LOCKS: dict[str, asyncio.Lock] = {}
78
-
79
-
80
- def _cache_store(key: str, value: Optional[dict[str, Any]], ttl: int) -> None:
81
- """Store a value while pruning expired entries and bounding process memory."""
82
- now = time.time()
83
- expired = [cache_key for cache_key, (expires, _value) in _CACHE.items() if expires <= now]
84
- for cache_key in expired:
85
- _CACHE.pop(cache_key, None)
86
- if key not in _CACHE and len(_CACHE) >= _CACHE_MAX:
87
- oldest = min(_CACHE, key=lambda cache_key: _CACHE[cache_key][0])
88
- _CACHE.pop(oldest, None)
89
- _CACHE[key] = (now + ttl, value)
90
-
91
-
92
- _ISSN_RE = re.compile(r"\b\d{4}-\d{3}[\dXx]\b")
93
- _RECORD_HREF_RE = re.compile(r"/(?:id/)?publication/(\d+)(?:[/?#]|$)", re.I)
94
- _OPTION_RE = re.compile(
95
- r"^(?:Option with\b|No\s+(?:published|accepted|submitted)\s+version\s+policy\s+in\s+place\b)",
96
- re.I,
97
- )
98
- _NO_POLICY_RE = re.compile(
99
- r"^No\s+(published|accepted|submitted)\s+version\s+policy\s+in\s+place",
100
- re.I,
101
- )
102
- _EMBARGO_RE = re.compile(r"\b(\d+)\s*(months?|years?)\s+embargo\b", re.I)
103
- _NO_EMBARGO_RE = re.compile(r"\bno\s+embargo\b", re.I)
104
- _CC_RE = re.compile(
105
- r"\b(CC\s+[A-Z0-9-]+(?:\s+and\s+\d+\s+other\s+licen[cs]e(?:s)?)?)\b",
106
- re.I,
107
- )
108
-
109
- _VERSION_TO_SLOT = {
110
- "submitted": "preprint",
111
- "accepted": "postprint",
112
- "published": "published_version",
113
- }
114
- _SLOT_LABELS = {
115
- "preprint": "Submitted version / preprint",
116
- "postprint": "Accepted manuscript (AAM / postprint)",
117
- "published_version": "Published version (Version of Record)",
118
- }
119
-
120
- _DETAIL_LABELS = {
121
- "location": "where",
122
- "locations": "where",
123
- "permitted location": "where",
124
- "permitted locations": "where",
125
- "repository": "where",
126
- "repositories": "where",
127
- "where": "where",
128
- "embargo": "embargo",
129
- "embargo period": "embargo",
130
- "licence": "licence",
131
- "license": "licence",
132
- "licences": "licence",
133
- "licenses": "licence",
134
- "conditions": "conditions",
135
- "condition": "conditions",
136
- "prerequisites": "conditions",
137
- "pre-requisites": "conditions",
138
- "requirements": "conditions",
139
- }
140
-
141
-
142
- def _space(value: Any) -> str:
143
- return " ".join(str(value or "").split())
144
-
145
-
146
- def _normalise_title(value: str) -> str:
147
- text = unicodedata.normalize("NFKD", value or "")
148
- text = "".join(ch for ch in text if not unicodedata.combining(ch))
149
- text = re.sub(r"\([^)]{1,20}\)\s*$", "", text) # trailing acronym, e.g. (JAL)
150
- text = re.sub(r"[^a-zA-Z0-9]+", " ", text).strip().lower()
151
- text = re.sub(r"^the\s+", "", text)
152
- return _space(text)
153
-
154
-
155
- def _title_score(expected: str, actual: str) -> float:
156
- a, b = _normalise_title(expected), _normalise_title(actual)
157
- if not a or not b:
158
- return 0.0
159
- if a == b:
160
- return 1.0
161
- if a in b or b in a:
162
- return min(len(a), len(b)) / max(len(a), len(b))
163
- return SequenceMatcher(None, a, b).ratio()
164
-
165
-
166
- def _canonical_record_url(url_or_href: str) -> str:
167
- absolute = urljoin(BASE_URL + "/", url_or_href or "")
168
- parsed = urlparse(absolute)
169
- if parsed.scheme != "https" or parsed.netloc.lower() != "openpolicyfinder.jisc.ac.uk":
170
- return ""
171
- match = _RECORD_HREF_RE.search(parsed.path)
172
- if not match:
173
- return ""
174
- return f"{BASE_URL}/id/publication/{match.group(1)}"
175
-
176
-
177
- def public_search_url(journal_name: str = "", issn: str = "") -> str:
178
- query = norm_issn(issn) or _space(journal_name)
179
- if not query:
180
- return BASE_URL
181
- return (
182
- f"{SEARCH_URL}?search={quote_plus(query)}&per_page=10"
183
- "&publication_page=1&publisher_page=1&funder_page=1"
184
- )
185
-
186
-
187
- async def _get_html(client: httpx.AsyncClient, url: str, params: Optional[dict] = None) -> Optional[str]:
188
- parsed = urlparse(url)
189
- if parsed.scheme != "https" or parsed.netloc.lower() != "openpolicyfinder.jisc.ac.uk":
190
- raise ValueError("Open Policy Finder lookup refused a non-Jisc URL")
191
-
192
- headers = {
193
- "User-Agent": USER_AGENT,
194
- "Accept": "text/html,application/xhtml+xml",
195
- "Accept-Language": "en-GB,en;q=0.9",
196
- }
197
- for attempt in range(2):
198
- try:
199
- response = await client.get(
200
- url,
201
- params=params,
202
- headers=headers,
203
- timeout=_TIMEOUT,
204
- follow_redirects=True,
205
- )
206
- except httpx.HTTPError as exc:
207
- if attempt == 0:
208
- await asyncio.sleep(0.8)
209
- continue
210
- raise RuntimeError(f"Open Policy Finder request failed: {type(exc).__name__}") from exc
211
-
212
- final = urlparse(str(response.url))
213
- if final.scheme != "https" or final.netloc.lower() != "openpolicyfinder.jisc.ac.uk":
214
- raise RuntimeError("Open Policy Finder redirected outside its own website")
215
- if response.status_code == 404:
216
- return None
217
- if response.status_code == 429 or 500 <= response.status_code < 600:
218
- if attempt == 0:
219
- retry_after = response.headers.get("Retry-After", "")
220
- try:
221
- delay = min(max(float(retry_after), 0.5), 3.0)
222
- except ValueError:
223
- delay = 1.0
224
- await asyncio.sleep(delay)
225
- continue
226
- response.raise_for_status()
227
-
228
- content_type = response.headers.get("content-type", "").lower()
229
- if "html" not in content_type:
230
- raise RuntimeError("Open Policy Finder returned non-HTML content")
231
- if len(response.content) > _MAX_HTML_BYTES:
232
- raise RuntimeError("Open Policy Finder page exceeded the safe size limit")
233
- html = response.text
234
- lowered = html.lower()
235
- if "just a moment" in lowered and "cloudflare" in lowered:
236
- raise RuntimeError("Open Policy Finder returned an anti-bot challenge")
237
- return html
238
-
239
- return None
240
-
241
-
242
- def _candidate_context(anchor) -> str:
243
- node = anchor
244
- best = _space(anchor.get_text(" ", strip=True))
245
- for _ in range(5):
246
- node = getattr(node, "parent", None)
247
- if node is None:
248
- break
249
- text = _space(node.get_text(" ", strip=True))
250
- if len(text) > len(best) and len(text) <= 1600:
251
- best = text
252
- if len(text) > 1600:
253
- break
254
- return best
255
-
256
-
257
- def _search_candidates(html: str, expected_title: str, expected_issn: str) -> list[dict[str, Any]]:
258
- soup = BeautifulSoup(html, "html.parser")
259
- clean_issn = norm_issn(expected_issn)
260
- candidates: dict[str, dict[str, Any]] = {}
261
-
262
- for anchor in soup.find_all("a", href=True):
263
- url = _canonical_record_url(anchor.get("href", ""))
264
- if not url:
265
- continue
266
- title = _space(anchor.get_text(" ", strip=True))
267
- context = _candidate_context(anchor)
268
- issns = sorted({norm_issn(i) for i in _ISSN_RE.findall(context) if norm_issn(i)})
269
- title_match = _title_score(expected_title, title)
270
- score = int(round(title_match * 60))
271
- if clean_issn and clean_issn in issns:
272
- score += 100
273
- if _normalise_title(title) == _normalise_title(expected_title) and expected_title:
274
- score += 40
275
-
276
- item = {
277
- "url": url,
278
- "title": title,
279
- "context": context,
280
- "issns": issns,
281
- "score": score,
282
- }
283
- previous = candidates.get(url)
284
- if previous is None or score > previous["score"]:
285
- candidates[url] = item
286
-
287
- return sorted(candidates.values(), key=lambda c: c["score"], reverse=True)
288
-
289
-
290
- def _label_value(soup: BeautifulSoup, label: str) -> str:
291
- wanted = label.casefold()
292
- for tag in soup.find_all(["dt", "th", "strong", "span", "div", "p"]):
293
- text = _space(tag.get_text(" ", strip=True)).rstrip(":").casefold()
294
- if text != wanted:
295
- continue
296
- # Definition-list/table markup.
297
- sibling = tag.find_next_sibling(["dd", "td", "span", "div", "p"])
298
- if sibling:
299
- value = _space(sibling.get_text(" ", strip=True))
300
- if value and value.casefold() != wanted:
301
- return value[:500]
302
- # Card/grid markup where label and value share a parent.
303
- parent = tag.parent
304
- if parent:
305
- parts = [_space(s) for s in parent.stripped_strings]
306
- for index, part in enumerate(parts):
307
- if part.rstrip(":").casefold() == wanted and index + 1 < len(parts):
308
- return parts[index + 1][:500]
309
- return ""
310
-
311
-
312
- def _extract_labeled_details(tokens: list[str]) -> dict[str, str]:
313
- result: dict[str, list[str]] = {}
314
- index = 0
315
- while index < len(tokens):
316
- token = _space(tokens[index]).rstrip(":")
317
- key = _DETAIL_LABELS.get(token.casefold())
318
- if not key:
319
- index += 1
320
- continue
321
- values: list[str] = []
322
- cursor = index + 1
323
- while cursor < len(tokens):
324
- value = _space(tokens[cursor])
325
- if value.casefold().rstrip(":") in _DETAIL_LABELS:
326
- break
327
- if value.casefold() not in {"show", "hide", "more", "less"}:
328
- values.append(value)
329
- cursor += 1
330
- if len(" ".join(values)) > 800:
331
- break
332
- joined = _space("; ".join(v for v in values if v))
333
- if joined:
334
- result.setdefault(key, []).append(joined[:800])
335
- index = max(cursor, index + 1)
336
- return {key: _space("; ".join(dict.fromkeys(values))) for key, values in result.items()}
337
-
338
-
339
- def _summary_embargo(summary: str) -> tuple[str, str, Optional[int]]:
340
- if _NO_EMBARGO_RE.search(summary):
341
- return "No embargo (website summary)", "zero", 0
342
- match = _EMBARGO_RE.search(summary)
343
- if match:
344
- amount = int(match.group(1))
345
- unit = match.group(2).lower()
346
- months = amount * 12 if unit.startswith("year") else amount
347
- return f"{amount} {unit}", "stated", months
348
- return "Not stated in page summary", "unknown", None
349
-
350
-
351
- def _summary_licence(summary: str) -> str:
352
- match = _CC_RE.search(summary)
353
- return _space(match.group(1).upper().replace("LICENCE", "licence")) if match else ""
354
-
355
-
356
- def _summary_conditions(summary: str) -> list[str]:
357
- lower = summary.casefold()
358
- conditions: list[str] = []
359
- if "associated oa fees" in lower:
360
- conditions.append("Associated open-access fees apply.")
361
- if "multiple pre-requisites" in lower or "multiple prerequisites" in lower:
362
- conditions.append("Multiple prerequisites apply.")
363
- elif "funder pre-requisites" in lower or "funder prerequisites" in lower:
364
- conditions.append("Funder prerequisites apply.")
365
- elif "pre-requisites" in lower or "prerequisites" in lower:
366
- conditions.append("Prerequisites apply.")
367
- return conditions
368
-
369
-
370
- def _option_from(summary: str, detail_tokens: list[str], record_url: str) -> dict[str, Any]:
371
- summary = _space(summary)
372
- no_policy = bool(_NO_POLICY_RE.match(summary))
373
- details = _extract_labeled_details(detail_tokens)
374
-
375
- if no_policy:
376
- return {
377
- "summary": summary,
378
- "allowed": "No policy listed",
379
- "permission_status": "unknown",
380
- "where": "",
381
- "embargo": "Not stated",
382
- "embargo_status": "unknown",
383
- "embargo_months": None,
384
- "licence": "",
385
- "conditions": summary + ". This does not by itself mean deposit is prohibited.",
386
- "source_url": record_url,
387
- "no_policy_listed": True,
388
- }
389
-
390
- embargo, embargo_status, embargo_months = _summary_embargo(summary)
391
- if details.get("embargo"):
392
- embargo = details["embargo"]
393
- # Preserve tri-state. Details may say "none" or a duration.
394
- if re.search(r"\b(?:no|none|zero|0)\b", embargo, re.I):
395
- embargo_status, embargo_months = "zero", 0
396
- else:
397
- duration = _EMBARGO_RE.search(embargo + " embargo") or re.search(
398
- r"\b(\d+)\s*(months?|years?)\b", embargo, re.I
399
- )
400
- if duration:
401
- amount = int(duration.group(1))
402
- unit = duration.group(2).lower()
403
- embargo_status = "stated"
404
- embargo_months = amount * 12 if unit.startswith("year") else amount
405
- else:
406
- embargo_status, embargo_months = "unknown", None
407
-
408
- conditions = _summary_conditions(summary)
409
- if details.get("conditions"):
410
- conditions.append(details["conditions"])
411
- conditional = bool(conditions)
412
-
413
- licence = details.get("licence") or _summary_licence(summary)
414
- where = details.get("where") or "See the live Open Policy Finder option"
415
-
416
- return {
417
- "summary": summary,
418
- "allowed": "Conditional" if conditional else "Yes",
419
- "permission_status": "conditional" if conditional else "allowed",
420
- "where": where,
421
- "embargo": embargo,
422
- "embargo_status": embargo_status,
423
- "embargo_months": embargo_months,
424
- "licence": licence,
425
- "conditions": _space(" ".join(dict.fromkeys(conditions))),
426
- "source_url": record_url,
427
- "no_policy_listed": False,
428
- }
429
-
430
-
431
- def _policy_tokens(soup: BeautifulSoup) -> list[str]:
432
- main = soup.find("main") or soup
433
- # Script/style text is never policy evidence.
434
- for tag in main.find_all(["script", "style", "noscript", "svg"]):
435
- tag.decompose()
436
- return [_space(token) for token in main.stripped_strings if _space(token)]
437
-
438
-
439
- def _parse_options(soup: BeautifulSoup, record_url: str) -> dict[str, list[dict[str, Any]]]:
440
- tokens = _policy_tokens(soup)
441
- start = next((i for i, token in enumerate(tokens) if token.casefold() == "journal policy"), 0)
442
- end = next(
443
- (i for i in range(start + 1, len(tokens)) if tokens[i].casefold() == "policy links"),
444
- len(tokens),
445
- )
446
- tokens = tokens[start:end]
447
-
448
- by_slot: dict[str, list[dict[str, Any]]] = {slot: [] for slot in _SLOT_LABELS}
449
- current_version = ""
450
- current_summary = ""
451
- current_details: list[str] = []
452
-
453
- def flush() -> None:
454
- nonlocal current_summary, current_details
455
- if not current_summary or not current_version:
456
- current_summary, current_details = "", []
457
- return
458
- slot = _VERSION_TO_SLOT[current_version]
459
- option = _option_from(current_summary, current_details, record_url)
460
- fingerprint = (
461
- option["summary"].casefold(), option["where"].casefold(),
462
- option["embargo"].casefold(), option["licence"].casefold(),
463
- )
464
- existing = {
465
- (item["summary"].casefold(), item["where"].casefold(),
466
- item["embargo"].casefold(), item["licence"].casefold())
467
- for item in by_slot[slot]
468
- }
469
- if fingerprint not in existing:
470
- by_slot[slot].append(option)
471
- current_summary, current_details = "", []
472
-
473
- for token in tokens:
474
- lower = token.casefold()
475
- if lower in _VERSION_TO_SLOT:
476
- flush()
477
- current_version = lower
478
- continue
479
- if _OPTION_RE.match(token):
480
- flush()
481
- missing = _NO_POLICY_RE.match(token)
482
- if missing:
483
- current_version = missing.group(1).casefold()
484
- current_summary = token
485
- continue
486
- if current_summary and lower not in {"show", "hide"}:
487
- current_details.append(token)
488
- flush()
489
- return by_slot
490
-
491
-
492
- def _aggregate_slot(options: list[dict[str, Any]], record_url: str) -> dict[str, Any]:
493
- real = [item for item in options if not item.get("no_policy_listed")]
494
- if not real:
495
- if options:
496
- first = options[0]
497
- return {
498
- "allowed": "No policy listed",
499
- "permission_status": "unknown",
500
- "embargo_status": "unknown",
501
- "embargo_months": None,
502
- "where": "",
503
- "embargo": "Not stated",
504
- "licence": "",
505
- "conditions": first["conditions"],
506
- "options": options,
507
- "source": "Open Policy Finder public website",
508
- "source_url": record_url,
509
- }
510
- return {
511
- "allowed": "Unclear",
512
- "permission_status": "unknown",
513
- "embargo_status": "unknown",
514
- "embargo_months": None,
515
- "where": "",
516
- "embargo": "Not stated in the public page summary",
517
- "licence": "",
518
- "conditions": "No version summary could be parsed. Check the live record.",
519
- "options": [],
520
- "source": "Open Policy Finder public website",
521
- "source_url": record_url,
522
- }
523
-
524
- allowed = "Conditional" if len(real) > 1 or any(
525
- item["permission_status"] == "conditional" for item in real
526
- ) else "Yes"
527
-
528
- def unique(field: str) -> list[str]:
529
- return list(dict.fromkeys(_space(item.get(field, "")) for item in real if _space(item.get(field, ""))))
530
-
531
- where_values = unique("where")
532
- embargo_values = unique("embargo")
533
- licence_values = unique("licence")
534
- condition_values = unique("conditions")
535
-
536
- if len(real) > 1:
537
- condition_values.insert(
538
- 0,
539
- "Multiple policy options are listed. Choose the option matching your funder, fees, licence and deposit location.",
540
- )
541
-
542
- statuses = {item.get("embargo_status", "unknown") for item in real}
543
- embargo_status = statuses.pop() if len(statuses) == 1 else "varies"
544
- embargo_month_values = list(dict.fromkeys(
545
- item.get("embargo_months")
546
- for item in real
547
- if item.get("embargo_months") is not None
548
- ))
549
- embargo_months = (
550
- embargo_month_values[0]
551
- if len(embargo_month_values) == 1 and embargo_status in {"zero", "stated"}
552
- else None
553
- )
554
-
555
- return {
556
- "allowed": allowed,
557
- "permission_status": "conditional" if allowed == "Conditional" else "allowed",
558
- "embargo_status": embargo_status,
559
- "embargo_months": embargo_months,
560
- "where": where_values[0] if len(where_values) == 1 else (
561
- "Varies by option — see live record" if where_values else "See live record"
562
- ),
563
- "embargo": embargo_values[0] if len(embargo_values) == 1 else (
564
- "Varies by option: " + "; ".join(embargo_values) if embargo_values else "Not stated"
565
- ),
566
- "licence": licence_values[0] if len(licence_values) == 1 else (
567
- "Varies by option: " + "; ".join(licence_values) if licence_values else ""
568
- ),
569
- "conditions": _space(" ".join(condition_values)),
570
- "options": real,
571
- "source": "Open Policy Finder public website",
572
- "source_url": record_url,
573
- }
574
-
575
-
576
- def _policy_links(soup: BeautifulSoup) -> list[dict[str, str]]:
577
- main = soup.find("main") or soup
578
- links: list[dict[str, str]] = []
579
- for anchor in main.find_all("a", href=True):
580
- href = urljoin(BASE_URL + "/", anchor.get("href", ""))
581
- parsed = urlparse(href)
582
- if parsed.scheme not in {"http", "https"}:
583
- continue
584
- if parsed.netloc.lower() == "openpolicyfinder.jisc.ac.uk":
585
- continue
586
- label = _space(anchor.get_text(" ", strip=True)) or parsed.netloc
587
- if any(item["url"] == href for item in links):
588
- continue
589
- links.append({"label": label[:160], "url": href})
590
- if len(links) >= 8:
591
- break
592
- return links
593
-
594
-
595
- def parse_record_html(
596
- html: str,
597
- record_url: str,
598
- expected_title: str = "",
599
- expected_issn: str = "",
600
- ) -> Optional[dict[str, Any]]:
601
- """Parse one public journal page. Pure function used by regression tests."""
602
- if not html:
603
- return None
604
- soup = BeautifulSoup(html, "html.parser")
605
- h1 = soup.find("h1")
606
- title = _space(h1.get_text(" ", strip=True)) if h1 else ""
607
- if not title:
608
- return None
609
-
610
- page_text = _space((soup.find("main") or soup).get_text(" ", strip=True))
611
- issns = sorted({norm_issn(i) for i in _ISSN_RE.findall(page_text) if norm_issn(i)})
612
- expected_clean = norm_issn(expected_issn)
613
- title_match = _title_score(expected_title, title) if expected_title else 0.0
614
-
615
- if expected_clean and issns and expected_clean not in issns:
616
- return None
617
- if expected_clean and expected_clean in issns:
618
- match_quality = "exact_issn"
619
- elif expected_title and title_match >= 0.96:
620
- match_quality = "exact_title"
621
- elif expected_title and title_match >= 0.88 and not issns:
622
- match_quality = "strong_title"
623
- elif not expected_title and not expected_clean:
624
- match_quality = "record_url"
625
- else:
626
- return None
627
-
628
- options = _parse_options(soup, record_url)
629
- if not any(options.values()):
630
- # A page without any policy option text is not sufficient evidence.
631
- return None
632
-
633
- publisher = _label_value(soup, "Publisher")
634
- updated = ""
635
- update_match = re.search(
636
- r"\b(?:last\s+updated|last\s+modified|record\s+updated)\s*:?\s*"
637
- r"([0-9]{1,2}[/-][0-9]{1,2}[/-][0-9]{2,4}|[0-9]{4}-[0-9]{2}-[0-9]{2}|"
638
- r"[0-9]{1,2}\s+[A-Za-z]+\s+[0-9]{4})",
639
- page_text,
640
- re.I,
641
- )
642
- if update_match:
643
- updated = update_match.group(1)
644
-
645
- return {
646
- "journal_title": title,
647
- "publisher": publisher,
648
- "issns": issns,
649
- "record_url": record_url,
650
- "record_id": (_RECORD_HREF_RE.search(urlparse(record_url).path) or [None, ""])[1],
651
- "match_quality": match_quality,
652
- "title_match_score": round(title_match, 3),
653
- "versions": {
654
- slot: _aggregate_slot(slot_options, record_url)
655
- for slot, slot_options in options.items()
656
- },
657
- "policy_links": _policy_links(soup),
658
- "record_updated": updated,
659
- "retrieved_at": time.strftime("%Y-%m-%d"),
660
- "source": "Open Policy Finder public website",
661
- }
662
-
663
-
664
- async def _lookup_uncached(
665
- client: httpx.AsyncClient,
666
- journal_name: str,
667
- issn: str,
668
- ) -> Optional[dict[str, Any]]:
669
- clean_issn = norm_issn(issn)
670
- queries = list(dict.fromkeys(q for q in (clean_issn, _space(journal_name)) if q))
671
-
672
- for query in queries:
673
- params = {
674
- "search": query,
675
- "per_page": 10,
676
- "publication_page": 1,
677
- "publisher_page": 1,
678
- "funder_page": 1,
679
- }
680
- html = await _get_html(client, SEARCH_URL, params=params)
681
- if not html:
682
- continue
683
- candidates = _search_candidates(html, journal_name, clean_issn)
684
- # Fetch only the most plausible few records; this is a user-triggered lookup,
685
- # not a crawl.
686
- for candidate in candidates[:3]:
687
- if candidate["score"] < 45 and not clean_issn:
688
- continue
689
- record_html = await _get_html(client, candidate["url"])
690
- if not record_html:
691
- continue
692
- parsed = parse_record_html(
693
- record_html,
694
- candidate["url"],
695
- expected_title=journal_name,
696
- expected_issn=clean_issn,
697
- )
698
- if parsed:
699
- return parsed
700
- return None
701
-
702
-
703
- async def lookup_policy(
704
- journal_name: str,
705
- issn: str = "",
706
- client: Optional[httpx.AsyncClient] = None,
707
- ) -> Optional[dict[str, Any]]:
708
- """Resolve and parse one live OPF journal page, with a bounded TTL cache."""
709
- if not _ENABLED:
710
- return None
711
- clean_issn = norm_issn(issn)
712
- key = f"{clean_issn}|{_normalise_title(journal_name)}"
713
- if not key.strip("|"):
714
- return None
715
-
716
- now = time.time()
717
- cached = _CACHE.get(key)
718
- if cached and cached[0] > now:
719
- return cached[1]
720
-
721
- lock = _LOCKS.setdefault(key, asyncio.Lock())
722
- async with lock:
723
- cached = _CACHE.get(key)
724
- if cached and cached[0] > time.time():
725
- return cached[1]
726
-
727
- owns_client = client is None
728
- if client is None:
729
- client = httpx.AsyncClient(timeout=_TIMEOUT)
730
- try:
731
- result = await _lookup_uncached(client, journal_name, clean_issn)
732
- ttl = _SUCCESS_TTL if result else _MISS_TTL
733
- _cache_store(key, result, ttl)
734
- return result
735
- except Exception as exc: # conservative fallback: caller renders unknown
736
- logger.warning(
737
- "[OPF web] lookup failed for journal=%r issn=%r: %s: %s",
738
- journal_name,
739
- clean_issn,
740
- type(exc).__name__,
741
- exc,
742
- )
743
- _cache_store(key, None, _ERROR_TTL)
744
- return None
745
- finally:
746
- if owns_client and client is not None:
747
- await client.aclose()
748
- _LOCKS.pop(key, None)
749
-
750
-
751
- def to_green_oa(record: dict[str, Any]) -> dict[str, Any]:
752
- """Convert a parsed public-page record to ResearchBee's policy card shape."""
753
- record_url = record.get("record_url") or BASE_URL
754
- retrieved = record.get("retrieved_at") or time.strftime("%Y-%m-%d")
755
- versions = record.get("versions") or {}
756
- preprint = versions.get("preprint") or _aggregate_slot([], record_url)
757
- postprint = versions.get("postprint") or _aggregate_slot([], record_url)
758
- published = versions.get("published_version") or _aggregate_slot([], record_url)
759
-
760
- evidence_urls = [record_url]
761
- evidence_urls.extend(
762
- item.get("url", "") for item in (record.get("policy_links") or [])
763
- if item.get("url")
764
- )
765
- evidence_urls = list(dict.fromkeys(evidence_urls))[:8]
766
-
767
- licences = list(dict.fromkeys(
768
- slot.get("licence", "") for slot in (preprint, postprint, published)
769
- if slot.get("licence")
770
- ))
771
- evidence_note = (
772
- "Source: live Open Policy Finder public journal page. "
773
- "Temporary website lookup while direct API access is pending. "
774
- f"Retrieved {retrieved}."
775
- )
776
- if record.get("match_quality") == "strong_title":
777
- evidence_note += " The record was matched by journal title because no ISSN was visible on the page."
778
-
779
- field_provenance = {
780
- slot: {
781
- "source": "Open Policy Finder public website",
782
- "url": record_url,
783
- "retrieved_at": retrieved,
784
- "verification": "web_summary",
785
- }
786
- for slot in ("preprint", "postprint", "published_version")
787
- }
788
-
789
- return {
790
- "policy_status": "Web sourced",
791
- "source_type": "open_policy_finder_web",
792
- # A journal page can list possible routes, but it cannot establish which
793
- # article-specific route applies. Keep the overall deposit decision unknown.
794
- "permission_status": "unknown",
795
- "embargo_status": "unknown",
796
- "preprint": preprint,
797
- "postprint": postprint,
798
- "published_version": published,
799
- "licence_notes": "; ".join(licences),
800
- "repository_action_note": (
801
- "Use the live Open Policy Finder record to select the option that matches "
802
- "your manuscript version, funder, licence and deposit location."
803
- ),
804
- "evidence_note": evidence_note,
805
- "risk_flag": (
806
- "Journal-level website summary only. Check the live option before depositing; "
807
- "an article DOI may return a more specific permission result."
808
- ),
809
- "deposit_statement": "",
810
- "copyright_owner": "",
811
- "copyright_name": record.get("journal_title", ""),
812
- "journal_oa_type": "",
813
- "policy_issuer": record.get("publisher", ""),
814
- "issuer_issns": record.get("issns", []),
815
- "embargo_end": "",
816
- "policy_updated": record.get("record_updated", ""),
817
- "evidence_urls": evidence_urls,
818
- "monitoring": "",
819
- "field_provenance": field_provenance,
820
- "web_record": {
821
- "url": record_url,
822
- "match_quality": record.get("match_quality", ""),
823
- "retrieved_at": retrieved,
824
- "policy_links": record.get("policy_links", []),
825
- },
826
- # Preserve tri-state safety for deterministic recommendation logic.
827
- "_can_archive": None,
828
- "_versions": [slot for slot, value in versions.items() if value.get("options")],
829
- "_best_version": "",
830
- "_locations": [],
831
- "_institutional_ok": False,
832
- "_embargo_months": None,
833
- "_embargo_status": "unknown",
834
- "_embargo_text": "Varies by journal-level option; check the live record",
835
- "_licence": "",
836
- "_verify_url": record_url,
837
- }
838
-
839
-
840
- def next_actions(record: dict[str, Any], requested_version: str = "") -> list[str]:
841
- version = _space(requested_version)
842
- actions = [
843
- f"Open the live Open Policy Finder record and review the option for {version}."
844
- if version and version.casefold() != "all versions"
845
- else "Open the live Open Policy Finder record and choose the option for your manuscript version.",
846
- "Confirm any funder prerequisite, fee, licence, embargo and permitted deposit location before uploading the file.",
847
- "When an article DOI is available, run the DOI check for an article-level permission result and a Khazna status check.",
848
- ]
849
- return actions
850
-
851
-
852
- def format_for_chat(record: Optional[dict[str, Any]]) -> str:
853
- if not record:
854
- return ""
855
- lines = [
856
- f"Open Policy Finder public-page summary for {record.get('journal_title', 'the journal')}",
857
- f"Live record: {record.get('record_url', BASE_URL)}",
858
- ]
859
- for slot, label in _SLOT_LABELS.items():
860
- version = (record.get("versions") or {}).get(slot) or {}
861
- lines.append(
862
- f"- {label}: {version.get('allowed', 'Unclear')}; "
863
- f"embargo: {version.get('embargo', 'not stated')}; "
864
- f"licence: {version.get('licence') or 'not stated'}; "
865
- f"conditions: {version.get('conditions') or 'none stated in the summary'}"
866
- )
867
- lines.append(
868
- "This is a journal-level website summary, not an article-level permission decision. "
869
- "Tell the user to check the live record before deposit."
870
- )
871
- return "\n".join(lines)
872
-
873
-
874
-
875
-
876
- def is_enabled() -> bool:
877
- """Whether the temporary public-page fallback is enabled by configuration."""
878
- return _ENABLED
879
-
880
-
881
- def clear_cache() -> None:
882
- """Test/admin helper."""
883
- _CACHE.clear()