ICSAC commited on
Commit
212a83a
·
1 Parent(s): 86b827a

Preprints from any server: the upload form takes a preprint DOI, checked against Crossref and DataCite when submitted (a published article, a preprint already published, or a DOI that does not exist is refused; an unreachable registry fails open and the curation team is told), and the Persistence DOI is linked to it; the DOI route sends other servers to the upload form with the DOI; the exclusivity confirmation is stored on both routes (it never was); the acceptance email says it only when it was confirmed

Browse files
crossref_deposit.py CHANGED
@@ -312,24 +312,20 @@ def _license(parent: ET.Element, license_id: str) -> None:
312
  ref.text = url
313
 
314
 
315
- _ARXIV_ID = re.compile(r"^(?:arxiv:)?(?:10\.48550/arxiv\.)?(\d{4}\.\d{4,5})(?:v\d+)?$", re.I)
316
-
317
-
318
  def preprint_doi(submission: dict) -> Optional[str]:
319
- """The DOI of the preprint a DOI-route paper was submitted from (a Zenodo
320
- DOI, or arXiv's 10.48550 DOI for an arXiv id), else None. The Persistence
321
- article gets its own DOI and points back to it: Crossref's versioning
322
- practice gives a preprint and its published version separate DOIs, linked
323
- by a relation (policy 2026-09-28; the five founding papers keep their
324
- Zenodo DOIs, as stated to Crossref on 2026-07-06)."""
325
- if submission.get("source") != "doi":
326
- return None
327
- ref = (submission.get("doi") or submission.get("source_ref") or "").strip()
328
- ref = re.sub(r"^(?:https?://(?:dx\.)?doi\.org/|doi:)", "", ref, flags=re.I)
329
- m = _ARXIV_ID.match(ref)
330
- if m:
331
- return f"10.48550/arXiv.{m.group(1)}"
332
- return ref if re.match(r"^10\.\d{4,9}/\S+$", ref) else None
333
 
334
 
335
  def _relations(parent: ET.Element, submission: dict) -> None:
 
312
  ref.text = url
313
 
314
 
 
 
 
315
  def preprint_doi(submission: dict) -> Optional[str]:
316
+ """The DOI of the preprint the paper was posted as: the DOI route's own DOI
317
+ (Zenodo, or arXiv's 10.48550 DOI for an arXiv id), or the upload route's
318
+ preprint DOI field (any server). None otherwise. The Persistence article
319
+ gets its own DOI and points back to it: Crossref's versioning practice
320
+ gives a preprint and its published version separate DOIs, linked by a
321
+ relation (policy 2026-09-28; the five founding papers keep their Zenodo
322
+ DOIs, as stated to Crossref on 2026-07-06)."""
323
+ import preprint_check
324
+ if submission.get("source") == "doi":
325
+ ref = submission.get("doi") or submission.get("source_ref")
326
+ else:
327
+ ref = submission.get("preprint_doi")
328
+ return preprint_check.normalize(ref) if ref else None
 
329
 
330
 
331
  def _relations(parent: ET.Element, submission: dict) -> None:
intake/apply_decision.py CHANGED
@@ -220,7 +220,7 @@ def main(argv: list[str]) -> int:
220
  crossref_path = (verdict == "accept" and source in ("upload", "doi")
221
  and registrar == "crossref" and not (test_mode and tier == 2))
222
  preprint = None
223
- if crossref_path and source == "doi":
224
  import crossref_deposit as _cdp
225
  preprint = _cdp.preprint_doi(submission)
226
  if crossref_path and not state_pre.get("crossref_doi"):
@@ -284,18 +284,22 @@ def main(argv: list[str]) -> int:
284
  f"Will resolve to: {state_pre.get('crossref_landing_url')}"]
285
  else:
286
  lines.append("Crossref draft: FAILED to stage -- see audit log")
287
- if source == "doi":
288
- lines.append(f"Preprint (linked as hasPreprint): "
289
- f"{('https://doi.org/' + preprint) if preprint else 'NOT RESOLVED -- check the relation before --live'}")
290
- if not (submission.get("license") or "").lower().startswith("cc"):
291
- lines.append(f"Licence on the preprint: {submission.get('license') or 'none recorded'} "
292
- f"-- check it before --live (the site hosts the PDF)")
293
- elif state_pre.get("deposit_draft_url"):
294
- lines.append(f"Zenodo archive DRAFT (unpublished): {state_pre['deposit_draft_url']}")
295
- elif form.get("deposit_consent"):
296
- lines.append("Zenodo archive draft: NOT staged -- see audit log")
297
- else:
298
- lines.append("Zenodo archive: author did not consent to deposit")
 
 
 
 
299
  lines += ["Acceptance email: Gmail Drafts (review and send by hand).",
300
  f"When ready: intake/register-doi.sh {sub_id} --live"]
301
  notify.send_to_curator("\n".join(lines), parse_mode=None,
@@ -419,6 +423,7 @@ def main(argv: list[str]) -> int:
419
  terms_version=submission.get("terms_version") or getattr(config, "TERMS_VERSION", ""),
420
  approval_url=approval_url, objection_deadline=objection_deadline,
421
  code_link_claims=code_link_claims, preprint_doi=preprint or "",
 
422
  )
423
  if ok:
424
  # Decision emails go to Gmail Drafts (curator-applied decision path).
 
220
  crossref_path = (verdict == "accept" and source in ("upload", "doi")
221
  and registrar == "crossref" and not (test_mode and tier == 2))
222
  preprint = None
223
+ if crossref_path:
224
  import crossref_deposit as _cdp
225
  preprint = _cdp.preprint_doi(submission)
226
  if crossref_path and not state_pre.get("crossref_doi"):
 
284
  f"Will resolve to: {state_pre.get('crossref_landing_url')}"]
285
  else:
286
  lines.append("Crossref draft: FAILED to stage -- see audit log")
287
+ if preprint:
288
+ lines.append(f"Preprint (linked as hasPreprint): https://doi.org/{preprint}")
289
+ for flag in ((submission.get("preprint_meta") or {}).get("flags") or []):
290
+ lines.append(f" check the preprint: {flag}")
291
+ elif source == "doi":
292
+ lines.append("Preprint: NOT RESOLVED -- check the relation before --live")
293
+ if source == "doi" and not (submission.get("license") or "").lower().startswith("cc"):
294
+ lines.append(f"Licence on the preprint: {submission.get('license') or 'none recorded'} "
295
+ f"-- check it before --live (the site hosts the PDF)")
296
+ if source == "upload": # a DOI-route preprint is archived where it is
297
+ if state_pre.get("deposit_draft_url"):
298
+ lines.append(f"Zenodo archive DRAFT (unpublished): {state_pre['deposit_draft_url']}")
299
+ elif form.get("deposit_consent"):
300
+ lines.append("Zenodo archive draft: NOT staged -- see audit log")
301
+ else:
302
+ lines.append("Zenodo archive: author did not consent to deposit")
303
  lines += ["Acceptance email: Gmail Drafts (review and send by hand).",
304
  f"When ready: intake/register-doi.sh {sub_id} --live"]
305
  notify.send_to_curator("\n".join(lines), parse_mode=None,
 
423
  terms_version=submission.get("terms_version") or getattr(config, "TERMS_VERSION", ""),
424
  approval_url=approval_url, objection_deadline=objection_deadline,
425
  code_link_claims=code_link_claims, preprint_doi=preprint or "",
426
+ exclusivity_confirmed=form.get("exclusivity_acknowledged") is True,
427
  )
428
  if ok:
429
  # Decision emails go to Gmail Drafts (curator-applied decision path).
intake/intake_server.py CHANGED
@@ -45,6 +45,7 @@ if str(_REPO_ROOT) not in sys.path:
45
  import config # noqa: E402
46
 
47
  import submission_intake as ingest # noqa: E402 — pipeline module
 
48
  import notify # noqa: E402
49
 
50
  from . import notify_author # local
@@ -745,6 +746,7 @@ async def handle_test_pipeline_submission(
745
 
746
  if has_doi:
747
  _validate_doi_shape(doi)
 
748
 
749
  sub_id = _allocate_test_sub_id()
750
  token = _test_token(tier, seed=sub_id)
@@ -877,6 +879,8 @@ async def handle_test_pipeline_submission(
877
  "funding": funding,
878
  "related_identifiers": related_identifiers,
879
  "code_data": code_data or {"available": None, "url": None},
 
 
880
  "pdf": {
881
  "filename": "paper.pdf",
882
  "size_bytes": pdf_size,
@@ -952,6 +956,31 @@ def healthz():
952
  return {"ok": True, "ts": _now_iso()}
953
 
954
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
955
  def _validate_doi_shape(doi: str) -> None:
956
  """Synchronous shape-only check on a DOI or arXiv reference. The actual
957
  resolution + PDF fetch is deferred to the worker so the handler
@@ -969,11 +998,10 @@ def _validate_doi_shape(doi: str) -> None:
969
  raise HTTPException(422, {
970
  "error": "doi_unsupported",
971
  "message": (
972
- "DOI mode supports Zenodo (10.5281/zenodo.NNNNN) and arXiv "
973
- "(10.48550/arXiv.YYMM.NNNNN, or a bare arXiv ID like "
974
- "2103.12345). Other DOI sources (Crossref, DataCite, "
975
- "publisher DOIs) are a known gap — for those, upload the "
976
- "PDF directly using the 'Upload PDF' tab."
977
  ),
978
  })
979
  raise HTTPException(422, {
@@ -1006,6 +1034,8 @@ async def api_submit(request: Request):
1006
  "email": form.get("email"),
1007
  "orcid": form.get("orcid", ""),
1008
  "coi": form.get("coi", ""),
 
 
1009
  })
1010
  code_data = _parse_code_data(form.get("code_data_available"), form.get("code_data_url"))
1011
 
@@ -1093,6 +1123,7 @@ async def api_submit(request: Request):
1093
  # previously blew CF Pages' upstream-fetch timeout and blocked
1094
  # the form's redirect on success).
1095
  _validate_doi_shape(doi)
 
1096
 
1097
  sub_id = _allocate_sub_id()
1098
  sub_dir = SUBMISSIONS_ROOT / sub_id
@@ -1228,6 +1259,8 @@ async def api_submit(request: Request):
1228
  "funding": funding,
1229
  "related_identifiers": related_identifiers,
1230
  "code_data": code_data or {"available": None, "url": None},
 
 
1231
  "pdf": {
1232
  "filename": "paper.pdf",
1233
  "size_bytes": pdf_size,
 
45
  import config # noqa: E402
46
 
47
  import submission_intake as ingest # noqa: E402 — pipeline module
48
+ import preprint_check # noqa: E402
49
  import notify # noqa: E402
50
 
51
  from . import notify_author # local
 
746
 
747
  if has_doi:
748
  _validate_doi_shape(doi)
749
+ preprint = None if has_doi else await _preprint_from_form(form)
750
 
751
  sub_id = _allocate_test_sub_id()
752
  token = _test_token(tier, seed=sub_id)
 
879
  "funding": funding,
880
  "related_identifiers": related_identifiers,
881
  "code_data": code_data or {"available": None, "url": None},
882
+ "preprint_doi": preprint["doi"] if preprint else None,
883
+ "preprint_meta": preprint,
884
  "pdf": {
885
  "filename": "paper.pdf",
886
  "size_bytes": pdf_size,
 
956
  return {"ok": True, "ts": _now_iso()}
957
 
958
 
959
+ async def _preprint_from_form(form) -> dict | None:
960
+ """The upload route's optional preprint DOI, from any preprint server
961
+ (policy 2026-09-28). The preprint keeps its DOI; the Persistence article gets
962
+ its own, linked to it. Refused (422) when it is not a DOI, does not exist, or
963
+ is a published work; a registry that cannot be reached fails open (the
964
+ record says unverified and the curation team checks by hand)."""
965
+ raw = form.get("preprint_doi")
966
+ raw = raw.strip() if isinstance(raw, str) else ""
967
+ if not raw:
968
+ return None
969
+ doi = preprint_check.normalize(raw)
970
+ if not doi:
971
+ raise HTTPException(422, {
972
+ "error": "preprint_doi_invalid",
973
+ "message": ("The preprint DOI should look like 10.xxxx/… (or an arXiv ID such as "
974
+ "2401.12345). Check it, or leave the field empty."),
975
+ })
976
+ title = form.get("title")
977
+ try:
978
+ return await asyncio.to_thread(preprint_check.check, doi,
979
+ title=title if isinstance(title, str) else "")
980
+ except preprint_check.Refused as exc:
981
+ raise HTTPException(422, {"error": "preprint_doi_refused", "message": str(exc)})
982
+
983
+
984
  def _validate_doi_shape(doi: str) -> None:
985
  """Synchronous shape-only check on a DOI or arXiv reference. The actual
986
  resolution + PDF fetch is deferred to the worker so the handler
 
998
  raise HTTPException(422, {
999
  "error": "doi_unsupported",
1000
  "message": (
1001
+ "The DOI route fetches papers from Zenodo and arXiv. For a "
1002
+ "preprint on any other server, use the upload form: upload "
1003
+ "the PDF and enter this DOI as your preprint DOI "
1004
+ f"(https://icsacinstitute.org/submit/upload?preprint_doi={doi})."
 
1005
  ),
1006
  })
1007
  raise HTTPException(422, {
 
1034
  "email": form.get("email"),
1035
  "orcid": form.get("orcid", ""),
1036
  "coi": form.get("coi", ""),
1037
+ # Never passed before 2026-09-28, so every submission stored None.
1038
+ "exclusivity_acknowledged": form.get("exclusivity_acknowledged"),
1039
  })
1040
  code_data = _parse_code_data(form.get("code_data_available"), form.get("code_data_url"))
1041
 
 
1123
  # previously blew CF Pages' upstream-fetch timeout and blocked
1124
  # the form's redirect on success).
1125
  _validate_doi_shape(doi)
1126
+ preprint = None if has_doi else await _preprint_from_form(form)
1127
 
1128
  sub_id = _allocate_sub_id()
1129
  sub_dir = SUBMISSIONS_ROOT / sub_id
 
1259
  "funding": funding,
1260
  "related_identifiers": related_identifiers,
1261
  "code_data": code_data or {"available": None, "url": None},
1262
+ "preprint_doi": preprint["doi"] if preprint else None,
1263
+ "preprint_meta": preprint,
1264
  "pdf": {
1265
  "filename": "paper.pdf",
1266
  "size_bytes": pdf_size,
intake/notify_author.py CHANGED
@@ -310,6 +310,7 @@ def send_decision(*, to: str, sub_id: str, title: str, author_name: str,
310
  objection_deadline: str = "",
311
  code_link_claims: list[str] | None = None,
312
  preprint_doi: str = "",
 
313
  ) -> tuple[bool, str]:
314
  """Send the decision email with two PDF attachments (panel report + RQC).
315
 
@@ -438,6 +439,8 @@ def send_decision(*, to: str, sub_id: str, title: str, author_name: str,
438
  "prior_doi_line": (
439
  f"Your preprint keeps its own DOI ({preprint_doi}) and stays where it is. The *Persistence* "
440
  f"DOI is the published version, and the two are linked in their DOI records."
 
 
441
  if preprint_doi else
442
  "We take it that this paper has no other DOI and is not under review elsewhere, "
443
  "as confirmed at submission."),
 
310
  objection_deadline: str = "",
311
  code_link_claims: list[str] | None = None,
312
  preprint_doi: str = "",
313
+ exclusivity_confirmed: bool = False,
314
  ) -> tuple[bool, str]:
315
  """Send the decision email with two PDF attachments (panel report + RQC).
316
 
 
439
  "prior_doi_line": (
440
  f"Your preprint keeps its own DOI ({preprint_doi}) and stays where it is. The *Persistence* "
441
  f"DOI is the published version, and the two are linked in their DOI records."
442
+ + (" We take it that the paper is not under review elsewhere, as confirmed at submission."
443
+ if exclusivity_confirmed else "")
444
  if preprint_doi else
445
  "We take it that this paper has no other DOI and is not under review elsewhere, "
446
  "as confirmed at submission."),
intake/scripts/test_preprint_doi.py ADDED
@@ -0,0 +1,215 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Checks for preprint DOIs from any server (policy 2026-09-28) and the
3
+ exclusivity confirmation on both routes. Offline: the DOI registries are
4
+ replaced by canned records, no mail, no pings; every write lands in one
5
+ temporary directory that is removed at the end, pass or fail. Run from the
6
+ repo root with the venv:
7
+ .venv/bin/python intake/scripts/test_preprint_doi.py
8
+ """
9
+ from __future__ import annotations
10
+
11
+ import email as _email
12
+ import email.policy as _policy
13
+ import hashlib
14
+ import hmac
15
+ import json
16
+ import pathlib
17
+ import shutil
18
+ import sys
19
+ import tempfile
20
+ import time
21
+ from pathlib import Path
22
+
23
+ ROOT = Path(__file__).resolve().parents[2]
24
+ sys.path.insert(0, str(ROOT))
25
+ PAPER_PDF = Path.home() / "icsac-submissions" / "ICSAC-SUB-00008" / "paper.pdf"
26
+ TEST_ORCID = "0000-0002-1825-0097"
27
+
28
+ failures: list[str] = []
29
+
30
+
31
+ def check(cond: bool, name: str) -> None:
32
+ print((" ok " if cond else " FAIL ") + name)
33
+ if not cond:
34
+ failures.append(name)
35
+
36
+
37
+ # Canned registry answers, keyed by URL fragment.
38
+ RA = {"10.20944/preprints202607.0102.v1": "Crossref", "10.1101/2024.09.14.613029": "Crossref",
39
+ "10.1038/nature12373": "Crossref", "10.1364/opticaopen.29459153.v1": "Crossref",
40
+ "10.48550/arXiv.2401.12345": "DataCite", "10.5281/zenodo.1": "DataCite",
41
+ "10.51094/jxiv.1": "JaLC"}
42
+ CR = {
43
+ "10.20944/preprints202607.0102.v1": {"type": "posted-content", "subtype": "preprint",
44
+ "title": ["A test paper on boundaries"], "publisher": "MDPI AG"},
45
+ "10.1101/2024.09.14.613029": {"type": "posted-content", "subtype": "preprint",
46
+ "title": ["Something else entirely about proteins"],
47
+ "institution": [{"name": "bioRxiv"}]},
48
+ "10.1038/nature12373": {"type": "journal-article", "title": ["Nanometre-scale thermometry"],
49
+ "container-title": ["Nature"]},
50
+ "10.1364/opticaopen.29459153.v1": {"type": "posted-content", "subtype": "preprint", "title": ["X"],
51
+ "relation": {"is-preprint-of": [{"id": "10.1364/OE.572415"}]}},
52
+ }
53
+ DC = {"10.48550/arXiv.2401.12345": {"types": {"resourceTypeGeneral": "Preprint"}, "publisher": "arXiv",
54
+ "titles": [{"title": "A test paper on boundaries"}]},
55
+ "10.5281/zenodo.1": {"types": {"resourceTypeGeneral": "JournalArticle"}, "publisher": {"name": "Zenodo"},
56
+ "titles": [{"title": "A test paper on boundaries"}]}}
57
+ NETWORK_DOWN = {"on": False}
58
+
59
+
60
+ def fake_get(url: str, timeout: float):
61
+ import urllib.parse
62
+ if NETWORK_DOWN["on"]:
63
+ raise OSError("network down")
64
+ tail = urllib.parse.unquote(url.split("/ra/", 1)[-1] if "/ra/" in url else
65
+ url.split("/works/", 1)[-1] if "/works/" in url else url.split("/dois/", 1)[-1])
66
+ if "/ra/" in url:
67
+ return [{"DOI": tail, "RA": RA[tail]}] if tail in RA else [{"DOI": tail, "status": "DOI does not exist"}]
68
+ if "/works/" in url:
69
+ return {"message": CR[tail]}
70
+ return {"data": {"attributes": DC[tail]}}
71
+
72
+
73
+ tmp = Path(tempfile.mkdtemp(prefix="preprint-"))
74
+ real_home = pathlib.Path.home
75
+ try:
76
+ import preprint_check as pc
77
+ pc._get = fake_get
78
+
79
+ print("1. the check")
80
+ r = pc.check("10.20944/preprints202607.0102.v1", title="A test paper on boundaries")
81
+ check(r["verified"] and r["type"] == "posted-content/preprint" and not r["flags"], "Preprints.org preprint: accepted, clean")
82
+ r = pc.check("10.1101/2024.09.14.613029", title="A test paper on boundaries")
83
+ check(r["server"] == "bioRxiv" and any("title differs" in f for f in r["flags"]), "bioRxiv preprint with another title: accepted, flagged")
84
+ for d, why in (("10.1038/nature12373", "a journal article"), ("10.1364/opticaopen.29459153.v1", "a preprint already published"),
85
+ ("10.9999/nope", "a DOI that does not exist")):
86
+ try:
87
+ pc.check(d)
88
+ check(False, f"refused: {why}")
89
+ except pc.Refused:
90
+ check(True, f"refused: {why}")
91
+ r = pc.check("10.48550/arXiv.2401.12345")
92
+ check(r["verified"] and r["type"] == "Preprint" and not r["flags"], "arXiv (DataCite Preprint): accepted, clean")
93
+ r = pc.check("10.5281/zenodo.1")
94
+ check(r["server"] == "Zenodo" and any("JournalArticle" in f for f in r["flags"]), "Zenodo filed as a journal article: accepted, flagged")
95
+ r = pc.check("10.51094/jxiv.1")
96
+ check(r["ra"] == "JaLC" and not r["verified"] and r["flags"], "another agency (JaLC): accepted, flagged")
97
+ NETWORK_DOWN["on"] = True
98
+ r = pc.check("10.20944/preprints202607.0102.v1")
99
+ NETWORK_DOWN["on"] = False
100
+ check(not r["verified"] and any("could not be reached" in f for f in r["flags"]), "registries down: fails open, flagged")
101
+ check(pc.normalize("https://doi.org/10.1101/2024.09.14.613029") == "10.1101/2024.09.14.613029", "resolver URL normalised")
102
+ check(pc.normalize("arXiv:2401.12345v3") == "10.48550/arXiv.2401.12345", "arXiv: prefix and version normalised")
103
+ check(pc.normalize("not a doi") is None, "garbage refused")
104
+
105
+ print("2. /api/submit (T2, signed like the proxy)")
106
+ import httpx
107
+ from fastapi.testclient import TestClient
108
+ from intake import intake_server as iss
109
+ iss.HMAC_SECRET = b"preprint-test-secret"
110
+ iss.TEST_ORCID_WHITELIST = frozenset({TEST_ORCID})
111
+ subs = tmp / "subs"
112
+ iss.SUBMISSIONS_ROOT = subs
113
+ iss.QUEUE_DIR = subs / "queue"
114
+ iss.COUNTER_FILE = subs / ".counter"
115
+ iss.TEST_SUBMISSIONS_ROOT = subs / "test"
116
+ iss.TEST_QUEUE_DIR = subs / "test" / "queue"
117
+ iss._audit_append = lambda entry, test_mode=False: None
118
+ _n = iter(range(1791000000, 1791000100))
119
+ iss._allocate_test_sub_id = lambda: f"ICSAC-SUB-TEST-{next(_n)}"
120
+ client = TestClient(iss.app)
121
+ pdf_bytes = PAPER_PDF.read_bytes()
122
+
123
+ def post(data: dict, with_pdf: bool) -> tuple[int, dict]:
124
+ req = httpx.Request("POST", "http://intake/api/submit", data=data,
125
+ files={"pdf": ("paper.pdf", pdf_bytes, "application/pdf")} if with_pdf else None)
126
+ body = req.read()
127
+ ts = str(int(time.time()))
128
+ sig = hmac.new(iss.HMAC_SECRET, f"{ts}.".encode() + body, hashlib.sha256).hexdigest()
129
+ resp = client.post("/api/submit", content=body, headers={
130
+ "content-type": req.headers["content-type"], "x-icsac-signature": f"sha256={sig}",
131
+ "x-icsac-timestamp": ts, "x-icsac-auth-orcid": TEST_ORCID, "x-icsac-test-tier": "t2"})
132
+ try:
133
+ return resp.status_code, resp.json()
134
+ except Exception:
135
+ return resp.status_code, {"raw": resp.text[:300]}
136
+
137
+ base = {"name": "Test Author", "email": "author@example.com", "orcid": TEST_ORCID, "coi": "true",
138
+ "exclusivity_acknowledged": "true", "code_data_available": "no"}
139
+ upload = dict(base, mode="upload", deposit_consent="true", title="A test paper on boundaries",
140
+ abstract="An abstract long enough to pass the fifty-character minimum for uploads.",
141
+ keywords="testing", license="cc-by-4.0", resource_type="preprint", publication_date="2026-09-28",
142
+ subject="", funding="", creators=json.dumps([{"name": "Test Author", "orcid": TEST_ORCID}]),
143
+ related_identifiers="[]")
144
+
145
+ def record(resp: dict) -> dict:
146
+ sid = resp.get("sub_id") or ""
147
+ return json.loads((subs / "test" / sid / "submission.json").read_text()) if sid else {}
148
+
149
+ def dirs() -> int:
150
+ return len([p for p in (subs / "test").iterdir() if p.name.startswith("ICSAC-SUB-TEST-")]) if (subs / "test").exists() else 0
151
+
152
+ st, resp = post(dict(upload, preprint_doi="https://doi.org/10.20944/preprints202607.0102.v1"), True)
153
+ rec = record(resp)
154
+ check(st in (200, 202) and rec.get("preprint_doi") == "10.20944/preprints202607.0102.v1"
155
+ and (rec.get("preprint_meta") or {}).get("verified"), f"upload + Preprints.org DOI: stored, verified ({st})")
156
+ check(rec.get("form", {}).get("exclusivity_acknowledged") is True, "upload: the exclusivity confirmation is stored")
157
+ up_rec = rec
158
+ st, resp = post(dict(upload), True)
159
+ check(st in (200, 202) and record(resp).get("preprint_doi") is None, "upload without a preprint: none stored")
160
+ before = dirs()
161
+ st, resp = post(dict(upload, preprint_doi="10.1038/nature12373"), True)
162
+ check(st == 422 and resp.get("error") == "preprint_doi_refused" and "Nature" in json.dumps(resp),
163
+ f"upload + a journal article DOI: refused with the journal named ({st})")
164
+ check(dirs() == before, "a refused preprint leaves no submission behind")
165
+ st, resp = post(dict(upload, preprint_doi="banana"), True)
166
+ check(st == 422 and resp.get("error") == "preprint_doi_invalid", f"upload + garbage: refused ({st})")
167
+ st, resp = post(dict(base, mode="doi", doi="10.5281/zenodo.1"), False)
168
+ check(st in (200, 202) and record(resp).get("form", {}).get("exclusivity_acknowledged") is True,
169
+ f"DOI route: the exclusivity confirmation is stored ({st})")
170
+ st, resp = post(dict(base, mode="doi", doi="10.1101/2024.09.14.613029"), False)
171
+ check(st == 422 and "submit/upload?preprint_doi=10.1101/2024.09.14.613029" in json.dumps(resp),
172
+ f"DOI route + a bioRxiv DOI: sent to the upload form with the DOI ({st})")
173
+
174
+ print("3. the deposit and the emails")
175
+ import crossref_deposit as cd
176
+ check(cd.preprint_doi(up_rec) == "10.20944/preprints202607.0102.v1", "the pipeline reads the upload route's preprint")
177
+ sub = dict(up_rec, sub_id="ICSAC-SUB-00099")
178
+ xml = cd.build_deposit_xml(sub, doi="10.67697/icsac.2026.099", landing_url="https://icsacinstitute.org/publications/t",
179
+ pdf_url=None, content_type="journal-article")
180
+ cd.validate_xml(xml)
181
+ check(b'relationship-type="hasPreprint"' in xml and b"10.20944/preprints202607.0102.v1" in xml,
182
+ "upload + preprint: the deposit validates and links the preprint")
183
+ from intake import notify_author as na
184
+ pathlib.Path.home = classmethod(lambda cls: tmp / "home")
185
+ common = dict(to="author@example.com", title="T", author_name="A", verdict="accept", panel_report_md="",
186
+ rqc_md="", tier=2, compaction_manifest={"_failure": "test"},
187
+ approval_url="https://icsacinstitute.org/approve/?t=X", objection_deadline="October 5, 2026")
188
+ out = tmp / "home" / "icsac-submissions" / "test" / "_outbox"
189
+
190
+ def body(sid: str) -> str:
191
+ p = out / f"{sid}.eml"
192
+ if not p.exists():
193
+ return ""
194
+ m = _email.message_from_bytes(p.read_bytes(), policy=_policy.default)
195
+ return "".join(x.get_payload(decode=True).decode() for x in m.walk() if x.get_content_type() == "text/plain")
196
+
197
+ na.send_decision(sub_id="ICSAC-SUB-TEST-A", source="upload", source_ref="paper.pdf",
198
+ preprint_doi="10.20944/preprints202607.0102.v1", exclusivity_confirmed=True, **common)
199
+ t = body("ICSAC-SUB-TEST-A")
200
+ check("keeps its own DOI (10.20944/preprints202607.0102.v1)" in t and "not under review elsewhere, as confirmed" in t
201
+ and "has no other DOI" not in t, "upload + preprint: the email names the preprint and the confirmation")
202
+ na.send_decision(sub_id="ICSAC-SUB-TEST-B", source="doi", source_ref="10.5281/zenodo.1",
203
+ preprint_doi="10.5281/zenodo.1", exclusivity_confirmed=False, **common)
204
+ t = body("ICSAC-SUB-TEST-B")
205
+ check("keeps its own DOI" in t and "as confirmed at submission" not in t,
206
+ "a paper whose confirmation was never recorded is not told it confirmed")
207
+ pathlib.Path.home = real_home
208
+ src = (ROOT / "intake" / "apply_decision.py").read_text()
209
+ check("exclusivity_confirmed=form.get(\"exclusivity_acknowledged\") is True" in src, "the accept passes the recorded confirmation")
210
+ finally:
211
+ pathlib.Path.home = real_home
212
+ shutil.rmtree(tmp, ignore_errors=True)
213
+
214
+ print(f"\n{'ALL GREEN' if not failures else f'{len(failures)} FAILURE(S)'}")
215
+ sys.exit(1 if failures else 0)
preprint_check.py ADDED
@@ -0,0 +1,134 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Preprint DOI check (policy 2026-09-28).
2
+
3
+ A paper already posted as a preprint may be submitted: the preprint keeps its
4
+ DOI and the Persistence article gets its own under the Institute prefix, linked
5
+ to it (Crossref hasPreprint). Any preprint server counts. The check asks doi.org
6
+ which registration agency holds the DOI, then reads that agency's record:
7
+
8
+ Crossref preprints are type "posted-content", subtype "preprint", on every
9
+ server checked on 2026-09-28 (Preprints.org, bioRxiv/medRxiv, SSRN,
10
+ Research Square, ChemRxiv, TechRxiv, Authorea, the OSF family,
11
+ EarthArXiv, engrXiv, Cambridge Open Engage, ESSOAr, JMIR Preprints,
12
+ Beilstein Archives). A journal article, proceedings paper or book
13
+ record is REFUSED: that work is published. So is a preprint whose
14
+ record says it "is-preprint-of" a published version. Some prefixes
15
+ (SSRN, JMIR, SciELO, Wiley) hold both, so the record decides, not
16
+ the prefix.
17
+ DataCite arXiv and Zenodo preprints are resourceTypeGeneral "Preprint";
18
+ other types are accepted and flagged for the curation team
19
+ (Zenodo authors often file a preprint as "Journal article").
20
+ other RAs accepted and flagged (the record was not read).
21
+
22
+ A DOI that does not exist is refused. A registry that cannot be reached fails
23
+ OPEN: accepted, marked unverified and flagged; the curation team checks by hand.
24
+ """
25
+ from __future__ import annotations
26
+
27
+ import json
28
+ import re
29
+ import urllib.error
30
+ import urllib.parse
31
+ import urllib.request
32
+
33
+ USER_AGENT = "ICSAC-intake/1.0 (mailto:help@icsacinstitute.org)"
34
+ _ARXIV_ID = re.compile(r"^(?:arxiv:)?(?:10\.48550/arxiv\.)?(\d{4}\.\d{4,5})(?:v\d+)?$", re.I)
35
+ _DOI = re.compile(r"^10\.\d{4,9}/\S+$")
36
+ PUBLISHED_TYPES = {
37
+ "journal-article", "proceedings-article", "book-chapter", "book", "monograph",
38
+ "edited-book", "reference-book", "book-part", "book-section", "book-track",
39
+ "reference-entry",
40
+ }
41
+
42
+
43
+ class Refused(ValueError):
44
+ """The DOI cannot be accepted as a preprint. The message is author-facing."""
45
+
46
+
47
+ def normalize(raw) -> str | None:
48
+ """A bare DOI (arXiv ids become arXiv's 10.48550 DOI, version dropped), or None."""
49
+ s = (raw or "").strip() if isinstance(raw, str) else ""
50
+ s = re.sub(r"^(?:https?://(?:dx\.)?doi\.org/|doi:\s*)", "", s, flags=re.I).strip()
51
+ m = _ARXIV_ID.match(s)
52
+ if m:
53
+ return f"10.48550/arXiv.{m.group(1)}"
54
+ return s if _DOI.match(s) and len(s) <= 300 else None
55
+
56
+
57
+ def _get(url: str, timeout: float):
58
+ req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT,
59
+ "Accept": "application/json"})
60
+ with urllib.request.urlopen(req, timeout=timeout) as r:
61
+ return json.load(r)
62
+
63
+
64
+ def _words(text) -> set[str]:
65
+ return set(re.findall(r"[a-z0-9]{4,}", (text or "").lower()))
66
+
67
+
68
+ def title_differs(a, b) -> bool:
69
+ """True when two titles share under 40 % of their longer words."""
70
+ wa, wb = _words(a), _words(b)
71
+ if not wa or not wb:
72
+ return False
73
+ return len(wa & wb) / len(wa | wb) < 0.4
74
+
75
+
76
+ def check(doi: str, *, title: str = "", timeout: float = 10.0) -> dict:
77
+ """Read the preprint's record. Returns {doi, ra, type, server, title,
78
+ verified, flags}; raises Refused when the DOI cannot be a preprint."""
79
+ out: dict = {"doi": doi, "ra": None, "type": None, "server": None, "title": None,
80
+ "verified": False, "flags": []}
81
+ q = urllib.parse.quote(doi, safe="/")
82
+ try:
83
+ rows = _get(f"https://doi.org/ra/{q}", timeout)
84
+ except Exception as exc:
85
+ out["flags"].append(f"the DOI registries could not be reached ({type(exc).__name__}); "
86
+ f"check the preprint by hand")
87
+ return out
88
+ row = rows[0] if isinstance(rows, list) and rows else {}
89
+ ra = row.get("RA") or ""
90
+ if not ra:
91
+ raise Refused(f"The preprint DOI {doi} does not exist. Check it, or leave the field empty.")
92
+ out["ra"] = ra
93
+ try:
94
+ if ra == "Crossref":
95
+ m = _get(f"https://api.crossref.org/works/{q}", timeout)["message"]
96
+ typ = m.get("type") or ""
97
+ sub = m.get("subtype")
98
+ out["type"] = f"{typ}/{sub}" if sub else typ
99
+ out["title"] = (m.get("title") or [None])[0]
100
+ inst = m.get("institution") or []
101
+ out["server"] = (inst[0].get("name") if inst else None) or m.get("publisher")
102
+ if typ in PUBLISHED_TYPES:
103
+ where = (m.get("container-title") or [None])[0] or m.get("publisher") or "a journal"
104
+ raise Refused(
105
+ f"The DOI {doi} is a published {typ.replace('-', ' ')} ({where}), not a preprint. "
106
+ f"ICSAC publishes work that has not been published elsewhere; a preprint of it is fine.")
107
+ published = (m.get("relation") or {}).get("is-preprint-of") or []
108
+ if published:
109
+ raise Refused(
110
+ f"The preprint {doi} is already linked to a published version "
111
+ f"({published[0].get('id')}). ICSAC publishes work that has not been published elsewhere.")
112
+ if typ != "posted-content":
113
+ out["flags"].append(f"Crossref type {typ}, not a preprint record")
114
+ elif sub and sub != "preprint":
115
+ out["flags"].append(f"posted content of subtype {sub}, not a preprint")
116
+ elif ra == "DataCite":
117
+ a = _get(f"https://api.datacite.org/dois/{q}", timeout)["data"]["attributes"]
118
+ rtg = (a.get("types") or {}).get("resourceTypeGeneral")
119
+ pub = a.get("publisher")
120
+ out["type"] = rtg
121
+ out["server"] = pub.get("name") if isinstance(pub, dict) else pub
122
+ out["title"] = ((a.get("titles") or [{}])[0]).get("title")
123
+ if rtg != "Preprint":
124
+ out["flags"].append(f"DataCite type {rtg}, not marked as a preprint")
125
+ else:
126
+ out["flags"].append(f"registered with {ra}; the record was not read")
127
+ out["verified"] = ra in ("Crossref", "DataCite")
128
+ except Refused:
129
+ raise
130
+ except Exception as exc:
131
+ out["flags"].append(f"the {ra} record could not be read ({type(exc).__name__}); check it by hand")
132
+ if title and out["title"] and title_differs(title, out["title"]):
133
+ out["flags"].append(f"the preprint's title differs: {out['title'][:150]!r}")
134
+ return out