Release: citation evidence, panel providers, decision letters
Browse files- Citation verification treats a reference cited by URL as a web source, not
a registry match. Only an identity-confirmed reference can ground an
integrity finding, and an empty citation-results block tells the panel
that nothing was checked.
- Workers AI runs as an OpenAI-compatible panel provider, metered against its
free daily allocation, with a usage guard per provider. The panel roster
and its fallback routes are updated, and the example config matches the
live roster.
- A replay harness runs real submissions through intake up to the panel and
grades each stage against an answer key.
- Keywords reach the public record through every registry writer.
- Curator notifications go out without link previews, and test fixtures use
Crossref's test prefix.
- Decision letters name the removed references list by its size and name the
earlier submission a revision answers. The HTML part keeps the templates'
line breaks, and revise letters describe what a resubmission's review sees.
- Comments, prompt examples, rubrics and test fixtures describe defect
classes and the deployment generically.
- README.md +5 -1
- action.py +5 -1
- arxiv_gate.py +2 -2
- citation_misattribution.py +9 -2
- citation_verify.py +113 -15
- citestamp_check.py +6 -0
- config.example.py +70 -16
- crossref_deposit.py +7 -3
- email_send.py +6 -2
- intake/apply_decision.py +1 -0
- intake/intake_server.py +6 -2
- intake/newsletter.py +1 -2
- intake/notify_author.py +7 -0
- intake/scripts/replay.py +429 -0
- intake/scripts/sweep-test-tiers.sh +2 -2
- intake/scripts/test_audit_20260928.py +5 -5
- intake/scripts/test_audit_20260929.py +1 -1
- intake/scripts/test_citation_identity.py +80 -6
- intake/scripts/test_code_data_link.py +1 -1
- intake/scripts/test_crossref_supplement.py +8 -8
- intake/scripts/test_disclosure_wording.py +37 -0
- intake/scripts/test_doi_hygiene.py +107 -0
- intake/scripts/test_doi_license.py +1 -1
- intake/scripts/test_doi_route.py +4 -4
- intake/scripts/test_email_line_breaks.py +59 -0
- intake/scripts/test_evidence_fail_closed.py +156 -0
- intake/scripts/test_keywords_on_record.py +142 -0
- intake/scripts/test_post_publication.py +4 -4
- intake/scripts/test_preprint_doi.py +1 -1
- intake/scripts/test_publish_gate.py +1 -1
- intake/scripts/test_register_resume.py +1 -1
- intake/scripts/test_revision_line.py +75 -0
- intake/scripts/test_rqc_v2.py +2 -2
- intake/scripts/test_workers_ai.py +151 -0
- intake/scripts/test_zenodo_community.py +1 -1
- intake/sponsor_logo.py +2 -3
- intake/submission_worker.py +6 -1
- intake/templates/submission_accept_doi.md +1 -1
- intake/templates/submission_accept_upload.md +1 -1
- intake/templates/submission_accept_upload_pending.md +1 -1
- intake/templates/submission_revise_codelink.md +1 -1
- intake/templates/submission_revise_doi.md +2 -2
- intake/templates/submission_revise_upload.md +2 -2
- intake/templates/submission_scope_reject_doi.md +1 -1
- intake/templates/submission_scope_reject_upload.md +1 -1
- notify.py +4 -0
- publications.py +26 -0
- publish_watcher.py +3 -0
- redaction.py +1 -0
- review.py +155 -5
|
@@ -142,7 +142,11 @@ this part of the pipeline sends an email or registers anything on its own.
|
|
| 142 |
without an answer, then registers the DOI at Crossref, pushes the landing
|
| 143 |
page, publishes the Zenodo archive under the same DOI, and drafts the
|
| 144 |
author's "published" notice. Each step is checkpointed so a failed run
|
| 145 |
-
resumes where it stopped instead of repeating anything.
|
|
|
|
|
|
|
|
|
|
|
|
|
| 146 |
4. **One follow-up** ([`intake/post_publication.py`](intake/post_publication.py)),
|
| 147 |
seven days after registration: a drafted invitation to join the
|
| 148 |
institute's reviewer pool. It is skipped when the author opted out of
|
|
|
|
| 142 |
without an answer, then registers the DOI at Crossref, pushes the landing
|
| 143 |
page, publishes the Zenodo archive under the same DOI, and drafts the
|
| 144 |
author's "published" notice. Each step is checkpointed so a failed run
|
| 145 |
+
resumes where it stopped instead of repeating anything. Until then the
|
| 146 |
+
reserved DOI resolves nowhere: check it at `api.crossref.org/works/<doi>`
|
| 147 |
+
(404 until it is registered), never at doi.org, where a lookup of an
|
| 148 |
+
unregistered DOI counts as a failed resolution in Crossref's monthly
|
| 149 |
+
report for the prefix.
|
| 150 |
4. **One follow-up** ([`intake/post_publication.py`](intake/post_publication.py)),
|
| 151 |
seven days after registration: a drafted invitation to join the
|
| 152 |
institute's reviewer pool. It is skipped when the author opted out of
|
|
@@ -328,7 +328,7 @@ def _extract_registry_entry(record_id: str, metadata: dict,
|
|
| 328 |
last, after = [s.strip() for s in name.split(",", 1)]
|
| 329 |
name = f"{after} {last}".strip() if after else last
|
| 330 |
authors.append(name)
|
| 331 |
-
|
| 332 |
"record_id": str(record_id),
|
| 333 |
"title": m.get("title", "Untitled"),
|
| 334 |
"authors": authors or ["Unknown"],
|
|
@@ -337,6 +337,10 @@ def _extract_registry_entry(record_id: str, metadata: dict,
|
|
| 337 |
"source": source,
|
| 338 |
"source_ref": f"https://zenodo.org/records/{record_id}",
|
| 339 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
| 340 |
|
| 341 |
|
| 342 |
def _publish_public_review(record_id: str) -> str | None:
|
|
|
|
| 328 |
last, after = [s.strip() for s in name.split(",", 1)]
|
| 329 |
name = f"{after} {last}".strip() if after else last
|
| 330 |
authors.append(name)
|
| 331 |
+
proto = {
|
| 332 |
"record_id": str(record_id),
|
| 333 |
"title": m.get("title", "Untitled"),
|
| 334 |
"authors": authors or ["Unknown"],
|
|
|
|
| 337 |
"source": source,
|
| 338 |
"source_ref": f"https://zenodo.org/records/{record_id}",
|
| 339 |
}
|
| 340 |
+
kw = publications.clean_keywords(m.get("keywords"))
|
| 341 |
+
if kw:
|
| 342 |
+
proto["keywords"] = kw
|
| 343 |
+
return proto
|
| 344 |
|
| 345 |
|
| 346 |
def _publish_public_review(record_id: str) -> str | None:
|
|
@@ -3,8 +3,8 @@
|
|
| 3 |
arXiv's terms (https://info.arxiv.org/help/api/tou.html, read 2026-09-29):
|
| 4 |
"make no more than one request every three seconds, and limit requests to a
|
| 5 |
single connection at a time", applied to "all of the machines under your
|
| 6 |
-
control as a whole". Machines that share one public IP share
|
| 7 |
-
|
| 8 |
|
| 9 |
Every call to the API goes through get(): a lock shared by every process and
|
| 10 |
thread on this host holds the single connection, calls start at least
|
|
|
|
| 3 |
arXiv's terms (https://info.arxiv.org/help/api/tou.html, read 2026-09-29):
|
| 4 |
"make no more than one request every three seconds, and limit requests to a
|
| 5 |
single connection at a time", applied to "all of the machines under your
|
| 6 |
+
control as a whole". Machines that share one public IP share that one
|
| 7 |
+
budget.
|
| 8 |
|
| 9 |
Every call to the API goes through get(): a lock shared by every process and
|
| 10 |
thread on this host holds the single connection, calls start at least
|
|
@@ -139,7 +139,11 @@ def select_load_bearing(citations: list[dict], full_text: str, max_n: int = 10)
|
|
| 139 |
"""
|
| 140 |
if not citations:
|
| 141 |
return []
|
| 142 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 143 |
if not eligible:
|
| 144 |
return []
|
| 145 |
if len(eligible) <= max_n:
|
|
@@ -529,7 +533,10 @@ def _short_label(c: dict) -> str:
|
|
| 529 |
else:
|
| 530 |
base = f"{authors[0]} et al."
|
| 531 |
if year:
|
| 532 |
-
|
|
|
|
|
|
|
|
|
|
| 533 |
return base
|
| 534 |
if c.get("title"):
|
| 535 |
t = c["title"]
|
|
|
|
| 139 |
"""
|
| 140 |
if not citations:
|
| 141 |
return []
|
| 142 |
+
# Only a reference whose identity is confirmed by its own identifier can be
|
| 143 |
+
# judged against the claim: a text-search match may be another work, and a
|
| 144 |
+
# verdict on it would be a verdict on the wrong paper.
|
| 145 |
+
import citation_verify
|
| 146 |
+
eligible = [c for c in citations if citation_verify.is_identity_confirmed(c)]
|
| 147 |
if not eligible:
|
| 148 |
return []
|
| 149 |
if len(eligible) <= max_n:
|
|
|
|
| 533 |
else:
|
| 534 |
base = f"{authors[0]} et al."
|
| 535 |
if year:
|
| 536 |
+
# "2019a" / "2019b": two works by the same authors in one year
|
| 537 |
+
# must not share a label in the report.
|
| 538 |
+
m = re.search(rf"\b{year}([a-z])\b", c.get("raw") or "")
|
| 539 |
+
return f"{base} {year}{m.group(1) if m else ''}"
|
| 540 |
return base
|
| 541 |
if c.get("title"):
|
| 542 |
t = c["title"]
|
|
@@ -653,6 +653,12 @@ def _normalize_for_match(s: str) -> str:
|
|
| 653 |
return " ".join(s.split())
|
| 654 |
|
| 655 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 656 |
def _title_matches(claimed: str | None, resolved: str) -> bool:
|
| 657 |
if not claimed or not resolved:
|
| 658 |
return False
|
|
@@ -662,6 +668,13 @@ def _title_matches(claimed: str | None, resolved: str) -> bool:
|
|
| 662 |
return False
|
| 663 |
if a == b:
|
| 664 |
return True
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 665 |
# Substring match in either direction (handles subtitle truncation)
|
| 666 |
if len(a) >= 20 and a in b:
|
| 667 |
return True
|
|
@@ -717,7 +730,47 @@ def _find_probable_record(c: dict) -> dict | None:
|
|
| 717 |
return candidates[0] if candidates else None
|
| 718 |
|
| 719 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 720 |
def verify_citation(c: dict) -> dict:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 721 |
"""Single citation lookup — arXiv → Crossref/DataCite → Semantic Scholar.
|
| 722 |
|
| 723 |
Order matters. Exact-id resolvers (arXiv ID, DOI) get exact-id
|
|
@@ -978,10 +1031,12 @@ def verify_all(citations: list[dict], max_concurrent: int = 8) -> list[dict]:
|
|
| 978 |
# expect the registry title there); keep what the paper cited.
|
| 979 |
if "cited_title" not in merged:
|
| 980 |
merged["cited_title"] = citations[i].get("title")
|
| 981 |
-
if merged.get("verified") and not v.get("verified")
|
|
|
|
| 982 |
# A recheck that could not confirm never downgrades a citation
|
| 983 |
# verified earlier (audit 2026-09-28 item 17) -- unless the
|
| 984 |
-
# citation's own DOI now disagrees
|
|
|
|
| 985 |
merged["recheck_reason"] = str(v.get("reason", ""))[:200]
|
| 986 |
else:
|
| 987 |
merged.update(v)
|
|
@@ -999,6 +1054,23 @@ def identity_counts(citations: list[dict]) -> dict:
|
|
| 999 |
return counts
|
| 1000 |
|
| 1001 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1002 |
def build_verification_report(citations: list[dict]) -> str:
|
| 1003 |
"""Render the verification report as a markdown block suitable for
|
| 1004 |
prompt injection above the DEFENSIVE_PREAMBLE + submission block.
|
|
@@ -1006,10 +1078,11 @@ def build_verification_report(citations: list[dict]) -> str:
|
|
| 1006 |
if not citations:
|
| 1007 |
return ""
|
| 1008 |
|
| 1009 |
-
confirmed = [c for c in citations if
|
| 1010 |
-
|
| 1011 |
disagree = [c for c in citations if not c.get("verified") and c.get("doi_identity") in ("mismatch", "dead")]
|
| 1012 |
-
|
|
|
|
| 1013 |
|
| 1014 |
lines = [
|
| 1015 |
"## Citation verification (independently verified before review)",
|
|
@@ -1021,12 +1094,13 @@ def build_verification_report(citations: list[dict]) -> str:
|
|
| 1021 |
"- CONFIRMED [exact-id]: the cited identifier (DOI or arXiv ID) resolves and",
|
| 1022 |
" its record matches the cited title (or the authors and year when no title",
|
| 1023 |
" was cited). The work exists as cited. Do not call it fabricated.",
|
| 1024 |
-
"-
|
| 1025 |
-
"
|
| 1026 |
-
"
|
| 1027 |
-
" work
|
| 1028 |
-
"
|
| 1029 |
-
" the
|
|
|
|
| 1030 |
"- DOI MISMATCH: the citation carries a DOI that resolves to a DIFFERENT work.",
|
| 1031 |
" DOI DEAD: the cited DOI resolves nowhere. Both are citation-integrity",
|
| 1032 |
" defects in their own right (a wrong identifier, wrong authors, or a",
|
|
@@ -1035,6 +1109,10 @@ def build_verification_report(citations: list[dict]) -> str:
|
|
| 1035 |
" matching the cited title exists it is given, so the author can be pointed",
|
| 1036 |
" to it.",
|
| 1037 |
"- UNVERIFIABLE: no registry record was found. Not evidence of fabrication.",
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1038 |
"",
|
| 1039 |
]
|
| 1040 |
|
|
@@ -1058,10 +1136,14 @@ def build_verification_report(citations: list[dict]) -> str:
|
|
| 1058 |
lines.extend(_line(c, "CONFIRMED") for c in confirmed)
|
| 1059 |
lines.append("")
|
| 1060 |
|
| 1061 |
-
if
|
| 1062 |
-
lines.append("###
|
| 1063 |
lines.append("")
|
| 1064 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1065 |
lines.append("")
|
| 1066 |
|
| 1067 |
if disagree:
|
|
@@ -1077,6 +1159,17 @@ def build_verification_report(citations: list[dict]) -> str:
|
|
| 1077 |
lines.append(f"{head} {c.get('reason') or ''}".rstrip())
|
| 1078 |
lines.append("")
|
| 1079 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1080 |
if unverifiable:
|
| 1081 |
lines.append("### Unverifiable from public registries")
|
| 1082 |
lines.append("")
|
|
@@ -1111,7 +1204,10 @@ def _short_label(c: dict) -> str:
|
|
| 1111 |
else:
|
| 1112 |
base = f"{authors[0]} et al."
|
| 1113 |
if year:
|
| 1114 |
-
|
|
|
|
|
|
|
|
|
|
| 1115 |
return base
|
| 1116 |
if c.get("title"):
|
| 1117 |
t = c["title"]
|
|
@@ -1141,6 +1237,8 @@ def save_citation_report(record_id: str, citations: list[dict], report: str) ->
|
|
| 1141 |
"citation_count": len(citations),
|
| 1142 |
"verified_count": sum(1 for c in citations if c.get("verified")),
|
| 1143 |
"unverifiable_count": sum(1 for c in citations if not c.get("verified")),
|
|
|
|
|
|
|
| 1144 |
"citations": citations,
|
| 1145 |
}
|
| 1146 |
with open(json_path, "w") as f:
|
|
|
|
| 653 |
return " ".join(s.split())
|
| 654 |
|
| 655 |
|
| 656 |
+
def _main_title(s: str) -> str:
|
| 657 |
+
"""The title before its subtitle ('Spin Glasses: Experimental Facts…'
|
| 658 |
+
-> 'spin glasses'), normalized."""
|
| 659 |
+
return _normalize_for_match(re.split(r"\s*[:—–?]\s*|\s+-\s+", s or "", maxsplit=1)[0])
|
| 660 |
+
|
| 661 |
+
|
| 662 |
def _title_matches(claimed: str | None, resolved: str) -> bool:
|
| 663 |
if not claimed or not resolved:
|
| 664 |
return False
|
|
|
|
| 668 |
return False
|
| 669 |
if a == b:
|
| 670 |
return True
|
| 671 |
+
# A title of three words or fewer matches only the other side's main
|
| 672 |
+
# title. Word overlap let a two-word title ("Data Policy") match any
|
| 673 |
+
# longer title that happens to contain both words.
|
| 674 |
+
if len(a.split()) < 4:
|
| 675 |
+
return a == _main_title(resolved)
|
| 676 |
+
if len(b.split()) < 4:
|
| 677 |
+
return b == _main_title(claimed)
|
| 678 |
# Substring match in either direction (handles subtitle truncation)
|
| 679 |
if len(a) >= 20 and a in b:
|
| 680 |
return True
|
|
|
|
| 730 |
return candidates[0] if candidates else None
|
| 731 |
|
| 732 |
|
| 733 |
+
_URL_RE = re.compile(r"https?://[^\s<>\"]+", re.I)
|
| 734 |
+
_REGISTRY_URL_RE = re.compile(r"https?://(?:dx\.)?doi\.org/|https?://(?:export\.)?arxiv\.org/", re.I)
|
| 735 |
+
|
| 736 |
+
|
| 737 |
+
def _cited_web_url(c: dict) -> str | None:
|
| 738 |
+
"""The URL a reference gives as its handle when the extractor typed it a
|
| 739 |
+
web source ("url") and it carries no DOI or arXiv ID; else None."""
|
| 740 |
+
if c.get("type") != "url" or c.get("doi") or c.get("arxiv_id"):
|
| 741 |
+
return None
|
| 742 |
+
for m in _URL_RE.finditer(c.get("raw") or ""):
|
| 743 |
+
url = m.group(0).rstrip(".,;)")
|
| 744 |
+
if not _REGISTRY_URL_RE.match(url):
|
| 745 |
+
return url
|
| 746 |
+
return None
|
| 747 |
+
|
| 748 |
+
|
| 749 |
def verify_citation(c: dict) -> dict:
|
| 750 |
+
"""A reference cited by URL (a policy page, documentation) is a web
|
| 751 |
+
source: the URL is its identifier, as a DOI is a paper's. The registry
|
| 752 |
+
searches may still find it as a scholarly work, but only on title AND
|
| 753 |
+
author; anything weaker is a neighbouring record, so the reference is
|
| 754 |
+
reported as cited by URL and nothing is matched to it. Otherwise a short
|
| 755 |
+
page title matches an unrelated paper whose title contains the same
|
| 756 |
+
words, and the panel judges that paper against the claim."""
|
| 757 |
+
out = _verify_against_registries(c)
|
| 758 |
+
url = _cited_web_url(c)
|
| 759 |
+
if url and out.get("confidence") != "title-author-match":
|
| 760 |
+
return {
|
| 761 |
+
"verified": False,
|
| 762 |
+
"resolver": "url",
|
| 763 |
+
"resolved_id": url,
|
| 764 |
+
"title": c.get("title") or "",
|
| 765 |
+
"abstract": "",
|
| 766 |
+
"confidence": "web-source",
|
| 767 |
+
"reason": "Cited by URL: a web page, not a registry work. Not matched to any registry "
|
| 768 |
+
"record and not opened by the verifier.",
|
| 769 |
+
}
|
| 770 |
+
return out
|
| 771 |
+
|
| 772 |
+
|
| 773 |
+
def _verify_against_registries(c: dict) -> dict:
|
| 774 |
"""Single citation lookup — arXiv → Crossref/DataCite → Semantic Scholar.
|
| 775 |
|
| 776 |
Order matters. Exact-id resolvers (arXiv ID, DOI) get exact-id
|
|
|
|
| 1031 |
# expect the registry title there); keep what the paper cited.
|
| 1032 |
if "cited_title" not in merged:
|
| 1033 |
merged["cited_title"] = citations[i].get("title")
|
| 1034 |
+
if (merged.get("verified") and not v.get("verified")
|
| 1035 |
+
and v.get("doi_identity") not in ("dead", "mismatch") and v.get("confidence") != "web-source"):
|
| 1036 |
# A recheck that could not confirm never downgrades a citation
|
| 1037 |
# verified earlier (audit 2026-09-28 item 17) -- unless the
|
| 1038 |
+
# citation's own DOI now disagrees, or it is a web page an
|
| 1039 |
+
# earlier fuzzy search matched to a paper: findings, not misses.
|
| 1040 |
merged["recheck_reason"] = str(v.get("reason", ""))[:200]
|
| 1041 |
else:
|
| 1042 |
merged.update(v)
|
|
|
|
| 1054 |
return counts
|
| 1055 |
|
| 1056 |
|
| 1057 |
+
def is_identity_confirmed(c: dict) -> bool:
|
| 1058 |
+
"""CONFIRMED: the reference cites an identifier (DOI or arXiv ID) that resolves
|
| 1059 |
+
to the cited work. Only this level can ground an integrity finding -- a
|
| 1060 |
+
misattribution verdict, a citation-graph refutation, a deposited citation
|
| 1061 |
+
link: a work nobody is sure is the cited one cannot be judged against a claim."""
|
| 1062 |
+
return bool(c.get("verified")) and c.get("confidence") == "exact-id"
|
| 1063 |
+
|
| 1064 |
+
|
| 1065 |
+
def is_unconfirmed_match(c: dict) -> bool:
|
| 1066 |
+
"""A registry record found by text search for a reference that cites no
|
| 1067 |
+
identifier. It may be the cited work or a neighbour of it (a review of the
|
| 1068 |
+
book, one of its chapters, another paper with the same words in its title),
|
| 1069 |
+
so it is UNCONFIRMED: kept in the JSON for the curator, shown to no
|
| 1070 |
+
reviewer, never evidence for or against the paper."""
|
| 1071 |
+
return bool(c.get("verified")) and c.get("confidence") not in ("exact-id", "web-source")
|
| 1072 |
+
|
| 1073 |
+
|
| 1074 |
def build_verification_report(citations: list[dict]) -> str:
|
| 1075 |
"""Render the verification report as a markdown block suitable for
|
| 1076 |
prompt injection above the DEFENSIVE_PREAMBLE + submission block.
|
|
|
|
| 1078 |
if not citations:
|
| 1079 |
return ""
|
| 1080 |
|
| 1081 |
+
confirmed = [c for c in citations if is_identity_confirmed(c)]
|
| 1082 |
+
unconfirmed = [c for c in citations if is_unconfirmed_match(c)]
|
| 1083 |
disagree = [c for c in citations if not c.get("verified") and c.get("doi_identity") in ("mismatch", "dead")]
|
| 1084 |
+
web = [c for c in citations if c.get("confidence") == "web-source"]
|
| 1085 |
+
unverifiable = [c for c in citations if not c.get("verified") and c not in disagree and c not in web]
|
| 1086 |
|
| 1087 |
lines = [
|
| 1088 |
"## Citation verification (independently verified before review)",
|
|
|
|
| 1094 |
"- CONFIRMED [exact-id]: the cited identifier (DOI or arXiv ID) resolves and",
|
| 1095 |
" its record matches the cited title (or the authors and year when no title",
|
| 1096 |
" was cited). The work exists as cited. Do not call it fabricated.",
|
| 1097 |
+
"- UNCONFIRMED: no identifier was cited, so the reference's identity could",
|
| 1098 |
+
" not be confirmed. A text search of the registries may have found a",
|
| 1099 |
+
" possible record; it is deliberately not shown, because it may be a",
|
| 1100 |
+
" different work (a review of the book, a chapter, a paper with similar",
|
| 1101 |
+
" words in its title). UNCONFIRMED is not evidence of anything: do not call",
|
| 1102 |
+
" the reference fabricated, misattributed or citation stuffing on its",
|
| 1103 |
+
" account, and do not credit it as verified either. Judge it from the paper.",
|
| 1104 |
"- DOI MISMATCH: the citation carries a DOI that resolves to a DIFFERENT work.",
|
| 1105 |
" DOI DEAD: the cited DOI resolves nowhere. Both are citation-integrity",
|
| 1106 |
" defects in their own right (a wrong identifier, wrong authors, or a",
|
|
|
|
| 1109 |
" matching the cited title exists it is given, so the author can be pointed",
|
| 1110 |
" to it.",
|
| 1111 |
"- UNVERIFIABLE: no registry record was found. Not evidence of fabrication.",
|
| 1112 |
+
"- WEB SOURCE: the reference is a web page cited by its URL (a policy page,",
|
| 1113 |
+
" documentation). It is not a registry work, so no registry record was",
|
| 1114 |
+
" matched to it and the verifier did not open it. Not evidence of",
|
| 1115 |
+
" fabrication or of misattribution; the page is the one at that URL.",
|
| 1116 |
"",
|
| 1117 |
]
|
| 1118 |
|
|
|
|
| 1136 |
lines.extend(_line(c, "CONFIRMED") for c in confirmed)
|
| 1137 |
lines.append("")
|
| 1138 |
|
| 1139 |
+
if unconfirmed:
|
| 1140 |
+
lines.append("### Unconfirmed: no identifier cited (not evidence of anything)")
|
| 1141 |
lines.append("")
|
| 1142 |
+
for c in unconfirmed:
|
| 1143 |
+
claim = c.get("claim_context") or ""
|
| 1144 |
+
tail = f" Submission claim context: \"{claim}\"" if claim else ""
|
| 1145 |
+
lines.append(f"- **{_short_label(c)}** — UNCONFIRMED. No identifier was cited; "
|
| 1146 |
+
f"not checked against any record.{tail}")
|
| 1147 |
lines.append("")
|
| 1148 |
|
| 1149 |
if disagree:
|
|
|
|
| 1159 |
lines.append(f"{head} {c.get('reason') or ''}".rstrip())
|
| 1160 |
lines.append("")
|
| 1161 |
|
| 1162 |
+
if web:
|
| 1163 |
+
lines.append("### Cited by URL (web pages, not registry works)")
|
| 1164 |
+
lines.append("")
|
| 1165 |
+
for c in web:
|
| 1166 |
+
title = c.get("cited_title") or c.get("title") or ""
|
| 1167 |
+
named = f" *{title}*," if title else ""
|
| 1168 |
+
claim = c.get("claim_context") or ""
|
| 1169 |
+
tail = f" Submission claim context: \"{claim}\"" if claim else ""
|
| 1170 |
+
lines.append(f"- **{_short_label(c)}** — WEB SOURCE.{named} {c.get('resolved_id')}.{tail}")
|
| 1171 |
+
lines.append("")
|
| 1172 |
+
|
| 1173 |
if unverifiable:
|
| 1174 |
lines.append("### Unverifiable from public registries")
|
| 1175 |
lines.append("")
|
|
|
|
| 1204 |
else:
|
| 1205 |
base = f"{authors[0]} et al."
|
| 1206 |
if year:
|
| 1207 |
+
# "2019a" / "2019b": two works by the same authors in one year
|
| 1208 |
+
# must not share a label in the report.
|
| 1209 |
+
m = re.search(rf"\b{year}([a-z])\b", c.get("raw") or "")
|
| 1210 |
+
return f"{base} {year}{m.group(1) if m else ''}"
|
| 1211 |
return base
|
| 1212 |
if c.get("title"):
|
| 1213 |
t = c["title"]
|
|
|
|
| 1237 |
"citation_count": len(citations),
|
| 1238 |
"verified_count": sum(1 for c in citations if c.get("verified")),
|
| 1239 |
"unverifiable_count": sum(1 for c in citations if not c.get("verified")),
|
| 1240 |
+
"confirmed_count": sum(1 for c in citations if is_identity_confirmed(c)),
|
| 1241 |
+
"unconfirmed_count": sum(1 for c in citations if is_unconfirmed_match(c)),
|
| 1242 |
"citations": citations,
|
| 1243 |
}
|
| 1244 |
with open(json_path, "w") as f:
|
|
@@ -137,7 +137,13 @@ def check_citations(citations: list[dict], *, log=print) -> dict:
|
|
| 137 |
return out
|
| 138 |
out["available"] = True
|
| 139 |
rpc_id = 10
|
|
|
|
| 140 |
for c in citations:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 141 |
doi = _doi_of(c)
|
| 142 |
if not doi:
|
| 143 |
continue
|
|
|
|
| 137 |
return out
|
| 138 |
out["available"] = True
|
| 139 |
rpc_id = 10
|
| 140 |
+
import citation_verify
|
| 141 |
for c in citations:
|
| 142 |
+
# Only identity-confirmed references: a DOI that points to another work,
|
| 143 |
+
# or a record found by text search, would put that other work's
|
| 144 |
+
# refutations and supports in front of the panel.
|
| 145 |
+
if not citation_verify.is_identity_confirmed(c):
|
| 146 |
+
continue
|
| 147 |
doi = _doi_of(c)
|
| 148 |
if not doi:
|
| 149 |
continue
|
|
@@ -96,6 +96,10 @@ HF_TOKEN = os.environ.get("HF_TOKEN", "")
|
|
| 96 |
# different model on the same provider absorbs it. Same trade-off as 04-27:
|
| 97 |
# a fallback can duplicate another slot's primary; reliability beats diversity.
|
| 98 |
OPENROUTER_MODELS = [
|
|
|
|
|
|
|
|
|
|
|
|
|
| 99 |
# 2026-09-28 (the curator): a free direct provider first wherever one exists, the
|
| 100 |
# HF router second (a $0.10/month credit pool), OpenRouter :free last (a shared
|
| 101 |
# pool that 429s). `oai|<provider>|<model>` entries need the provider's key in
|
|
@@ -105,37 +109,40 @@ OPENROUTER_MODELS = [
|
|
| 105 |
# Slot 1: DeepSeek family.
|
| 106 |
[
|
| 107 |
"oai|sambanova|DeepSeek-V3.1",
|
| 108 |
-
|
| 109 |
-
|
| 110 |
-
|
| 111 |
-
|
|
|
|
|
|
|
| 112 |
"or|google/gemma-4-31b-it:free",
|
| 113 |
],
|
| 114 |
-
# Slot 2: OpenAI open-weights.
|
|
|
|
|
|
|
| 115 |
[
|
|
|
|
| 116 |
"oai|sambanova|gpt-oss-120b",
|
| 117 |
"hf|openai/gpt-oss-120b:deepinfra",
|
| 118 |
"hf|Qwen/Qwen3-235B-A22B-Instruct-2507:deepinfra",
|
| 119 |
-
"or|nvidia/nemotron-3-super-120b-a12b:free",
|
| 120 |
-
"or|qwen/qwen3.8-27b:free",
|
| 121 |
"or|google/gemma-4-26b-a4b-it:free",
|
| 122 |
],
|
| 123 |
-
# Slot 3: Mistral / Qwen family (Mistral activates when MISTRAL_API_KEY exists
|
|
|
|
| 124 |
[
|
| 125 |
-
"oai|mistral|mistral-
|
| 126 |
"hf|Qwen/Qwen3-235B-A22B-Instruct-2507:deepinfra",
|
| 127 |
"hf|google/gemma-4-31B-it:deepinfra",
|
| 128 |
-
"or|nvidia/nemotron-3-ultra-550b-a55b:free",
|
| 129 |
"or|google/gemma-4-26b-a4b-it:free",
|
| 130 |
"or|google/gemma-4-31b-it:free",
|
| 131 |
],
|
| 132 |
-
# Slot 4: Google family.
|
|
|
|
| 133 |
[
|
|
|
|
| 134 |
"hf|google/gemma-4-31B-it:deepinfra",
|
| 135 |
"or|google/gemma-4-31b-it:free",
|
| 136 |
"oai|sambanova|gemma-4-31B-it",
|
| 137 |
-
"or|nvidia/nemotron-3-super-120b-a12b:free",
|
| 138 |
-
"or|qwen/qwen3.8-27b:free",
|
| 139 |
"or|google/gemma-4-26b-a4b-it:free",
|
| 140 |
],
|
| 141 |
]
|
|
@@ -154,10 +161,57 @@ OPENROUTER_MODELS_API_URL = "https://openrouter.ai/api/v1/models"
|
|
| 154 |
# (2026-09-28). Keys come from the environment; an entry whose key is absent is
|
| 155 |
# skipped, not counted as a dead chain link.
|
| 156 |
OAI_COMPAT_PROVIDERS = {
|
| 157 |
-
#
|
|
|
|
| 158 |
"sambanova": {"base_url": "https://api.sambanova.ai/v1", "key_env": "SAMBANOVA_API_KEY"},
|
| 159 |
-
#
|
| 160 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 161 |
}
|
| 162 |
# panel_retry.py (batch-tick step 2b): a paper the panel could not staff is
|
| 163 |
# re-queued every tick and SCREAMS (Telegram + pain) after this many hours of
|
|
|
|
| 96 |
# different model on the same provider absorbs it. Same trade-off as 04-27:
|
| 97 |
# a fallback can duplicate another slot's primary; reliability beats diversity.
|
| 98 |
OPENROUTER_MODELS = [
|
| 99 |
+
# 2026-10-02: no entry may route a manuscript to a provider that trains on
|
| 100 |
+
# prompts. OpenRouter lists its free nemotron routes as training ones, so
|
| 101 |
+
# they are out. The OR free routes kept here are listed as not training;
|
| 102 |
+
# Workers AI does not train; the HF upstreams' own terms are still to check.
|
| 103 |
# 2026-09-28 (the curator): a free direct provider first wherever one exists, the
|
| 104 |
# HF router second (a $0.10/month credit pool), OpenRouter :free last (a shared
|
| 105 |
# pool that 429s). `oai|<provider>|<model>` entries need the provider's key in
|
|
|
|
| 109 |
# Slot 1: DeepSeek family.
|
| 110 |
[
|
| 111 |
"oai|sambanova|DeepSeek-V3.1",
|
| 112 |
+
# 2026-10-12: HF's deepinfra route stopped serving DeepSeek-V4-Flash and
|
| 113 |
+
# GLM-4.7-Flash. GLM-4.7-Flash now comes from Workers AI. Workers AI's own
|
| 114 |
+
# DeepSeek-V4-Flash is left out: on a full panel prompt it spent its whole
|
| 115 |
+
# output on hidden reasoning, and one call cost ~2,600 of the daily neurons
|
| 116 |
+
# the gpt-oss slot also draws on.
|
| 117 |
+
"oai|cloudflare|@cf/zai-org/glm-4.7-flash",
|
| 118 |
"or|google/gemma-4-31b-it:free",
|
| 119 |
],
|
| 120 |
+
# Slot 2: OpenAI open-weights. Workers AI first (2026-10-02): free within
|
| 121 |
+
# the daily allocation, metered by review._neuron_reserve, and it does not
|
| 122 |
+
# train on customer content.
|
| 123 |
[
|
| 124 |
+
"oai|cloudflare|@cf/openai/gpt-oss-120b",
|
| 125 |
"oai|sambanova|gpt-oss-120b",
|
| 126 |
"hf|openai/gpt-oss-120b:deepinfra",
|
| 127 |
"hf|Qwen/Qwen3-235B-A22B-Instruct-2507:deepinfra",
|
|
|
|
|
|
|
| 128 |
"or|google/gemma-4-26b-a4b-it:free",
|
| 129 |
],
|
| 130 |
+
# Slot 3: Mistral / Qwen family (Mistral activates when MISTRAL_API_KEY exists;
|
| 131 |
+
# medium, not large: the key's model list has no large model, 2026-10-02).
|
| 132 |
[
|
| 133 |
+
"oai|mistral|mistral-medium-latest",
|
| 134 |
"hf|Qwen/Qwen3-235B-A22B-Instruct-2507:deepinfra",
|
| 135 |
"hf|google/gemma-4-31B-it:deepinfra",
|
|
|
|
| 136 |
"or|google/gemma-4-26b-a4b-it:free",
|
| 137 |
"or|google/gemma-4-31b-it:free",
|
| 138 |
],
|
| 139 |
+
# Slot 4: Google family. Workers AI first (2026-10-12): free within the
|
| 140 |
+
# daily allocation, no training on customer content.
|
| 141 |
[
|
| 142 |
+
"oai|cloudflare|@cf/google/gemma-4-26b-a4b-it",
|
| 143 |
"hf|google/gemma-4-31B-it:deepinfra",
|
| 144 |
"or|google/gemma-4-31b-it:free",
|
| 145 |
"oai|sambanova|gemma-4-31B-it",
|
|
|
|
|
|
|
| 146 |
"or|google/gemma-4-26b-a4b-it:free",
|
| 147 |
],
|
| 148 |
]
|
|
|
|
| 161 |
# (2026-09-28). Keys come from the environment; an entry whose key is absent is
|
| 162 |
# skipped, not counted as a dead chain link.
|
| 163 |
OAI_COMPAT_PROVIDERS = {
|
| 164 |
+
# no longer free (checked 2026-10-02: a payment method and purchased credits
|
| 165 |
+
# are required); its entries stay inert while no key is set
|
| 166 |
"sambanova": {"base_url": "https://api.sambanova.ai/v1", "key_env": "SAMBANOVA_API_KEY"},
|
| 167 |
+
# A card-backed account (2026-10-02) with a monthly allowance that must never
|
| 168 |
+
# be exceeded, and training on inputs switched off in the account. Every call
|
| 169 |
+
# is metered against monthly_cap_usd (ICSAC_MISTRAL_MONTHLY_CAP_USD, default
|
| 170 |
+
# 8) before it is sent. usd_rates are deliberately ABOVE the list price
|
| 171 |
+
# (mistral-medium-latest = Medium 3.5, listed at $1.5 / $7.5 per million
|
| 172 |
+
# input / output tokens), so the ledger over-counts and a price rise stays
|
| 173 |
+
# inside the margin; a model with no rate here is refused.
|
| 174 |
+
"mistral": {
|
| 175 |
+
"base_url": "https://api.mistral.ai/v1",
|
| 176 |
+
"key_env": "MISTRAL_API_KEY",
|
| 177 |
+
"monthly_cap_usd": float(os.environ.get("ICSAC_MISTRAL_MONTHLY_CAP_USD", "8")),
|
| 178 |
+
"usd_rates": {
|
| 179 |
+
"mistral-medium-latest": (2.0, 10.0),
|
| 180 |
+
},
|
| 181 |
+
},
|
| 182 |
+
# Cloudflare Workers AI (2026-10-02): a free daily allocation of 10,000
|
| 183 |
+
# neurons per account, and customer content is not used for training. On a
|
| 184 |
+
# Workers Paid account usage past the allocation is billed, so every call is
|
| 185 |
+
# metered against daily_neuron_cap before it is sent and refused when it
|
| 186 |
+
# could cross it (review._neuron_reserve). Rates are neurons per million
|
| 187 |
+
# input / output tokens from the Workers AI pricing page; a model with no
|
| 188 |
+
# rate here is refused.
|
| 189 |
+
"cloudflare": {
|
| 190 |
+
"base_url": ("https://api.cloudflare.com/client/v4/accounts/"
|
| 191 |
+
+ os.environ.get("CLOUDFLARE_AI_ACCOUNT_ID", "") + "/ai/v1"),
|
| 192 |
+
"key_env": "CLOUDFLARE_AI_TOKEN",
|
| 193 |
+
"daily_neuron_cap": int(os.environ.get("ICSAC_CF_NEURON_CAP", "9000")),
|
| 194 |
+
"neuron_rates": {
|
| 195 |
+
"@cf/openai/gpt-oss-120b": (31818, 68182),
|
| 196 |
+
"@cf/meta/llama-4-scout-17b-16e-instruct": (24545, 77273),
|
| 197 |
+
"@cf/mistralai/mistral-small-3.1-24b-instruct": (31876, 50488),
|
| 198 |
+
"@cf/google/gemma-3-12b-it": (31371, 50560),
|
| 199 |
+
"@cf/zai-org/glm-4.7-flash": (5500, 36364),
|
| 200 |
+
"@cf/google/gemma-4-26b-a4b-it": (9091, 27273),
|
| 201 |
+
},
|
| 202 |
+
# Request overrides per model (default: max_tokens 4000, no JSON mode).
|
| 203 |
+
# Workers AI counts a reasoning model's hidden reasoning inside
|
| 204 |
+
# max_tokens: at 4000 both of these cut their review JSON off mid-object
|
| 205 |
+
# on a full-length panel prompt. Without JSON mode both wrote LaTeX
|
| 206 |
+
# backslash commands unescaped inside JSON strings, which no parser accepts.
|
| 207 |
+
# In JSON mode Gemma spent all 16000 tokens reasoning and returned no
|
| 208 |
+
# content, so its thinking is off. Measured 2026-10-12.
|
| 209 |
+
"model_params": {
|
| 210 |
+
"@cf/zai-org/glm-4.7-flash": {"max_tokens": 16000, "response_format": {"type": "json_object"}},
|
| 211 |
+
"@cf/google/gemma-4-26b-a4b-it": {"max_tokens": 8000, "response_format": {"type": "json_object"},
|
| 212 |
+
"chat_template_kwargs": {"enable_thinking": False}},
|
| 213 |
+
},
|
| 214 |
+
},
|
| 215 |
}
|
| 216 |
# panel_retry.py (batch-tick step 2b): a paper the panel could not staff is
|
| 217 |
# re-queued every tick and SCREAMS (Telegram + pain) after this many hours of
|
|
@@ -390,15 +390,16 @@ def _doi_data(parent: ET.Element, doi: str, landing: str, pdf_url: Optional[str]
|
|
| 390 |
|
| 391 |
|
| 392 |
def _citations(parent: ET.Element, citations: list[dict]) -> None:
|
| 393 |
-
"""
|
| 394 |
-
|
|
|
|
| 395 |
if not citations:
|
| 396 |
return
|
| 397 |
cl = _sub(parent, "citation_list")
|
| 398 |
for i, c in enumerate(citations, 1):
|
| 399 |
cit = _sub(cl, "citation", attrib={"key": f"ref{i}"})
|
| 400 |
doi = (c.get("doi") or "").strip()
|
| 401 |
-
if doi and c.get("verified"):
|
| 402 |
_sub(cit, "doi", doi)
|
| 403 |
else:
|
| 404 |
raw = " ".join((c.get("raw") or c.get("title") or "").split())
|
|
@@ -1004,6 +1005,9 @@ def _push_publications(sub_dir: Path, submission: dict, doi: str, *, log) -> dic
|
|
| 1004 |
cdu = publications.code_data_url(submission)
|
| 1005 |
if cdu:
|
| 1006 |
proto["code_data_url"] = cdu
|
|
|
|
|
|
|
|
|
|
| 1007 |
pre = preprint_doi(submission)
|
| 1008 |
if pre:
|
| 1009 |
proto["preprint_doi"] = pre # the paper page links the preprint it was submitted from
|
|
|
|
| 390 |
|
| 391 |
|
| 392 |
def _citations(parent: ET.Element, citations: list[dict]) -> None:
|
| 393 |
+
"""DOIs confirmed by the verifier (the cited DOI resolves to the cited work)
|
| 394 |
+
go in as <doi>; the rest as unstructured text. Crossref matches unstructured
|
| 395 |
+
citations itself, so nothing verifiable is lost."""
|
| 396 |
if not citations:
|
| 397 |
return
|
| 398 |
cl = _sub(parent, "citation_list")
|
| 399 |
for i, c in enumerate(citations, 1):
|
| 400 |
cit = _sub(cl, "citation", attrib={"key": f"ref{i}"})
|
| 401 |
doi = (c.get("doi") or "").strip()
|
| 402 |
+
if doi and c.get("verified") and c.get("confidence") == "exact-id":
|
| 403 |
_sub(cit, "doi", doi)
|
| 404 |
else:
|
| 405 |
raw = " ".join((c.get("raw") or c.get("title") or "").split())
|
|
|
|
| 1005 |
cdu = publications.code_data_url(submission)
|
| 1006 |
if cdu:
|
| 1007 |
proto["code_data_url"] = cdu
|
| 1008 |
+
kw = publications.clean_keywords(submission.get("keywords"))
|
| 1009 |
+
if kw:
|
| 1010 |
+
proto["keywords"] = kw
|
| 1011 |
pre = preprint_doi(submission)
|
| 1012 |
if pre:
|
| 1013 |
proto["preprint_doi"] = pre # the paper page links the preprint it was submitted from
|
|
@@ -69,8 +69,12 @@ def _markdown_to_html(md: str) -> str:
|
|
| 69 |
|
| 70 |
The templates carry no raw HTML, so every `<` in the body came from a
|
| 71 |
substituted value (a title, a name, a note). It is rendered as text, not
|
| 72 |
-
as a tag (audit 2026-09-29); the plain-text part is untouched.
|
| 73 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 74 |
inner = _box_response_section(inner)
|
| 75 |
return HTML_WRAPPER.format(logo=LOGO_URL, body=inner)
|
| 76 |
|
|
|
|
| 69 |
|
| 70 |
The templates carry no raw HTML, so every `<` in the body came from a
|
| 71 |
substituted value (a title, a name, a note). It is rendered as text, not
|
| 72 |
+
as a tag (audit 2026-09-29); the plain-text part is untouched.
|
| 73 |
+
|
| 74 |
+
A single line break is a line break (nl2br): every one in the templates
|
| 75 |
+
is meant (the header block, the sign-off), and without it the header
|
| 76 |
+
read as one run-on line in the HTML part (2026-10-16)."""
|
| 77 |
+
inner = markdown.markdown(md.replace("<", "<"), extensions=["extra", "sane_lists", "nl2br"])
|
| 78 |
inner = _box_response_section(inner)
|
| 79 |
return HTML_WRAPPER.format(logo=LOGO_URL, body=inner)
|
| 80 |
|
|
@@ -466,6 +466,7 @@ def main(argv: list[str]) -> int:
|
|
| 466 |
code_link_claims=code_link_claims, preprint_doi=preprint or "",
|
| 467 |
exclusivity_confirmed=form.get("exclusivity_acknowledged") is True,
|
| 468 |
curator_findings=curator_findings,
|
|
|
|
| 469 |
)
|
| 470 |
if ok:
|
| 471 |
# Decision emails go to Gmail Drafts (curator-applied decision path).
|
|
|
|
| 466 |
code_link_claims=code_link_claims, preprint_doi=preprint or "",
|
| 467 |
exclusivity_confirmed=form.get("exclusivity_acknowledged") is True,
|
| 468 |
curator_findings=curator_findings,
|
| 469 |
+
revision_of=state_pre.get("revision_of") or "",
|
| 470 |
)
|
| 471 |
if ok:
|
| 472 |
# Decision emails go to Gmail Drafts (curator-applied decision path).
|
|
@@ -933,7 +933,9 @@ async def handle_test_pipeline_submission(
|
|
| 933 |
if has_doi:
|
| 934 |
title = "(deferred — resolving from DOI)"
|
| 935 |
abstract = ""
|
| 936 |
-
|
|
|
|
|
|
|
| 937 |
license_id = article_license or ""
|
| 938 |
creators = [{"name": submitter["name"], "orcid": token}]
|
| 939 |
publication_date = _now_iso()[:10]
|
|
@@ -1360,7 +1362,9 @@ async def api_submit(request: Request):
|
|
| 1360 |
# the deposit step is upload-only.
|
| 1361 |
title = "(deferred — resolving from DOI)"
|
| 1362 |
abstract = ""
|
| 1363 |
-
|
|
|
|
|
|
|
| 1364 |
license_id = article_license or ""
|
| 1365 |
creators = [{"name": submitter["name"], "orcid": submitter["orcid"]}]
|
| 1366 |
publication_date = _now_iso()[:10]
|
|
|
|
| 933 |
if has_doi:
|
| 934 |
title = "(deferred — resolving from DOI)"
|
| 935 |
abstract = ""
|
| 936 |
+
# Optional on the DOI route (2026-10-02): the author's own keywords;
|
| 937 |
+
# left empty, the worker takes the source record's.
|
| 938 |
+
keywords: list = [k.strip() for k in re.split(r"[,;]+", str(form.get("keywords") or "")) if k.strip()]
|
| 939 |
license_id = article_license or ""
|
| 940 |
creators = [{"name": submitter["name"], "orcid": token}]
|
| 941 |
publication_date = _now_iso()[:10]
|
|
|
|
| 1362 |
# the deposit step is upload-only.
|
| 1363 |
title = "(deferred — resolving from DOI)"
|
| 1364 |
abstract = ""
|
| 1365 |
+
# Optional on the DOI route (2026-10-02): the author's own keywords;
|
| 1366 |
+
# left empty, the worker takes the source record's.
|
| 1367 |
+
keywords: list = [k.strip() for k in re.split(r"[,;]+", str(form.get("keywords") or "")) if k.strip()]
|
| 1368 |
license_id = article_license or ""
|
| 1369 |
creators = [{"name": submitter["name"], "orcid": submitter["orcid"]}]
|
| 1370 |
publication_date = _now_iso()[:10]
|
|
@@ -7,8 +7,7 @@ tick is recorded here with the wording version the author saw and the time.
|
|
| 7 |
|
| 8 |
The list is ONE append-only JSONL file of events,
|
| 9 |
~/icsac-submissions/newsletter/subscribers.jsonl
|
| 10 |
-
(
|
| 11 |
-
backup.sh:344). A row is either
|
| 12 |
{"event": "subscribe", "email", "name", "sub_id", "consented_at", "wording_version", "source"}
|
| 13 |
or {"event": "unsubscribe", "email", "at", "via"}
|
| 14 |
and the current list is the fold of the events, latest wins. Nothing is ever
|
|
|
|
| 7 |
|
| 8 |
The list is ONE append-only JSONL file of events,
|
| 9 |
~/icsac-submissions/newsletter/subscribers.jsonl
|
| 10 |
+
(covered by the nightly encrypted off-site backup). A row is either
|
|
|
|
| 11 |
{"event": "subscribe", "email", "name", "sub_id", "consented_at", "wording_version", "source"}
|
| 12 |
or {"event": "unsubscribe", "email", "at", "via"}
|
| 13 |
and the current list is the fold of the events, latest wins. Nothing is ever
|
|
@@ -375,6 +375,7 @@ def send_decision(*, to: str, sub_id: str, title: str, author_name: str,
|
|
| 375 |
preprint_doi: str = "",
|
| 376 |
exclusivity_confirmed: bool = False,
|
| 377 |
curator_findings: str = "",
|
|
|
|
| 378 |
) -> tuple[bool, str]:
|
| 379 |
"""Send the decision email with two PDF attachments (panel report + RQC).
|
| 380 |
|
|
@@ -385,6 +386,11 @@ def send_decision(*, to: str, sub_id: str, title: str, author_name: str,
|
|
| 385 |
recommended and the curator decided otherwise). Empty renders a loud bracket so
|
| 386 |
the draft cannot go out without it.
|
| 387 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 388 |
verdict ∈ {accept, revise, scope_reject}; source ∈ {doi, upload}. Per-source
|
| 389 |
template variants live in templates/submission_<verdict>_<source>.md.
|
| 390 |
The body references the attachments rather than inlining the panel
|
|
@@ -466,6 +472,7 @@ def send_decision(*, to: str, sub_id: str, title: str, author_name: str,
|
|
| 466 |
|
| 467 |
subject, body = _render(template, {
|
| 468 |
"icsac_submission_id": sub_id,
|
|
|
|
| 469 |
"title": title, "author_name": author_name,
|
| 470 |
"source_ref": source_ref or "",
|
| 471 |
"deposit_doi": deposit_doi or "",
|
|
|
|
| 375 |
preprint_doi: str = "",
|
| 376 |
exclusivity_confirmed: bool = False,
|
| 377 |
curator_findings: str = "",
|
| 378 |
+
revision_of: str = "",
|
| 379 |
) -> tuple[bool, str]:
|
| 380 |
"""Send the decision email with two PDF attachments (panel report + RQC).
|
| 381 |
|
|
|
|
| 386 |
recommended and the curator decided otherwise). Empty renders a loud bracket so
|
| 387 |
the draft cannot go out without it.
|
| 388 |
|
| 389 |
+
`revision_of` is the earlier submission ID this paper revises, as the state
|
| 390 |
+
records it (only a link verified under the same ORCID is recorded). The
|
| 391 |
+
letter's header names it, so the author can tie this decision to the earlier
|
| 392 |
+
letter; empty for a first submission.
|
| 393 |
+
|
| 394 |
verdict ∈ {accept, revise, scope_reject}; source ∈ {doi, upload}. Per-source
|
| 395 |
template variants live in templates/submission_<verdict>_<source>.md.
|
| 396 |
The body references the attachments rather than inlining the panel
|
|
|
|
| 472 |
|
| 473 |
subject, body = _render(template, {
|
| 474 |
"icsac_submission_id": sub_id,
|
| 475 |
+
"revision_line": f"\n**Revision of:** {revision_of}" if revision_of else "",
|
| 476 |
"title": title, "author_name": author_name,
|
| 477 |
"source_ref": source_ref or "",
|
| 478 |
"deposit_doi": deposit_doi or "",
|
|
@@ -0,0 +1,429 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Replay real submissions through intake, up to the panel and never into it,
|
| 3 |
+
and grade the evidence layer against a hand-checked answer key.
|
| 4 |
+
|
| 5 |
+
The corpus lives OUTSIDE the repository (unpublished author content). Its path
|
| 6 |
+
comes from --corpus or ICSAC_REPLAY_CORPUS; each paper is a directory with
|
| 7 |
+
paper.pdf, submission.json as received
|
| 8 |
+
extraction.json a frozen run of the reference extractor
|
| 9 |
+
key.json the answer key (format below)
|
| 10 |
+
|
| 11 |
+
Everything runs in a shadow directory: config.REVIEWS_DIR and the CiteStamp
|
| 12 |
+
coverage-gap log point into it, notifications and pain signals are recorded
|
| 13 |
+
instead of sent, the misattribution check records which references it would
|
| 14 |
+
judge but calls no model, and the first panel call raises before any request
|
| 15 |
+
is made. Nothing is written to the live submission, its state or the public
|
| 16 |
+
status page. The shadow directory is removed at the end unless --keep.
|
| 17 |
+
|
| 18 |
+
Stages replayed (each from the code under --code, default this checkout):
|
| 19 |
+
code availability -> reference extraction (frozen by default; --extraction
|
| 20 |
+
live calls the extractor) -> citation verification -> misattribution
|
| 21 |
+
selection -> CiteStamp -> code package digest -> blind-review compaction ->
|
| 22 |
+
the panel prompt (captured, not sent).
|
| 23 |
+
|
| 24 |
+
Answer key (key.json):
|
| 25 |
+
{"references": [{"find": <regex on the raw reference>, "level": L,
|
| 26 |
+
"id": <right record id>, "alt_ids": [...], "ok_ids": [...],
|
| 27 |
+
"wrong_ids": [...], "suggest": <id>, "note": ""}],
|
| 28 |
+
"extraction": {"count": n},
|
| 29 |
+
"code_availability": {"missing_link": bool},
|
| 30 |
+
"code_digest": {"fetched": bool, "must_contain": [...]},
|
| 31 |
+
"compaction": {"panel_must_not_contain": [...], "min_kept_ratio": 0.5},
|
| 32 |
+
"evidence": {"must_contain": [...], "must_not_contain": [...]}}
|
| 33 |
+
L is CONFIRMED, DOI MISMATCH, DOI DEAD, WEB SOURCE or UNCONFIRMED (no
|
| 34 |
+
identifier was cited: an abstention or a match to one of ok_ids is right).
|
| 35 |
+
|
| 36 |
+
Check groups. A run is green when every HARD group passes:
|
| 37 |
+
identity HARD levels and record ids of references with an identifier
|
| 38 |
+
or a URL; a wrong record reported as the cited one fails
|
| 39 |
+
extraction HARD every key reference found (count checked in live mode)
|
| 40 |
+
code_avail HARD the code/data link precheck
|
| 41 |
+
code_digest HARD the digest fetched (or not) and names what it must
|
| 42 |
+
compaction HARD no author identifier reaches the panel prompt
|
| 43 |
+
evidence HARD the panel-facing citation block says what the key says
|
| 44 |
+
fail_closed --bar 2+ only identity-confirmed references can ground an
|
| 45 |
+
integrity finding: misattribution and CiteStamp see
|
| 46 |
+
confirmed references only, and a text-only match is
|
| 47 |
+
shown to the panel as unconfirmed, without the record
|
| 48 |
+
fuzzy --bar 3 no text-only match to a wrong record at all
|
| 49 |
+
|
| 50 |
+
Usage (repo root, prod venv):
|
| 51 |
+
ICSAC_REPLAY_CORPUS=~/corpus .venv/bin/python intake/scripts/replay.py
|
| 52 |
+
... --paper P1 --code /path/to/older/checkout --skip compaction,citestamp
|
| 53 |
+
Exit 0 when every paper is green at the requested bar, 1 otherwise.
|
| 54 |
+
"""
|
| 55 |
+
from __future__ import annotations
|
| 56 |
+
|
| 57 |
+
import argparse
|
| 58 |
+
import copy
|
| 59 |
+
import importlib
|
| 60 |
+
import json
|
| 61 |
+
import os
|
| 62 |
+
import re
|
| 63 |
+
import shutil
|
| 64 |
+
import sys
|
| 65 |
+
import tempfile
|
| 66 |
+
import time
|
| 67 |
+
from pathlib import Path
|
| 68 |
+
|
| 69 |
+
HERE = Path(__file__).resolve()
|
| 70 |
+
DEFAULT_CODE = HERE.parents[2]
|
| 71 |
+
STAGES = ("citestamp", "digest", "compaction")
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
class StopBeforePanel(Exception):
|
| 75 |
+
"""Raised by the first panel call; carries the prompt it would have sent."""
|
| 76 |
+
|
| 77 |
+
def __init__(self, prompt: str):
|
| 78 |
+
super().__init__("replay stops before the panel")
|
| 79 |
+
self.prompt = prompt
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
# ── identifiers and levels (independent of the code under test) ──────────
|
| 83 |
+
def norm_id(v) -> str:
|
| 84 |
+
s = str(v or "").strip().lower()
|
| 85 |
+
for p in ("https://doi.org/", "http://doi.org/", "https://dx.doi.org/", "doi:"):
|
| 86 |
+
if s.startswith(p):
|
| 87 |
+
s = s[len(p):]
|
| 88 |
+
m = re.fullmatch(r"(?:arxiv:|10\.48550/arxiv\.)(\d{4}\.\d{4,5})(v\d+)?", s)
|
| 89 |
+
return f"arxiv:{m.group(1)}" if m else s.rstrip("/.")
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
def level_of(c: dict) -> str:
|
| 93 |
+
"""The evidence level a verified citation record carries, read from its raw
|
| 94 |
+
fields so that the grader does not trust the labeller it is grading."""
|
| 95 |
+
if c.get("confidence") == "web-source":
|
| 96 |
+
return "WEB SOURCE"
|
| 97 |
+
ident = c.get("doi_identity")
|
| 98 |
+
if not c.get("verified") and ident == "mismatch":
|
| 99 |
+
return "DOI MISMATCH"
|
| 100 |
+
if not c.get("verified") and ident == "dead":
|
| 101 |
+
return "DOI DEAD"
|
| 102 |
+
if c.get("verified") and c.get("confidence") == "exact-id":
|
| 103 |
+
return "CONFIRMED"
|
| 104 |
+
if c.get("verified") or c.get("candidate_id"):
|
| 105 |
+
return "MATCHED"
|
| 106 |
+
return "UNVERIFIABLE"
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
def record_id_of(c: dict) -> str:
|
| 110 |
+
return norm_id(c.get("resolved_id") or c.get("candidate_id"))
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
# ── the run ───────────────────────────��──────────────────────────────────
|
| 114 |
+
def _load_code(code_dir: Path):
|
| 115 |
+
for name in list(sys.modules):
|
| 116 |
+
if name.split(".")[0] in ("config", "review", "citation_verify", "citation_misattribution",
|
| 117 |
+
"citestamp_check", "code_digest", "code_availability", "review_compaction",
|
| 118 |
+
"submission_intake", "notify", "intake", "redaction", "publications",
|
| 119 |
+
"review_quality_control", "panel_retry"):
|
| 120 |
+
del sys.modules[name]
|
| 121 |
+
sys.path[:] = [p for p in sys.path if Path(p or ".").resolve() != DEFAULT_CODE]
|
| 122 |
+
sys.path.insert(0, str(code_dir))
|
| 123 |
+
return importlib.import_module("config")
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
def run_paper(paper_dir: Path, code_dir: Path, extraction: str, skip: set, shadow: Path, log) -> dict:
|
| 127 |
+
config = _load_code(code_dir)
|
| 128 |
+
config.REVIEWS_DIR = str(shadow / "reviews")
|
| 129 |
+
os.makedirs(config.REVIEWS_DIR, exist_ok=True)
|
| 130 |
+
out = {"events": [], "notifications": [], "stages": {}}
|
| 131 |
+
rid = "ICSAC-SUB-TEST-REPLAY-" + re.sub(r"[^A-Za-z0-9]", "", paper_dir.name)
|
| 132 |
+
|
| 133 |
+
def rec(name):
|
| 134 |
+
def _f(*a, **k):
|
| 135 |
+
out["notifications"].append(name)
|
| 136 |
+
return None
|
| 137 |
+
return _f
|
| 138 |
+
|
| 139 |
+
import notify
|
| 140 |
+
for name in dir(notify):
|
| 141 |
+
if not name.startswith("_") and callable(getattr(notify, name)) and name[0].islower():
|
| 142 |
+
setattr(notify, name, rec(f"notify.{name}"))
|
| 143 |
+
import review
|
| 144 |
+
for name in ("_fire_compaction_pain", "_fire_pain", "fire_pain"):
|
| 145 |
+
if hasattr(review, name):
|
| 146 |
+
setattr(review, name, rec(f"review.{name}"))
|
| 147 |
+
|
| 148 |
+
def _guard(prompt=None, *a, **k):
|
| 149 |
+
raise StopBeforePanel(prompt if isinstance(prompt, str) else "")
|
| 150 |
+
for name in ("_run_single_pass", "_run_panel_chain", "run_claude_review", "run_openrouter_review"):
|
| 151 |
+
if hasattr(review, name):
|
| 152 |
+
setattr(review, name, _guard)
|
| 153 |
+
|
| 154 |
+
import citation_verify
|
| 155 |
+
frozen = json.loads((paper_dir / "extraction.json").read_text())
|
| 156 |
+
if extraction == "frozen":
|
| 157 |
+
citation_verify.extract_citations = lambda text, record_id, *a, **k: copy.deepcopy(frozen)
|
| 158 |
+
|
| 159 |
+
try:
|
| 160 |
+
import citestamp_check
|
| 161 |
+
citestamp_check.COVERAGE_GAP_LOG = str(shadow / "reviews" / "citestamp_coverage_gaps.jsonl")
|
| 162 |
+
except ImportError:
|
| 163 |
+
citestamp_check = None
|
| 164 |
+
if "citestamp" in skip and hasattr(review, "_run_citestamp_check"):
|
| 165 |
+
review._run_citestamp_check = lambda record_id, citations, report, *a, **k: report
|
| 166 |
+
|
| 167 |
+
import citation_misattribution as cm
|
| 168 |
+
seen = {"eligible": None, "checked": None}
|
| 169 |
+
real_select = cm.select_load_bearing
|
| 170 |
+
|
| 171 |
+
def _select(citations, full_text, *a, **k):
|
| 172 |
+
eligible = real_select(citations, full_text, max_n=10 ** 6)
|
| 173 |
+
seen["eligible"] = [copy.deepcopy(c) for c in eligible]
|
| 174 |
+
return eligible
|
| 175 |
+
|
| 176 |
+
def _check(load_bearing, *a, **k):
|
| 177 |
+
seen["checked"] = [copy.deepcopy(c) for c in load_bearing]
|
| 178 |
+
return [] # no model is called in a replay
|
| 179 |
+
cm.select_load_bearing, cm.check_misattribution_batch = _select, _check
|
| 180 |
+
|
| 181 |
+
if "digest" in skip and hasattr(review, "_run_code_digest"):
|
| 182 |
+
review._run_code_digest = lambda review_data, *a, **k: "(code package digest skipped by replay)"
|
| 183 |
+
if "compaction" in skip:
|
| 184 |
+
import review_compaction
|
| 185 |
+
review_compaction.compact_paper = lambda text, *a, **k: (text, {
|
| 186 |
+
"original_chars": len(text), "redacted_chars": len(text), "reduction_pct": 0, "replay_skipped": True})
|
| 187 |
+
|
| 188 |
+
# review_data exactly as the worker builds it (a copy of the paper dir; the
|
| 189 |
+
# worker's builder only reads submission.json and paper.pdf)
|
| 190 |
+
sub_dir = shadow / "submission"
|
| 191 |
+
sub_dir.mkdir()
|
| 192 |
+
for f in ("paper.pdf", "submission.json"):
|
| 193 |
+
shutil.copyfile(paper_dir / f, sub_dir / f)
|
| 194 |
+
submission = json.loads((sub_dir / "submission.json").read_text())
|
| 195 |
+
try:
|
| 196 |
+
from intake import submission_worker
|
| 197 |
+
review_data = submission_worker._build_review_data(rid, sub_dir)
|
| 198 |
+
out["builder"] = "worker"
|
| 199 |
+
except Exception as exc:
|
| 200 |
+
import submission_intake
|
| 201 |
+
out["builder"] = f"harness ({type(exc).__name__})"
|
| 202 |
+
review_data = {"record_id": rid, "title": submission.get("title") or "", "creators": submission.get("creators") or [],
|
| 203 |
+
"description": submission.get("abstract") or "", "keywords": submission.get("keywords") or [],
|
| 204 |
+
"pdf_path": str(sub_dir / "paper.pdf"),
|
| 205 |
+
"full_text": submission_intake.extract_pdf_text(str(sub_dir / "paper.pdf")),
|
| 206 |
+
"submission": {"code_data": submission.get("code_data") or {},
|
| 207 |
+
"related_identifiers": submission.get("related_identifiers") or []}}
|
| 208 |
+
review_data["record_id"] = rid
|
| 209 |
+
|
| 210 |
+
t0 = time.time()
|
| 211 |
+
try:
|
| 212 |
+
import code_availability
|
| 213 |
+
out["code_availability"] = code_availability.assess(submission, review_data.get("full_text", ""))
|
| 214 |
+
except ImportError:
|
| 215 |
+
out["code_availability"] = None
|
| 216 |
+
out["stages"]["code_availability"] = round(time.time() - t0, 1)
|
| 217 |
+
|
| 218 |
+
prompt = None
|
| 219 |
+
t0 = time.time()
|
| 220 |
+
try:
|
| 221 |
+
markdown, aggregate = review.review_paper(review_data)
|
| 222 |
+
out["stopped"] = f"review_paper returned before the panel: {aggregate.get('recommendation')}"
|
| 223 |
+
out["aggregate"] = {k: aggregate.get(k) for k in ("recommendation", "reason")}
|
| 224 |
+
out["compaction_manifest"] = aggregate.get("compaction_manifest")
|
| 225 |
+
except StopBeforePanel as stop:
|
| 226 |
+
prompt = stop.prompt
|
| 227 |
+
out["stopped"] = "before the panel"
|
| 228 |
+
out["stages"]["review_paper_until_panel"] = round(time.time() - t0, 1)
|
| 229 |
+
out["prompt"] = prompt or ""
|
| 230 |
+
cj = shadow / "reviews" / f"{rid}_citations.json"
|
| 231 |
+
out["citations"] = json.loads(cj.read_text()) if cj.exists() else {}
|
| 232 |
+
cmd = shadow / "reviews" / f"{rid}_citations.md"
|
| 233 |
+
out["report"] = cmd.read_text() if cmd.exists() else ""
|
| 234 |
+
dg = shadow / "reviews" / f"{rid}_code_digest.md"
|
| 235 |
+
out["digest"] = dg.read_text() if dg.exists() else ""
|
| 236 |
+
out["misattribution_eligible"] = seen["eligible"]
|
| 237 |
+
out["full_text_chars"] = len(review_data.get("full_text") or "")
|
| 238 |
+
if out.get("compaction_manifest") is None and prompt is not None:
|
| 239 |
+
out["compaction_manifest"] = "(in prompt)"
|
| 240 |
+
return out
|
| 241 |
+
|
| 242 |
+
|
| 243 |
+
# ── grading ──────────────────────────────────────────────────────────────
|
| 244 |
+
def _find(cits: list, rx: str) -> list:
|
| 245 |
+
r = re.compile(rx, re.I)
|
| 246 |
+
return [c for c in cits if r.search(c.get("raw") or "")]
|
| 247 |
+
|
| 248 |
+
|
| 249 |
+
def grade(key: dict, run: dict, bar: int, skip: set, extraction: str) -> dict:
|
| 250 |
+
groups = {g: [] for g in ("identity", "extraction", "code_avail", "code_digest", "compaction",
|
| 251 |
+
"evidence", "fail_closed", "fuzzy")}
|
| 252 |
+
cits = (run.get("citations") or {}).get("citations") or []
|
| 253 |
+
prompt = run.get("prompt") or ""
|
| 254 |
+
if not prompt:
|
| 255 |
+
groups["evidence"].append(f"no panel prompt captured ({run.get('stopped')})")
|
| 256 |
+
|
| 257 |
+
def fail(group, msg):
|
| 258 |
+
groups[group].append(msg)
|
| 259 |
+
|
| 260 |
+
keyed = []
|
| 261 |
+
for k in key.get("references", []):
|
| 262 |
+
hits = _find(cits, k["find"])
|
| 263 |
+
if len(hits) != 1:
|
| 264 |
+
fail("extraction", f"{k['find']!r}: {len(hits)} extracted references match (want 1)")
|
| 265 |
+
continue
|
| 266 |
+
c = hits[0]
|
| 267 |
+
keyed.append((k, c))
|
| 268 |
+
got, rid = level_of(c), record_id_of(c)
|
| 269 |
+
want = k["level"]
|
| 270 |
+
right = {norm_id(x) for x in [k.get("id")] + k.get("alt_ids", []) + k.get("ok_ids", []) if x}
|
| 271 |
+
wrong = {norm_id(x) for x in k.get("wrong_ids", [])}
|
| 272 |
+
tag = f"{k['find']!r}: want {want}, got {got}" + (f" -> {rid}" if rid else "")
|
| 273 |
+
if want == "CONFIRMED":
|
| 274 |
+
if got != "CONFIRMED" or rid not in right:
|
| 275 |
+
fail("identity", tag)
|
| 276 |
+
elif want in ("DOI MISMATCH", "DOI DEAD"):
|
| 277 |
+
if got != want:
|
| 278 |
+
fail("identity", tag + (" (a bad identifier reported as real)" if got in ("CONFIRMED", "MATCHED") else ""))
|
| 279 |
+
elif k.get("suggest") and norm_id(c.get("suggested_id")) != norm_id(k["suggest"]):
|
| 280 |
+
fail("fuzzy", f"{k['find']!r}: suggestion {c.get('suggested_id')} (want {k['suggest']})")
|
| 281 |
+
elif want == "WEB SOURCE":
|
| 282 |
+
if got in ("CONFIRMED", "MATCHED") and rid not in right:
|
| 283 |
+
fail("identity", tag + " (a web page reported as an unrelated registry work)")
|
| 284 |
+
elif want == "UNCONFIRMED":
|
| 285 |
+
if got == "CONFIRMED":
|
| 286 |
+
fail("identity", tag + " (no identifier was cited, nothing can be confirmed)")
|
| 287 |
+
elif got == "MATCHED" and rid not in right and not k.get("any_match_ok"):
|
| 288 |
+
fail("fuzzy", tag + (" (a known wrong record)" if rid in wrong else " (not the cited work)"))
|
| 289 |
+
else:
|
| 290 |
+
fail("identity", f"{k['find']!r}: unknown key level {want}")
|
| 291 |
+
|
| 292 |
+
ex = key.get("extraction") or {}
|
| 293 |
+
if extraction == "live" and ex.get("count") is not None and abs(len(cits) - ex["count"]) > ex.get("tolerance", 0):
|
| 294 |
+
fail("extraction", f"{len(cits)} references extracted (key: {ex['count']})")
|
| 295 |
+
|
| 296 |
+
ca = key.get("code_availability")
|
| 297 |
+
if ca is not None:
|
| 298 |
+
got = run.get("code_availability")
|
| 299 |
+
if got is None:
|
| 300 |
+
fail("code_avail", "the code under test has no code/data precheck")
|
| 301 |
+
elif bool(got.get("missing_link")) != ca["missing_link"]:
|
| 302 |
+
fail("code_avail", f"missing_link {got.get('missing_link')} (key {ca['missing_link']})")
|
| 303 |
+
|
| 304 |
+
cd = key.get("code_digest")
|
| 305 |
+
if cd is not None and "digest" not in skip:
|
| 306 |
+
dig = run.get("digest") or ""
|
| 307 |
+
fetched = bool(dig) and "not fetched" not in dig and "(none" not in dig and "unavailable" not in dig
|
| 308 |
+
if fetched != cd["fetched"]:
|
| 309 |
+
fail("code_digest", f"package fetched={fetched} (key {cd['fetched']})")
|
| 310 |
+
for s in cd.get("must_contain", []):
|
| 311 |
+
if s not in dig:
|
| 312 |
+
fail("code_digest", f"digest lacks {s!r}")
|
| 313 |
+
elif prompt and s not in prompt:
|
| 314 |
+
fail("code_digest", f"the panel prompt lacks {s!r}")
|
| 315 |
+
|
| 316 |
+
cp = key.get("compaction") or {}
|
| 317 |
+
if "compaction" not in skip and prompt:
|
| 318 |
+
for s in cp.get("panel_must_not_contain", []):
|
| 319 |
+
if re.search(re.escape(s), prompt, re.I):
|
| 320 |
+
fail("compaction", f"author identifier reaches the panel: {s!r}")
|
| 321 |
+
man = run.get("compaction_manifest")
|
| 322 |
+
if isinstance(man, dict) and man.get("_failure"):
|
| 323 |
+
fail("compaction", f"compaction failed: {man['_failure']}")
|
| 324 |
+
if run.get("full_text_chars") and len(prompt) < cp.get("min_kept_ratio", 0.5) * run["full_text_chars"]:
|
| 325 |
+
fail("compaction", f"panel prompt {len(prompt)} chars vs paper {run['full_text_chars']}")
|
| 326 |
+
elif "compaction" not in skip and not prompt:
|
| 327 |
+
fail("compaction", f"no panel prompt ({run.get('stopped')})")
|
| 328 |
+
|
| 329 |
+
ev = key.get("evidence") or {}
|
| 330 |
+
for s in ev.get("must_contain", []):
|
| 331 |
+
if s not in prompt:
|
| 332 |
+
fail("evidence", f"panel prompt lacks {s!r}")
|
| 333 |
+
for s in ev.get("must_not_contain", []):
|
| 334 |
+
if s in prompt:
|
| 335 |
+
fail("evidence", f"panel prompt carries {s!r}")
|
| 336 |
+
|
| 337 |
+
# fail closed: nothing below confirmed identity grounds an integrity finding
|
| 338 |
+
confirmed_raw = {c.get("raw") for k, c in keyed if level_of(c) == "CONFIRMED"} | \
|
| 339 |
+
{c.get("raw") for c in cits if level_of(c) == "CONFIRMED"}
|
| 340 |
+
elig = run.get("misattribution_eligible")
|
| 341 |
+
if elig is None:
|
| 342 |
+
fail("fail_closed", "misattribution selection never ran")
|
| 343 |
+
else:
|
| 344 |
+
for c in elig:
|
| 345 |
+
if c.get("raw") not in confirmed_raw:
|
| 346 |
+
fail("fail_closed", f"misattribution would judge a {level_of(c)} reference: {(c.get('raw') or '')[:70]!r}")
|
| 347 |
+
cs = (run.get("citations") or {}).get("citestamp") or {}
|
| 348 |
+
confirmed_ids = {record_id_of(c) for c in cits if level_of(c) == "CONFIRMED"}
|
| 349 |
+
for row in cs.get("results") or []:
|
| 350 |
+
if norm_id(row.get("doi")) not in confirmed_ids:
|
| 351 |
+
fail("fail_closed", f"CiteStamp queried an unconfirmed DOI {row.get('doi')}")
|
| 352 |
+
for c in cits:
|
| 353 |
+
if level_of(c) == "MATCHED":
|
| 354 |
+
rid = record_id_of(c)
|
| 355 |
+
if rid and rid in prompt.lower():
|
| 356 |
+
fail("fail_closed", f"the panel sees the unconfirmed candidate {rid} for {(c.get('raw') or '')[:60]!r}")
|
| 357 |
+
|
| 358 |
+
hard = ("identity", "extraction", "code_avail", "code_digest", "compaction", "evidence")
|
| 359 |
+
need = list(hard) + (["fail_closed"] if bar >= 2 else []) + (["fuzzy"] if bar >= 3 else [])
|
| 360 |
+
green = all(not groups[g] for g in need)
|
| 361 |
+
return {"green": green, "bar": bar, "groups": groups, "levels": {
|
| 362 |
+
k["find"]: f"{level_of(c)} {record_id_of(c)}".strip() for k, c in keyed}}
|
| 363 |
+
|
| 364 |
+
|
| 365 |
+
def main() -> int:
|
| 366 |
+
ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
| 367 |
+
ap.add_argument("--corpus", default=os.environ.get("ICSAC_REPLAY_CORPUS"))
|
| 368 |
+
ap.add_argument("--paper", action="append", help="corpus subdirectory (repeatable; default all)")
|
| 369 |
+
ap.add_argument("--code", default=str(DEFAULT_CODE), help="pipeline checkout to replay")
|
| 370 |
+
ap.add_argument("--extraction", choices=("frozen", "live"), default="frozen")
|
| 371 |
+
ap.add_argument("--skip", default="", help=f"comma list of {','.join(STAGES)}")
|
| 372 |
+
ap.add_argument("--bar", type=int, default=1, choices=(1, 2, 3))
|
| 373 |
+
ap.add_argument("--keep", action="store_true", help="keep the shadow directory")
|
| 374 |
+
ap.add_argument("--out", help="write the full result JSON here")
|
| 375 |
+
a = ap.parse_args()
|
| 376 |
+
if not a.corpus or not Path(a.corpus).is_dir():
|
| 377 |
+
print("replay: no corpus (set ICSAC_REPLAY_CORPUS or --corpus)", file=sys.stderr)
|
| 378 |
+
return 2
|
| 379 |
+
skip = {s for s in a.skip.split(",") if s}
|
| 380 |
+
corpus = Path(a.corpus).expanduser()
|
| 381 |
+
papers = a.paper or sorted(p.name for p in corpus.iterdir() if (p / "key.json").exists())
|
| 382 |
+
code_dir = Path(a.code).expanduser().resolve()
|
| 383 |
+
results, all_green = {}, True
|
| 384 |
+
for name in papers:
|
| 385 |
+
pdir = corpus / name
|
| 386 |
+
key = json.loads((pdir / "key.json").read_text())
|
| 387 |
+
shadow = Path(tempfile.mkdtemp(prefix="icsac-replay-"))
|
| 388 |
+
cwd = os.getcwd()
|
| 389 |
+
try:
|
| 390 |
+
os.chdir(code_dir)
|
| 391 |
+
run = run_paper(pdir, code_dir, a.extraction, skip, shadow, log=print)
|
| 392 |
+
res = grade(key, run, a.bar, skip, a.extraction)
|
| 393 |
+
res["stages_s"] = run["stages"]
|
| 394 |
+
res["notifications"] = run["notifications"]
|
| 395 |
+
res["builder"] = run.get("builder")
|
| 396 |
+
if run.get("aggregate"):
|
| 397 |
+
man = run.get("compaction_manifest") if isinstance(run.get("compaction_manifest"), dict) else {}
|
| 398 |
+
res["stopped_early"] = f"{run['aggregate'].get('recommendation')}: {run['aggregate'].get('reason')}" + (
|
| 399 |
+
f" [compaction: {man.get('_failure')}]" if man.get("_failure") else "")
|
| 400 |
+
results[name] = res
|
| 401 |
+
if a.out:
|
| 402 |
+
res["run"] = {k: run.get(k) for k in ("stopped", "aggregate", "compaction_manifest", "code_availability",
|
| 403 |
+
"misattribution_eligible", "citations")}
|
| 404 |
+
except Exception as exc:
|
| 405 |
+
results[name] = {"green": False, "error": f"{type(exc).__name__}: {exc}"}
|
| 406 |
+
finally:
|
| 407 |
+
os.chdir(cwd)
|
| 408 |
+
if a.keep:
|
| 409 |
+
print(f" shadow kept: {shadow}")
|
| 410 |
+
else:
|
| 411 |
+
shutil.rmtree(shadow, ignore_errors=True)
|
| 412 |
+
r = results[name]
|
| 413 |
+
all_green &= bool(r.get("green"))
|
| 414 |
+
print(f"{'GREEN' if r.get('green') else 'RED '} {name} (code {code_dir.name}, bar {a.bar}, "
|
| 415 |
+
f"extraction {a.extraction}, skip {','.join(sorted(skip)) or '-'})")
|
| 416 |
+
if r.get("error"):
|
| 417 |
+
print(f" error: {r['error']}")
|
| 418 |
+
if r.get("stopped_early"):
|
| 419 |
+
print(f" stopped early: {r['stopped_early']}")
|
| 420 |
+
for g, msgs in (r.get("groups") or {}).items():
|
| 421 |
+
for m in msgs:
|
| 422 |
+
print(f" [{g}] {m}")
|
| 423 |
+
if a.out:
|
| 424 |
+
Path(a.out).write_text(json.dumps(results, indent=1, default=str))
|
| 425 |
+
return 0 if all_green else 1
|
| 426 |
+
|
| 427 |
+
|
| 428 |
+
if __name__ == "__main__":
|
| 429 |
+
sys.exit(main())
|
|
@@ -4,8 +4,8 @@
|
|
| 4 |
# LEAVES audit-log-test.jsonl untouched (durable history) and queue/
|
| 5 |
# untouched (active state).
|
| 6 |
#
|
| 7 |
-
#
|
| 8 |
-
#
|
| 9 |
set -u
|
| 10 |
RETENTION_DAYS="${RETENTION_DAYS:-7}"
|
| 11 |
TEST_ROOT="${HOME}/icsac-submissions/test"
|
|
|
|
| 4 |
# LEAVES audit-log-test.jsonl untouched (durable history) and queue/
|
| 5 |
# untouched (active state).
|
| 6 |
#
|
| 7 |
+
# Runs weekly from cron. On failure it posts to the monitoring endpoint,
|
| 8 |
+
# so a broken sweep doesn't go silent.
|
| 9 |
set -u
|
| 10 |
RETENTION_DAYS="${RETENTION_DAYS:-7}"
|
| 11 |
TEST_ROOT="${HOME}/icsac-submissions/test"
|
|
@@ -36,7 +36,7 @@ try:
|
|
| 36 |
check(not list(seq_file.parent.glob(".doi-seq.tmp-*")), "counter: no temp file left behind")
|
| 37 |
seq_file.unlink()
|
| 38 |
d = tmp / "subs" / "ICSAC-SUB-00042" / "crossref"; d.mkdir(parents=True)
|
| 39 |
-
(d / "doi.json").write_text(json.dumps({"doi": "10.
|
| 40 |
check(cd._next_seq(seq_file, 2026) == 6, "counter: a lost file resumes above the highest doi.json suffix")
|
| 41 |
check(cd._next_seq(seq_file, 2027) == 1, "counter: another year starts at 1")
|
| 42 |
seq_file.write_text("{not json")
|
|
@@ -51,11 +51,11 @@ try:
|
|
| 51 |
sub = tmp / "subs" / "ICSAC-SUB-00099"; (sub / "crossref").mkdir(parents=True)
|
| 52 |
shutil.copyfile(src, sub / "submission.json")
|
| 53 |
(sub / "crossref" / "staged.json").write_text(json.dumps(
|
| 54 |
-
{"doi": "10.
|
| 55 |
-
cd.stage(sub, doi="10.
|
| 56 |
st = json.loads((sub / "crossref" / "staged.json").read_text())
|
| 57 |
check(st.get("checkpoints", {}).get("crossref_batch_id") == "b1", "stage: an in-flight batch id survives a re-stage")
|
| 58 |
-
check(st.get("doi") == "10.
|
| 59 |
else:
|
| 60 |
print(" skip stage() check (no SUB-00008 submission.json on this host)")
|
| 61 |
|
|
@@ -74,7 +74,7 @@ try:
|
|
| 74 |
check(out.get("sub_id") == "ICSAC-SUB-00001" and out.get("ssrn_id") == "123"
|
| 75 |
and out.get("license_url", "").endswith("by/4.0/"), "registry: an update keeps sub_id, ssrn_id, licence")
|
| 76 |
check(out.get("promotion_opt_out") is True, "registry: an update takes the new exclusion")
|
| 77 |
-
new = pub.upsert_entry(approved=True, proto={"title": "New paper", "authors": ["B. Author"], "doi": "10.
|
| 78 |
"source": "submission-pdf", "sub_id": "ICSAC-SUB-00007",
|
| 79 |
"license_url": "https://creativecommons.org/licenses/by/4.0/",
|
| 80 |
"promotion_exclusions": ["social"], "persistence_opt_out": True})
|
|
|
|
| 36 |
check(not list(seq_file.parent.glob(".doi-seq.tmp-*")), "counter: no temp file left behind")
|
| 37 |
seq_file.unlink()
|
| 38 |
d = tmp / "subs" / "ICSAC-SUB-00042" / "crossref"; d.mkdir(parents=True)
|
| 39 |
+
(d / "doi.json").write_text(json.dumps({"doi": "10.5555/icsac.2026.005"}))
|
| 40 |
check(cd._next_seq(seq_file, 2026) == 6, "counter: a lost file resumes above the highest doi.json suffix")
|
| 41 |
check(cd._next_seq(seq_file, 2027) == 1, "counter: another year starts at 1")
|
| 42 |
seq_file.write_text("{not json")
|
|
|
|
| 51 |
sub = tmp / "subs" / "ICSAC-SUB-00099"; (sub / "crossref").mkdir(parents=True)
|
| 52 |
shutil.copyfile(src, sub / "submission.json")
|
| 53 |
(sub / "crossref" / "staged.json").write_text(json.dumps(
|
| 54 |
+
{"doi": "10.5555/icsac.2026.099", "checkpoints": {"crossref_batch_id": "b1", "crossref_deposited_at": "x"}}))
|
| 55 |
+
cd.stage(sub, doi="10.5555/icsac.2026.099", log=lambda m: None)
|
| 56 |
st = json.loads((sub / "crossref" / "staged.json").read_text())
|
| 57 |
check(st.get("checkpoints", {}).get("crossref_batch_id") == "b1", "stage: an in-flight batch id survives a re-stage")
|
| 58 |
+
check(st.get("doi") == "10.5555/icsac.2026.099" and (sub / "crossref" / "deposit.xml").exists(), "stage: XML + record written")
|
| 59 |
else:
|
| 60 |
print(" skip stage() check (no SUB-00008 submission.json on this host)")
|
| 61 |
|
|
|
|
| 74 |
check(out.get("sub_id") == "ICSAC-SUB-00001" and out.get("ssrn_id") == "123"
|
| 75 |
and out.get("license_url", "").endswith("by/4.0/"), "registry: an update keeps sub_id, ssrn_id, licence")
|
| 76 |
check(out.get("promotion_opt_out") is True, "registry: an update takes the new exclusion")
|
| 77 |
+
new = pub.upsert_entry(approved=True, proto={"title": "New paper", "authors": ["B. Author"], "doi": "10.5555/icsac.2026.007",
|
| 78 |
"source": "submission-pdf", "sub_id": "ICSAC-SUB-00007",
|
| 79 |
"license_url": "https://creativecommons.org/licenses/by/4.0/",
|
| 80 |
"promotion_exclusions": ["social"], "persistence_opt_out": True})
|
|
@@ -142,7 +142,7 @@ try:
|
|
| 142 |
sub3 = tmp / "ICSAC-SUB-00096"
|
| 143 |
(sub3 / "crossref").mkdir(parents=True)
|
| 144 |
(sub3 / "crossref" / "deposit.xml").write_bytes(b"<x/>")
|
| 145 |
-
(sub3 / "crossref" / "staged.json").write_text(json.dumps({"doi": "10.
|
| 146 |
(sub3 / "submission.json").write_text(json.dumps({"title": "T"}))
|
| 147 |
(sub3 / "state.json").write_text(json.dumps({"state": "completed_email_failed", "decision": "accept"}))
|
| 148 |
os.environ.pop("ICSAC_REGISTER_WITHOUT_NOTICE", None)
|
|
|
|
| 142 |
sub3 = tmp / "ICSAC-SUB-00096"
|
| 143 |
(sub3 / "crossref").mkdir(parents=True)
|
| 144 |
(sub3 / "crossref" / "deposit.xml").write_bytes(b"<x/>")
|
| 145 |
+
(sub3 / "crossref" / "staged.json").write_text(json.dumps({"doi": "10.5555/icsac.2026.999"}))
|
| 146 |
(sub3 / "submission.json").write_text(json.dumps({"title": "T"}))
|
| 147 |
(sub3 / "state.json").write_text(json.dumps({"state": "completed_email_failed", "decision": "accept"}))
|
| 148 |
os.environ.pop("ICSAC_REGISTER_WITHOUT_NOTICE", None)
|
|
@@ -14,8 +14,9 @@ plus: a Zenodo dataset DOI confirmed through DataCite; a short registry title
|
|
| 14 |
("Less Is Enough") confirmed; HTML in a Crossref title; a DOI with no cited
|
| 15 |
title judged on authors + year; verify_all keeping the cited title and letting
|
| 16 |
a DOI disagreement downgrade an earlier fuzzy "verified"; the report's
|
| 17 |
-
sections and wording; the header line; the CiteStamp coverage-gap log;
|
| 18 |
-
code package digest on a synthetic archive (no network)
|
|
|
|
| 19 |
"""
|
| 20 |
from __future__ import annotations
|
| 21 |
|
|
@@ -211,8 +212,8 @@ try:
|
|
| 211 |
check(re.search(r"Quarrie[^\n]*— DOI DEAD\.", rep) and "10.0000/jpc.2023.0929" in rep, "Quarrie listed as DOI DEAD with the real record")
|
| 212 |
check("Brook 2026** — UNVERIFIABLE" in rep and "different work" in rep, "Brook unverifiable with the reason")
|
| 213 |
check("— REAL." not in rep, "no line says REAL any more")
|
| 214 |
-
check("A. Author et al.
|
| 215 |
-
check("
|
| 216 |
check(re.search(r"Ferreira[^\n]*— CONFIRMED\.[^\n]*translated or variant title", rep) is not None, "the variant title is shown on the confirmed line")
|
| 217 |
cv.save_citation_report("ICSAC-SUB-TEST-1", rep_in, rep)
|
| 218 |
import review
|
|
@@ -281,15 +282,88 @@ try:
|
|
| 281 |
check("{{curator_findings}}" in raw and "Curation team findings" in raw, f"{tpl}: findings section present")
|
| 282 |
subj, body = na._render("submission_revise_upload.md", {
|
| 283 |
"icsac_submission_id": "ICSAC-SUB-TEST-1", "title": "T", "author_name": "A",
|
| 284 |
-
"attachments_block": "(att)", "compaction_disclosure": "(disc)",
|
| 285 |
"curator_findings": "1. Fix the unit in Table 2.\n2. Define the sample window."})
|
| 286 |
check("1. Fix the unit in Table 2." in body and "the decision is the curation team's" in body, "findings rendered")
|
| 287 |
subj, body = na._render("submission_revise_upload.md", {
|
| 288 |
"icsac_submission_id": "ICSAC-SUB-TEST-1", "title": "T", "author_name": "A",
|
| 289 |
-
"attachments_block": "(att)", "compaction_disclosure": "(disc)",
|
| 290 |
"curator_findings": "[CURATION TEAM FINDINGS - x]"})
|
| 291 |
check("[CURATION TEAM FINDINGS" in body, "empty findings leave the loud bracket")
|
| 292 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 293 |
finally:
|
| 294 |
shutil.rmtree(TMP, ignore_errors=True)
|
| 295 |
|
|
|
|
| 14 |
("Less Is Enough") confirmed; HTML in a Crossref title; a DOI with no cited
|
| 15 |
title judged on authors + year; verify_all keeping the cited title and letting
|
| 16 |
a DOI disagreement downgrade an earlier fuzzy "verified"; the report's
|
| 17 |
+
sections and wording; the header line; the CiteStamp coverage-gap log; the
|
| 18 |
+
code package digest on a synthetic archive (no network); and web pages cited
|
| 19 |
+
by URL, which must never be matched to a registry record by a fuzzy search.
|
| 20 |
"""
|
| 21 |
from __future__ import annotations
|
| 22 |
|
|
|
|
| 212 |
check(re.search(r"Quarrie[^\n]*— DOI DEAD\.", rep) and "10.0000/jpc.2023.0929" in rep, "Quarrie listed as DOI DEAD with the real record")
|
| 213 |
check("Brook 2026** — UNVERIFIABLE" in rep and "different work" in rep, "Brook unverifiable with the reason")
|
| 214 |
check("— REAL." not in rep, "no line says REAL any more")
|
| 215 |
+
check("A. Author et al. 2025b** — CONFIRMED" in rep and "Hollis 1972** — CONFIRMED" in rep, "confirmed lines (the cited 2025b keeps its letter)")
|
| 216 |
+
check("UNCONFIRMED" in rep and "not evidence of anything" in rep, "the text-match level is explained as no evidence")
|
| 217 |
check(re.search(r"Ferreira[^\n]*— CONFIRMED\.[^\n]*translated or variant title", rep) is not None, "the variant title is shown on the confirmed line")
|
| 218 |
cv.save_citation_report("ICSAC-SUB-TEST-1", rep_in, rep)
|
| 219 |
import review
|
|
|
|
| 282 |
check("{{curator_findings}}" in raw and "Curation team findings" in raw, f"{tpl}: findings section present")
|
| 283 |
subj, body = na._render("submission_revise_upload.md", {
|
| 284 |
"icsac_submission_id": "ICSAC-SUB-TEST-1", "title": "T", "author_name": "A",
|
| 285 |
+
"attachments_block": "(att)", "compaction_disclosure": "(disc)", "revision_line": "",
|
| 286 |
"curator_findings": "1. Fix the unit in Table 2.\n2. Define the sample window."})
|
| 287 |
check("1. Fix the unit in Table 2." in body and "the decision is the curation team's" in body, "findings rendered")
|
| 288 |
subj, body = na._render("submission_revise_upload.md", {
|
| 289 |
"icsac_submission_id": "ICSAC-SUB-TEST-1", "title": "T", "author_name": "A",
|
| 290 |
+
"attachments_block": "(att)", "compaction_disclosure": "(disc)", "revision_line": "",
|
| 291 |
"curator_findings": "[CURATION TEAM FINDINGS - x]"})
|
| 292 |
check("[CURATION TEAM FINDINGS" in body, "empty findings leave the loud bracket")
|
| 293 |
|
| 294 |
+
# ── web pages cited by URL ───────────────────────────────────────
|
| 295 |
+
print("[11] web sources")
|
| 296 |
+
check(not cv._title_matches("Data Policy", "Research Data Policy and Practice: An Overview"),
|
| 297 |
+
"a two-word title no longer matches a longer title that contains its words")
|
| 298 |
+
check(not cv._title_matches("Code of Conduct", "A Code of Conduct for Research Integrity"), "nor one with a different main title")
|
| 299 |
+
check(cv._title_matches("Spin glasses", "Spin Glasses: Experimental Facts, Theoretical Concepts, and Open Questions"),
|
| 300 |
+
"a short cited title still matches its record's main title")
|
| 301 |
+
check(cv._title_matches("Less is enough: Small samples and the nature of field notes", "Less Is Enough"),
|
| 302 |
+
"a short registry title still matches the cited main title")
|
| 303 |
+
WEBREC = {
|
| 304 |
+
"Style Guide": {"resolver": "crossref", "resolved_id": "10.0000/journal.style",
|
| 305 |
+
"title": "Style Guide", "abstract": "", "year": 2026, "authors": []},
|
| 306 |
+
"Code of Conduct": {"resolver": "crossref", "resolved_id": "10.0000/book.conduct",
|
| 307 |
+
"title": "A Code of Conduct for Research Integrity", "abstract": "", "year": 2026, "authors": []},
|
| 308 |
+
"Tide tables": {"resolver": "crossref", "resolved_id": "10.0000/paper.tides",
|
| 309 |
+
"title": "Tide tables for small harbours", "abstract": "", "year": 2021,
|
| 310 |
+
"authors": ["Jane Smith"]},
|
| 311 |
+
}
|
| 312 |
+
S2REC = {"resolver": "semanticscholar", "resolved_id": "10.0000/data.policy",
|
| 313 |
+
"title": "Research Data Policy and Practice: An Overview",
|
| 314 |
+
"abstract": "", "year": 2026, "authors": ["Q. Other"]}
|
| 315 |
+
old_bib, old_s2 = cv._search_crossref_bibliographic, cv._search_semanticscholar
|
| 316 |
+
cv._search_crossref_bibliographic = lambda raw, year=None: next(
|
| 317 |
+
(dict(r) for k, r in WEBREC.items() if k in raw), None)
|
| 318 |
+
cv._search_semanticscholar = lambda q, year=None: dict(S2REC) if "Data Policy" in q else None
|
| 319 |
+
inst = "Example Society"
|
| 320 |
+
|
| 321 |
+
def web(org, year, page, slug, host="example.org"):
|
| 322 |
+
return {"raw": f'{org}. {year}. "{page}." https://{host}/{slug}/.',
|
| 323 |
+
"authors": [org], "year": int(str(year)[:4]), "title": page, "doi": None, "arxiv_id": None,
|
| 324 |
+
"type": "url", "claim_context": ""}
|
| 325 |
+
guide, proc = web(inst, "2025a", "Style Guide", "style-guide"), web(inst, "2025b", "Data Policy", "data-policy")
|
| 326 |
+
eth = web("Sample Board", 2024, "Code of Conduct", "code-of-conduct", "example.net")
|
| 327 |
+
rules = web("Sample Board", 2023, "Membership Rules", "rules", "example.net")
|
| 328 |
+
for name, c in (("same-titled journal page", guide), ("S2 title-only hit", proc), ("reordered-title hit", eth),
|
| 329 |
+
("no registry hit at all", rules)):
|
| 330 |
+
v = cv.verify_citation(dict(c))
|
| 331 |
+
check(v["confidence"] == "web-source" and v["verified"] is False and v["resolved_id"].startswith("https://example.")
|
| 332 |
+
and "10.0000" not in json.dumps(v), f"{name}: WEB SOURCE, no DOI matched to it")
|
| 333 |
+
paper = {"raw": "Smith, J. 2021. Tide tables for small harbours. https://example.org/smith2021.pdf",
|
| 334 |
+
"authors": ["J. Smith"], "year": 2021, "title": "Tide tables for small harbours",
|
| 335 |
+
"doi": None, "arxiv_id": None, "type": "url", "claim_context": ""}
|
| 336 |
+
v = cv.verify_citation(dict(paper))
|
| 337 |
+
check(v["verified"] and v["confidence"] == "title-author-match" and v["resolved_id"] == "10.0000/paper.tides",
|
| 338 |
+
"a paper cited by URL that the registry matches on title AND author stays MATCHED")
|
| 339 |
+
check(cv._cited_web_url({"type": "url", "raw": "x https://doi.org/10.1/x"}) is None
|
| 340 |
+
and cv._cited_web_url({"type": "title-only", "raw": "x https://ssrn.com/abstract=1"}) is None,
|
| 341 |
+
"doi.org links and non-url types are not web sources")
|
| 342 |
+
stale = dict(proc, verified=True, confidence="title-only-match", resolved_id="10.0000/data.policy")
|
| 343 |
+
res = cv.verify_all([stale, dict(guide), dict(eth), dict(rules), dict(and_)])
|
| 344 |
+
check(res[0]["confidence"] == "web-source" and res[0]["verified"] is False,
|
| 345 |
+
"a re-check replaces an earlier fuzzy match on a web page")
|
| 346 |
+
rep = cv.build_verification_report(res)
|
| 347 |
+
check("### Cited by URL" in rep and "WEB SOURCE:" in rep, "web section + legend")
|
| 348 |
+
check(f"{inst} 2025b** — WEB SOURCE. *Data Policy*, https://example.org/data-policy/." in rep,
|
| 349 |
+
"the line names the cited page and its URL")
|
| 350 |
+
check("Practice: An Overview" not in rep and "10.0000/data.policy" not in rep and "MATCHED." not in rep,
|
| 351 |
+
"nothing unrelated is matched in the report")
|
| 352 |
+
check("### Unverifiable from public registries" not in rep, "web sources are not listed as unverifiable")
|
| 353 |
+
check(cv._short_label(guide) != cv._short_label(proc), "2025a and 2025b get distinct labels")
|
| 354 |
+
import citation_misattribution as cm
|
| 355 |
+
check(cm._short_label(guide).endswith("2025a"), "the misattribution labels carry the letter too")
|
| 356 |
+
check(not [c for c in res if c.get("confidence") == "web-source" and c.get("verified")],
|
| 357 |
+
"web sources are not verified, so the misattribution check never sees them")
|
| 358 |
+
cv.save_citation_report("ICSAC-SUB-TEST-2", res, rep)
|
| 359 |
+
line = review._citation_header_line("ICSAC-SUB-TEST-2")
|
| 360 |
+
check(line is not None and line.startswith("1/1 resolved · 4 cited by URL"), f"header line ({line})")
|
| 361 |
+
import review_quality_control as rqcm
|
| 362 |
+
ev = rqcm.build_evidence_block("ICSAC-SUB-TEST-2", config.REVIEWS_DIR)
|
| 363 |
+
check("0 UNVERIFIABLE, 4 WEB SOURCE" in ev and "— WEB SOURCE: https://example.org/data-policy/" in ev
|
| 364 |
+
and "every reference was confirmed" not in ev, "the RQC evidence names web sources as such")
|
| 365 |
+
cv._search_crossref_bibliographic, cv._search_semanticscholar = old_bib, old_s2
|
| 366 |
+
|
| 367 |
finally:
|
| 368 |
shutil.rmtree(TMP, ignore_errors=True)
|
| 369 |
|
|
@@ -281,7 +281,7 @@ try:
|
|
| 281 |
(site / "src" / "data" / "accepted.json").write_text("[]")
|
| 282 |
pub.WEBSITE_REPO = str(site); pub.REGISTRY_PATH = str(site / "src" / "data" / "accepted.json")
|
| 283 |
check(pub.code_data_url(sub) == "https://github.com/example/generator", "registry: helper reads the form link")
|
| 284 |
-
e = pub.upsert_entry(approved=True, proto={"title": "T", "authors": ["A"], "doi": "10.
|
| 285 |
"source": "submission-pdf", "code_data_url": pub.code_data_url(sub)})
|
| 286 |
check(e.get("code_data_url") == "https://github.com/example/generator", "registry: the entry keeps code_data_url")
|
| 287 |
check(pub.code_data_url({"code_data": {"available": False, "url": None}}) is None, "registry: 'no' gives no link")
|
|
|
|
| 281 |
(site / "src" / "data" / "accepted.json").write_text("[]")
|
| 282 |
pub.WEBSITE_REPO = str(site); pub.REGISTRY_PATH = str(site / "src" / "data" / "accepted.json")
|
| 283 |
check(pub.code_data_url(sub) == "https://github.com/example/generator", "registry: helper reads the form link")
|
| 284 |
+
e = pub.upsert_entry(approved=True, proto={"title": "T", "authors": ["A"], "doi": "10.5555/icsac.2026.999",
|
| 285 |
"source": "submission-pdf", "code_data_url": pub.code_data_url(sub)})
|
| 286 |
check(e.get("code_data_url") == "https://github.com/example/generator", "registry: the entry keeps code_data_url")
|
| 287 |
check(pub.code_data_url({"code_data": {"available": False, "url": None}}) is None, "registry: 'no' gives no link")
|
|
@@ -33,7 +33,7 @@ ZEN = "https://doi.org/10.5281/zenodo.23042786"
|
|
| 33 |
|
| 34 |
|
| 35 |
def build(sub: dict, content_type: str = "journal-article") -> str:
|
| 36 |
-
xml = cd.build_deposit_xml(sub, doi="10.
|
| 37 |
landing_url="https://icsacinstitute.org/publications/a-test-paper",
|
| 38 |
pdf_url="https://icsacinstitute.org/papers/ICSAC-SUB-00099.pdf",
|
| 39 |
content_type=content_type)
|
|
@@ -95,12 +95,12 @@ try:
|
|
| 95 |
x = build(dict(BASE))
|
| 96 |
jm = x[x.find("<journal_metadata"):x.find("</journal_metadata>")]
|
| 97 |
check(bool(x) and "<doi_data" not in jm, "unset: journal_metadata carries no doi_data (the deposit as before)")
|
| 98 |
-
config.CROSSREF_JOURNAL_DOI = "10.
|
| 99 |
x = build(dict(BASE, code_data={"available": True, "url": ZEN}))
|
| 100 |
jm = x[x.find("<journal_metadata"):x.find("</journal_metadata>")]
|
| 101 |
-
check(bool(x) and "<doi>10.
|
| 102 |
"set: journal_metadata carries the title DOI and the journal page, and validates")
|
| 103 |
-
check(x.count("<doi>10.
|
| 104 |
finally:
|
| 105 |
if saved is None:
|
| 106 |
delattr(config, "CROSSREF_JOURNAL_DOI") if hasattr(config, "CROSSREF_JOURNAL_DOI") else None
|
|
@@ -110,14 +110,14 @@ finally:
|
|
| 110 |
print("4. the Zenodo archive copy follows the Crossref record")
|
| 111 |
import repository_deposit as rd # noqa: E402
|
| 112 |
sub = dict(BASE, resource_type="preprint", publication_date="2026-09-29")
|
| 113 |
-
md = rd._build_metadata(sub, external_doi="10.
|
| 114 |
check(md.get("upload_type") == "publication" and md.get("publication_type") == "article",
|
| 115 |
"with an ICSAC DOI, a form 'preprint' becomes a journal article")
|
| 116 |
check(md.get("journal_title") == getattr(config, "CROSSREF_JOURNAL_TITLE", "Persistence")
|
| 117 |
and md.get("journal_volume") == str(getattr(config, "CROSSREF_JOURNAL_VOLUME", "1")),
|
| 118 |
"it names the journal and the volume from config")
|
| 119 |
-
check(md.get("doi") == "10.
|
| 120 |
-
md = rd._build_metadata(dict(sub, resource_type="dataset"), external_doi="10.
|
| 121 |
check(md.get("upload_type") == "publication" and md.get("publication_type") == "article",
|
| 122 |
"any form type is overridden once the work is an ICSAC-registered article")
|
| 123 |
md = rd._build_metadata(sub)
|
|
@@ -126,7 +126,7 @@ check(md.get("publication_type") == "preprint" and "journal_title" not in md,
|
|
| 126 |
saved_ct = getattr(config, "CROSSREF_CONTENT_TYPE", None)
|
| 127 |
try:
|
| 128 |
config.CROSSREF_CONTENT_TYPE = "report-paper"
|
| 129 |
-
md = rd._build_metadata(sub, external_doi="10.
|
| 130 |
check(md.get("publication_type") == "report" and "journal_title" not in md,
|
| 131 |
"a report-paper deployment archives a report, with no journal fields")
|
| 132 |
finally:
|
|
|
|
| 33 |
|
| 34 |
|
| 35 |
def build(sub: dict, content_type: str = "journal-article") -> str:
|
| 36 |
+
xml = cd.build_deposit_xml(sub, doi="10.5555/icsac.2026.099",
|
| 37 |
landing_url="https://icsacinstitute.org/publications/a-test-paper",
|
| 38 |
pdf_url="https://icsacinstitute.org/papers/ICSAC-SUB-00099.pdf",
|
| 39 |
content_type=content_type)
|
|
|
|
| 95 |
x = build(dict(BASE))
|
| 96 |
jm = x[x.find("<journal_metadata"):x.find("</journal_metadata>")]
|
| 97 |
check(bool(x) and "<doi_data" not in jm, "unset: journal_metadata carries no doi_data (the deposit as before)")
|
| 98 |
+
config.CROSSREF_JOURNAL_DOI = "10.5555/persistence"
|
| 99 |
x = build(dict(BASE, code_data={"available": True, "url": ZEN}))
|
| 100 |
jm = x[x.find("<journal_metadata"):x.find("</journal_metadata>")]
|
| 101 |
+
check(bool(x) and "<doi>10.5555/persistence</doi>" in jm and "<resource>https://icsacinstitute.org/journal/</resource>" in jm,
|
| 102 |
"set: journal_metadata carries the title DOI and the journal page, and validates")
|
| 103 |
+
check(x.count("<doi>10.5555/icsac.2026.099</doi>") == 1, "the article keeps its own DOI")
|
| 104 |
finally:
|
| 105 |
if saved is None:
|
| 106 |
delattr(config, "CROSSREF_JOURNAL_DOI") if hasattr(config, "CROSSREF_JOURNAL_DOI") else None
|
|
|
|
| 110 |
print("4. the Zenodo archive copy follows the Crossref record")
|
| 111 |
import repository_deposit as rd # noqa: E402
|
| 112 |
sub = dict(BASE, resource_type="preprint", publication_date="2026-09-29")
|
| 113 |
+
md = rd._build_metadata(sub, external_doi="10.5555/icsac.2026.099")
|
| 114 |
check(md.get("upload_type") == "publication" and md.get("publication_type") == "article",
|
| 115 |
"with an ICSAC DOI, a form 'preprint' becomes a journal article")
|
| 116 |
check(md.get("journal_title") == getattr(config, "CROSSREF_JOURNAL_TITLE", "Persistence")
|
| 117 |
and md.get("journal_volume") == str(getattr(config, "CROSSREF_JOURNAL_VOLUME", "1")),
|
| 118 |
"it names the journal and the volume from config")
|
| 119 |
+
check(md.get("doi") == "10.5555/icsac.2026.099", "the external DOI still rides on the draft")
|
| 120 |
+
md = rd._build_metadata(dict(sub, resource_type="dataset"), external_doi="10.5555/icsac.2026.099")
|
| 121 |
check(md.get("upload_type") == "publication" and md.get("publication_type") == "article",
|
| 122 |
"any form type is overridden once the work is an ICSAC-registered article")
|
| 123 |
md = rd._build_metadata(sub)
|
|
|
|
| 126 |
saved_ct = getattr(config, "CROSSREF_CONTENT_TYPE", None)
|
| 127 |
try:
|
| 128 |
config.CROSSREF_CONTENT_TYPE = "report-paper"
|
| 129 |
+
md = rd._build_metadata(sub, external_doi="10.5555/icsac.2026.099")
|
| 130 |
check(md.get("publication_type") == "report" and "journal_title" not in md,
|
| 131 |
"a report-paper deployment archives a report, with no journal fields")
|
| 132 |
finally:
|
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""The author-facing compaction disclosure states only what was measured. The
|
| 3 |
+
reference-entry count is a heuristic that undercounts author-year lists, so the
|
| 4 |
+
letter names the removed references list by its size, never by an entry count.
|
| 5 |
+
Offline, no files written. Run from the repo root with the venv:
|
| 6 |
+
.venv/bin/python intake/scripts/test_disclosure_wording.py
|
| 7 |
+
"""
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
|
| 10 |
+
import sys
|
| 11 |
+
from pathlib import Path
|
| 12 |
+
|
| 13 |
+
ROOT = Path(__file__).resolve().parents[2]
|
| 14 |
+
sys.path.insert(0, str(ROOT))
|
| 15 |
+
|
| 16 |
+
import review_compaction as rc # noqa: E402
|
| 17 |
+
|
| 18 |
+
failures: list[str] = []
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def check(cond: bool, msg: str) -> None:
|
| 22 |
+
print(("ok " if cond else "FAIL ") + msg)
|
| 23 |
+
if not cond:
|
| 24 |
+
failures.append(msg)
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
man = {"author_names": ["A. Author"], "references_count": 2, "references_section_chars": 6430}
|
| 28 |
+
out = rc.render_manifest(man)
|
| 29 |
+
ref = [line for line in out.splitlines() if "References list" in line]
|
| 30 |
+
check(len(ref) == 1, "one references line")
|
| 31 |
+
check(bool(ref) and "6430 characters" in ref[0], "the line gives the size of the removed list")
|
| 32 |
+
check(bool(ref) and "item" not in ref[0], "the line carries no entry count")
|
| 33 |
+
|
| 34 |
+
out0 = rc.render_manifest({"author_names": ["A. Author"]})
|
| 35 |
+
check("References list" not in out0, "no references line when no list was removed")
|
| 36 |
+
|
| 37 |
+
sys.exit(1 if failures else 0)
|
|
@@ -0,0 +1,107 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Checks that the system never looks up one of its own DOIs before Crossref
|
| 3 |
+
has registered it. A doi.org lookup of a DOI that does not exist yet counts as
|
| 4 |
+
a failed resolution in Crossref's monthly report for the prefix, so:
|
| 5 |
+
|
| 6 |
+
- curator pings go out with link previews off: Telegram would otherwise
|
| 7 |
+
fetch every link in a message, a doi.org link included;
|
| 8 |
+
- test fixtures never use the institute's live prefix: a fixture DOI under it
|
| 9 |
+
names a paper that does not exist, or a real one once the sequence gets there.
|
| 10 |
+
|
| 11 |
+
Offline: the Telegram request is captured, never sent. Run from the repo root:
|
| 12 |
+
.venv/bin/python intake/scripts/test_doi_hygiene.py
|
| 13 |
+
"""
|
| 14 |
+
from __future__ import annotations
|
| 15 |
+
|
| 16 |
+
import json
|
| 17 |
+
import re
|
| 18 |
+
import sys
|
| 19 |
+
from pathlib import Path
|
| 20 |
+
|
| 21 |
+
ROOT = Path(__file__).resolve().parents[2]
|
| 22 |
+
sys.path.insert(0, str(ROOT))
|
| 23 |
+
|
| 24 |
+
failures: list[str] = []
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def check(cond: bool, name: str) -> None:
|
| 28 |
+
print((" ok " if cond else " FAIL ") + name)
|
| 29 |
+
if not cond:
|
| 30 |
+
failures.append(name)
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
import config # noqa: E402
|
| 34 |
+
import notify # noqa: E402
|
| 35 |
+
|
| 36 |
+
# ── 1. Telegram link previews are off ────────────────────────────────────────
|
| 37 |
+
sent: list[dict] = []
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
class _Resp:
|
| 41 |
+
status = 200
|
| 42 |
+
|
| 43 |
+
def __init__(self, n: int):
|
| 44 |
+
self._n = n
|
| 45 |
+
|
| 46 |
+
def read(self) -> bytes:
|
| 47 |
+
return json.dumps({"ok": True, "result": {"message_id": self._n}}).encode()
|
| 48 |
+
|
| 49 |
+
def __enter__(self):
|
| 50 |
+
return self
|
| 51 |
+
|
| 52 |
+
def __exit__(self, *exc):
|
| 53 |
+
return False
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def fake_urlopen(req, timeout=None):
|
| 57 |
+
sent.append({"url": req.full_url, "body": json.loads(req.data.decode())})
|
| 58 |
+
return _Resp(len(sent))
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
real_urlopen = notify.urllib.request.urlopen
|
| 62 |
+
saved = {k: getattr(config, k, None) for k in ("TELEGRAM_TOKEN", "TELEGRAM_CHAT_ID", "TELEGRAM_THREAD_ID")}
|
| 63 |
+
notify.urllib.request.urlopen = fake_urlopen
|
| 64 |
+
try:
|
| 65 |
+
config.TELEGRAM_TOKEN, config.TELEGRAM_CHAT_ID, config.TELEGRAM_THREAD_ID = "t0k", "-100", "7"
|
| 66 |
+
text = "DOI REGISTERED for ICSAC-SUB-TEST-1: https://doi.org/10.5555/icsac.2026.999"
|
| 67 |
+
check(notify.send_telegram(text, parse_mode=None) == 1, "plain ping: sent, message id returned")
|
| 68 |
+
check(notify.send_telegram(text) == 2, "Markdown ping: sent")
|
| 69 |
+
check(notify.send_telegram(text, parse_mode=None, chat_override="-200", thread_override="9") == 3,
|
| 70 |
+
"test-tier ping: sent to the override chat")
|
| 71 |
+
check(notify.send_telegram(text, chat_override="__suppress__") is None and len(sent) == 3,
|
| 72 |
+
"suppressed ping: nothing sent")
|
| 73 |
+
check(len(sent) == 3 and all(s["url"].endswith("/sendMessage") for s in sent),
|
| 74 |
+
"three requests, all sendMessage")
|
| 75 |
+
check(all(s["body"].get("link_preview_options") == {"is_disabled": True} for s in sent),
|
| 76 |
+
"every request turns link previews off")
|
| 77 |
+
check(all(s["body"].get("text") == text for s in sent), "the text goes out unchanged")
|
| 78 |
+
check(sent[1]["body"].get("parse_mode") == "Markdown" and "parse_mode" not in sent[0]["body"],
|
| 79 |
+
"parse mode still applies only when asked")
|
| 80 |
+
check(sent[0]["body"].get("message_thread_id") == 7
|
| 81 |
+
and sent[2]["body"].get("chat_id") == "-200" and sent[2]["body"].get("message_thread_id") == 9,
|
| 82 |
+
"the configured thread and the test-tier override still apply")
|
| 83 |
+
finally:
|
| 84 |
+
notify.urllib.request.urlopen = real_urlopen
|
| 85 |
+
for k, v in saved.items():
|
| 86 |
+
setattr(config, k, v)
|
| 87 |
+
|
| 88 |
+
# ── 2. no test fixture uses the live prefix ──────────────────────────────────
|
| 89 |
+
prefix = str(getattr(config, "CROSSREF_PREFIX", "") or "")
|
| 90 |
+
check(bool(re.fullmatch(r"10\.\d{4,9}", prefix)), f"the live prefix is configured ({prefix or 'unset'})")
|
| 91 |
+
here = Path(__file__).resolve()
|
| 92 |
+
hits = []
|
| 93 |
+
for t in sorted(here.parent.glob("test_*.py")):
|
| 94 |
+
if t == here:
|
| 95 |
+
continue
|
| 96 |
+
for n, line in enumerate(t.read_text(errors="replace").splitlines(), 1):
|
| 97 |
+
if prefix and prefix + "/" in line:
|
| 98 |
+
hits.append(f"{t.name}:{n}")
|
| 99 |
+
check(not hits, "no fixture DOI under the live prefix" + (f": {', '.join(hits[:8])}" if hits else ""))
|
| 100 |
+
|
| 101 |
+
print()
|
| 102 |
+
if failures:
|
| 103 |
+
print(f"FAILED: {len(failures)}")
|
| 104 |
+
for f in failures:
|
| 105 |
+
print(" - " + f)
|
| 106 |
+
sys.exit(1)
|
| 107 |
+
print("ALL GREEN")
|
|
@@ -143,7 +143,7 @@ try:
|
|
| 143 |
sub = {"sub_id": "ICSAC-SUB-00099", "title": "T", "abstract": "A.", "source": "doi", "doi": "2401.12345",
|
| 144 |
"license": "cc-by-sa-4.0", "article_license": "cc-by-sa-4.0", "preprint_license": "",
|
| 145 |
"creators": [{"name": "Test, Author"}]}
|
| 146 |
-
xml = cd.build_deposit_xml(sub, doi="10.
|
| 147 |
pdf_url=None, content_type="journal-article")
|
| 148 |
cd.validate_xml(xml)
|
| 149 |
check(b"creativecommons.org/licenses/by-sa/4.0" in xml and b"hasPreprint" in xml,
|
|
|
|
| 143 |
sub = {"sub_id": "ICSAC-SUB-00099", "title": "T", "abstract": "A.", "source": "doi", "doi": "2401.12345",
|
| 144 |
"license": "cc-by-sa-4.0", "article_license": "cc-by-sa-4.0", "preprint_license": "",
|
| 145 |
"creators": [{"name": "Test, Author"}]}
|
| 146 |
+
xml = cd.build_deposit_xml(sub, doi="10.5555/icsac.2026.099", landing_url="https://icsacinstitute.org/publications/t",
|
| 147 |
pdf_url=None, content_type="journal-article")
|
| 148 |
cd.validate_xml(xml)
|
| 149 |
check(b"creativecommons.org/licenses/by-sa/4.0" in xml and b"hasPreprint" in xml,
|
|
@@ -53,7 +53,7 @@ try:
|
|
| 53 |
base = {"sub_id": "ICSAC-SUB-00099", "title": "A test paper", "abstract": "An abstract.",
|
| 54 |
"license": "cc-by-4.0", "creators": [{"name": "Test, Author", "orcid": "0000-0002-1825-0097"}]}
|
| 55 |
doi_sub = dict(base, source="doi", doi="10.5281/zenodo.123456")
|
| 56 |
-
xml = cd.build_deposit_xml(doi_sub, doi="10.
|
| 57 |
landing_url="https://icsacinstitute.org/publications/a-test-paper",
|
| 58 |
pdf_url="https://icsacinstitute.org/papers/ICSAC-SUB-00099.pdf",
|
| 59 |
content_type="journal-article")
|
|
@@ -67,12 +67,12 @@ try:
|
|
| 67 |
"the deposit links the preprint (hasPreprint)")
|
| 68 |
art = x[x.find("<journal_article"):] # journal_metadata may carry the journal title DOI's own doi_data
|
| 69 |
check(0 < art.find("<rel:program") < art.find("<doi_data"), "the relation sits before the article's doi_data")
|
| 70 |
-
up = cd.build_deposit_xml(dict(base, source="upload"), doi="10.
|
| 71 |
landing_url="https://icsacinstitute.org/publications/x", pdf_url=None,
|
| 72 |
content_type="journal-article")
|
| 73 |
cd.validate_xml(up)
|
| 74 |
check(b"hasPreprint" not in up, "an upload deposit carries no preprint relation")
|
| 75 |
-
rp = cd.build_deposit_xml(doi_sub, doi="10.
|
| 76 |
landing_url="https://icsacinstitute.org/publications/y", pdf_url=None,
|
| 77 |
content_type="report-paper")
|
| 78 |
try:
|
|
@@ -102,7 +102,7 @@ try:
|
|
| 102 |
check("has no other DOI and is not under review elsewhere" in t and "preprint" not in t.lower(),
|
| 103 |
"upload route keeps its no-other-DOI line")
|
| 104 |
ok, _ = na.send_published(to="author@example.com", sub_id="ICSAC-SUB-TEST-PUB", title="A test paper",
|
| 105 |
-
author_name="Test Author", deposit_doi="10.
|
| 106 |
deposit_url="https://icsacinstitute.org/publications/a-test-paper",
|
| 107 |
publications_url="https://icsacinstitute.org/publications/a-test-paper",
|
| 108 |
preprint_doi="10.5281/zenodo.123456", stay_involved=False, tier=2)
|
|
|
|
| 53 |
base = {"sub_id": "ICSAC-SUB-00099", "title": "A test paper", "abstract": "An abstract.",
|
| 54 |
"license": "cc-by-4.0", "creators": [{"name": "Test, Author", "orcid": "0000-0002-1825-0097"}]}
|
| 55 |
doi_sub = dict(base, source="doi", doi="10.5281/zenodo.123456")
|
| 56 |
+
xml = cd.build_deposit_xml(doi_sub, doi="10.5555/icsac.2026.099",
|
| 57 |
landing_url="https://icsacinstitute.org/publications/a-test-paper",
|
| 58 |
pdf_url="https://icsacinstitute.org/papers/ICSAC-SUB-00099.pdf",
|
| 59 |
content_type="journal-article")
|
|
|
|
| 67 |
"the deposit links the preprint (hasPreprint)")
|
| 68 |
art = x[x.find("<journal_article"):] # journal_metadata may carry the journal title DOI's own doi_data
|
| 69 |
check(0 < art.find("<rel:program") < art.find("<doi_data"), "the relation sits before the article's doi_data")
|
| 70 |
+
up = cd.build_deposit_xml(dict(base, source="upload"), doi="10.5555/icsac.2026.098",
|
| 71 |
landing_url="https://icsacinstitute.org/publications/x", pdf_url=None,
|
| 72 |
content_type="journal-article")
|
| 73 |
cd.validate_xml(up)
|
| 74 |
check(b"hasPreprint" not in up, "an upload deposit carries no preprint relation")
|
| 75 |
+
rp = cd.build_deposit_xml(doi_sub, doi="10.5555/icsac.2026.097",
|
| 76 |
landing_url="https://icsacinstitute.org/publications/y", pdf_url=None,
|
| 77 |
content_type="report-paper")
|
| 78 |
try:
|
|
|
|
| 102 |
check("has no other DOI and is not under review elsewhere" in t and "preprint" not in t.lower(),
|
| 103 |
"upload route keeps its no-other-DOI line")
|
| 104 |
ok, _ = na.send_published(to="author@example.com", sub_id="ICSAC-SUB-TEST-PUB", title="A test paper",
|
| 105 |
+
author_name="Test Author", deposit_doi="10.5555/icsac.2026.099",
|
| 106 |
deposit_url="https://icsacinstitute.org/publications/a-test-paper",
|
| 107 |
publications_url="https://icsacinstitute.org/publications/a-test-paper",
|
| 108 |
preprint_doi="10.5281/zenodo.123456", stay_involved=False, tier=2)
|
|
@@ -0,0 +1,59 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""The HTML part of an email shows the templates' header block and sign-off
|
| 3 |
+
one item per line, as the plain-text part does; paragraphs stay paragraphs.
|
| 4 |
+
Every template, both template sets. Offline, no files written. Run from the
|
| 5 |
+
repo root with the venv:
|
| 6 |
+
.venv/bin/python intake/scripts/test_email_line_breaks.py
|
| 7 |
+
"""
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
|
| 10 |
+
import re
|
| 11 |
+
import sys
|
| 12 |
+
from pathlib import Path
|
| 13 |
+
|
| 14 |
+
ROOT = Path(__file__).resolve().parents[2]
|
| 15 |
+
sys.path.insert(0, str(ROOT))
|
| 16 |
+
|
| 17 |
+
import email_send # noqa: E402
|
| 18 |
+
|
| 19 |
+
failures: list[str] = []
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def check(cond: bool, msg: str) -> None:
|
| 23 |
+
print(("ok " if cond else "FAIL ") + msg)
|
| 24 |
+
if not cond:
|
| 25 |
+
failures.append(msg)
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
html = email_send._markdown_to_html(
|
| 29 |
+
"The review is complete.\n\n"
|
| 30 |
+
"**Submission ID:** ICSAC-SUB-00002\n**Revision of:** ICSAC-SUB-00001\n**Title:** T\n\n"
|
| 31 |
+
"Next paragraph.\n\n— ICSAC\nInstitute for Complexity Science and Advanced Computing")
|
| 32 |
+
check("<strong>Submission ID:</strong> ICSAC-SUB-00002<br />\n<strong>Revision of:</strong> ICSAC-SUB-00001<br />\n"
|
| 33 |
+
"<strong>Title:</strong> T</p>" in html, "the header block is one item per line")
|
| 34 |
+
check("— ICSAC<br />\nInstitute for Complexity" in html, "the sign-off is one item per line")
|
| 35 |
+
check("<p>The review is complete.</p>" in html and "<p>Next paragraph.</p>" in html,
|
| 36 |
+
"paragraphs stay separate paragraphs")
|
| 37 |
+
check("<" in email_send._markdown_to_html("a <b>x</b>"), "a substituted < is still text")
|
| 38 |
+
|
| 39 |
+
# Every soft line break in every template comes out as a <br />.
|
| 40 |
+
dirs = [ROOT / "intake" / "templates", ROOT / "templates"]
|
| 41 |
+
n_templates = n_breaks = 0
|
| 42 |
+
for d in dirs:
|
| 43 |
+
for f in sorted(d.glob("*.md")):
|
| 44 |
+
lines = [ln for ln in f.read_text().splitlines() if not ln.lower().startswith("subject:")]
|
| 45 |
+
body = "\n".join(lines)
|
| 46 |
+
out = email_send._markdown_to_html(body)
|
| 47 |
+
n_templates += 1
|
| 48 |
+
for a, b in zip(lines, lines[1:]):
|
| 49 |
+
if not a.strip() or not b.strip():
|
| 50 |
+
continue
|
| 51 |
+
if re.match(r"\s*([-*] |\d+\. |#|>|\||---)", a) or re.match(r"\s*([-*] |\d+\. |#|\||---)", b):
|
| 52 |
+
continue
|
| 53 |
+
n_breaks += 1
|
| 54 |
+
tail = re.sub(r"\{\{\w+\}\}|\*\*|\[|\]\([^)]*\)|\]", "", a).strip()[-12:]
|
| 55 |
+
check(bool(tail) and re.search(re.escape(tail) + r"[^\n]*<br />", out) is not None,
|
| 56 |
+
f"{d.name}/{f.name}: the break after ...{tail!r} is kept")
|
| 57 |
+
check(n_templates >= 20 and n_breaks >= 30, f"looked at {n_templates} templates, {n_breaks} line breaks")
|
| 58 |
+
|
| 59 |
+
sys.exit(1 if failures else 0)
|
|
@@ -0,0 +1,156 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Evidence fails closed. Offline: every registry, model and CiteStamp call is
|
| 3 |
+
stubbed; one temporary directory, removed at the end, pass or fail. Run from
|
| 4 |
+
the repo root with the venv:
|
| 5 |
+
.venv/bin/python intake/scripts/test_evidence_fail_closed.py
|
| 6 |
+
|
| 7 |
+
Only a reference whose identity is confirmed by its own identifier can ground
|
| 8 |
+
an integrity finding. A record found by text search for a reference with no
|
| 9 |
+
identifier is UNCONFIRMED: the panel and the review-quality auditor see the
|
| 10 |
+
level and never the record; the misattribution check never judges it;
|
| 11 |
+
CiteStamp is never asked about it (nor about a DOI that points to another
|
| 12 |
+
work); a deposit never links it as a <doi>. And blind-review compaction lets a
|
| 13 |
+
manuscript through with no author spans only when none of the submission's own
|
| 14 |
+
names, ORCID iDs, emails or affiliations occurs in its text.
|
| 15 |
+
"""
|
| 16 |
+
from __future__ import annotations
|
| 17 |
+
|
| 18 |
+
import shutil
|
| 19 |
+
import sys
|
| 20 |
+
import tempfile
|
| 21 |
+
import xml.etree.ElementTree as ET
|
| 22 |
+
from pathlib import Path
|
| 23 |
+
|
| 24 |
+
ROOT = Path(__file__).resolve().parents[2]
|
| 25 |
+
sys.path.insert(0, str(ROOT))
|
| 26 |
+
|
| 27 |
+
failures: list[str] = []
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def check(cond: bool, msg: str) -> None:
|
| 31 |
+
print((" ok " if cond else " FAIL ") + msg)
|
| 32 |
+
if not cond:
|
| 33 |
+
failures.append(msg)
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
TMP = Path(tempfile.mkdtemp(prefix="icsac-failclosed-"))
|
| 37 |
+
try:
|
| 38 |
+
import config
|
| 39 |
+
config.REVIEWS_DIR = str(TMP / "reviews")
|
| 40 |
+
import citation_verify as cv
|
| 41 |
+
import citation_misattribution as cm
|
| 42 |
+
import citestamp_check as cc
|
| 43 |
+
cc.COVERAGE_GAP_LOG = str(TMP / "reviews" / "gaps.jsonl")
|
| 44 |
+
cc.CALL_GAP = 0
|
| 45 |
+
|
| 46 |
+
confirmed = {"raw": "Holm, P. (1999). Ripples on sand. Field Notes, 3, 1-9.", "authors": ["P. Holm"], "year": 1999,
|
| 47 |
+
"title": "Ripples on sand", "cited_title": "Ripples on sand", "doi": "10.0000/fn.1999.1",
|
| 48 |
+
"verified": True, "confidence": "exact-id", "doi_identity": "confirmed",
|
| 49 |
+
"resolver": "crossref", "resolved_id": "10.0000/fn.1999.1", "abstract": "ripples", "claim_context": "ripples form"}
|
| 50 |
+
book = {"raw": "Moss, R. (2001). A Field Guide to Lichens. Example Press.", "authors": ["R. Moss"], "year": 2001,
|
| 51 |
+
"title": "Review of: A Field Guide to Lichens", "cited_title": "A Field Guide to Lichens", "doi": None,
|
| 52 |
+
"verified": True, "confidence": "title-year-match", "resolver": "crossref",
|
| 53 |
+
"resolved_id": "10.0000/rev.2002.77", "abstract": "a review", "claim_context": "lichens grow slowly"}
|
| 54 |
+
with_doi_fuzzy = dict(book, raw="Moss, R. (2001b). Lichen dates. Example Press.", doi="10.0000/lich.2001",
|
| 55 |
+
confidence="title-author-match", resolved_id="10.0000/other.9")
|
| 56 |
+
dead = {"raw": "Nobody, A. (2020). Gone. J. Lost, 1.", "authors": ["A. Nobody"], "year": 2020, "title": "Gone",
|
| 57 |
+
"doi": "10.0000/gone.1", "verified": False, "confidence": "doi-dead", "doi_identity": "dead",
|
| 58 |
+
"reason": "Cited DOI 10.0000/gone.1 does not exist at doi.org."}
|
| 59 |
+
mismatch = {"raw": "Nobody, B. (2021). Wrong. J. Else, 2.", "authors": ["B. Nobody"], "year": 2021, "title": "Wrong",
|
| 60 |
+
"doi": "10.0000/else.2", "verified": False, "confidence": "doi-mismatch", "doi_identity": "mismatch",
|
| 61 |
+
"reason": "DOI 10.0000/else.2 resolves to another work."}
|
| 62 |
+
web = {"raw": "Example Society. 2025. \"Rules.\" https://example.org/rules/.", "authors": ["Example Society"],
|
| 63 |
+
"year": 2025, "title": "Rules", "verified": False, "confidence": "web-source", "resolved_id": "https://example.org/rules/"}
|
| 64 |
+
cits = [confirmed, book, with_doi_fuzzy, dead, mismatch, web]
|
| 65 |
+
|
| 66 |
+
print("[1] levels")
|
| 67 |
+
check(cv.is_identity_confirmed(confirmed) and not cv.is_identity_confirmed(book)
|
| 68 |
+
and not cv.is_identity_confirmed(with_doi_fuzzy) and not cv.is_identity_confirmed(dead), "only exact-id is confirmed")
|
| 69 |
+
check(cv.is_unconfirmed_match(book) and cv.is_unconfirmed_match(with_doi_fuzzy)
|
| 70 |
+
and not cv.is_unconfirmed_match(web) and not cv.is_unconfirmed_match(confirmed), "text matches are unconfirmed; web sources are not")
|
| 71 |
+
|
| 72 |
+
print("[2] the panel's report")
|
| 73 |
+
rep = cv.build_verification_report(cits)
|
| 74 |
+
check("UNCONFIRMED" in rep and "not evidence of anything" in rep, "level + legend")
|
| 75 |
+
check("10.0000/rev.2002.77" not in rep and "Review of:" not in rep and "10.0000/other.9" not in rep,
|
| 76 |
+
"the text-search record is never shown")
|
| 77 |
+
check("MATCHED" not in rep, "no MATCHED level left")
|
| 78 |
+
check('Submission claim context: "lichens grow slowly"' in rep, "the claim context stays")
|
| 79 |
+
check("Moss 2001** — UNCONFIRMED" in rep, "the unconfirmed line names the reference")
|
| 80 |
+
|
| 81 |
+
print("[3] misattribution sees confirmed references only")
|
| 82 |
+
sel = cm.select_load_bearing(cits, "text")
|
| 83 |
+
check([c["raw"] for c in sel] == [confirmed["raw"]], f"eligible = the confirmed one ({len(sel)})")
|
| 84 |
+
|
| 85 |
+
print("[4] CiteStamp is asked about confirmed DOIs only")
|
| 86 |
+
asked = []
|
| 87 |
+
cc.server_info = lambda: {"name": "stub"}
|
| 88 |
+
def _tool(name, identifier, limit=25, rpc_id=2):
|
| 89 |
+
asked.append(identifier)
|
| 90 |
+
return {"groundings": [], "page": {"inferred_total": 0}}
|
| 91 |
+
cc.tool = _tool
|
| 92 |
+
res = cc.check_citations(cits, log=lambda m: None)
|
| 93 |
+
check(set(asked) == {"10.0000/fn.1999.1"} and res["queried"] == 1, f"queried {sorted(set(asked))}")
|
| 94 |
+
|
| 95 |
+
print("[5] a deposit links confirmed DOIs only")
|
| 96 |
+
import crossref_deposit as cd
|
| 97 |
+
root = ET.Element("x")
|
| 98 |
+
cd._citations(root, cits)
|
| 99 |
+
dois = [e.text for e in root.iter() if e.tag.endswith("doi")]
|
| 100 |
+
check(dois == ["10.0000/fn.1999.1"], f"<doi> links {dois}")
|
| 101 |
+
|
| 102 |
+
print("[6] the review-quality auditor's evidence block")
|
| 103 |
+
cv.save_citation_report("ICSAC-SUB-TEST-FC", cits, rep)
|
| 104 |
+
import review_quality_control as rqc
|
| 105 |
+
ev = rqc.build_evidence_block("ICSAC-SUB-TEST-FC", config.REVIEWS_DIR)
|
| 106 |
+
check("2 UNCONFIRMED (no identifier cited; not evidence)" in ev and "Moss 2001 — UNCONFIRMED" in ev, "counts + line")
|
| 107 |
+
check("10.0000/rev.2002.77" not in ev and "title-year-match" not in ev, "no record, no match mode")
|
| 108 |
+
|
| 109 |
+
print("[7] header + prompt")
|
| 110 |
+
import review
|
| 111 |
+
line = review._citation_header_line("ICSAC-SUB-TEST-FC")
|
| 112 |
+
check(line is not None and line.startswith("1/5 resolved · 2 unconfirmed · 1 cited by URL"), f"header ({line})")
|
| 113 |
+
check("UNCONFIRMED (no identifier cited" in review.REVIEW_PROMPT_TEMPLATE
|
| 114 |
+
and "MATCHED (found by text only" not in review.REVIEW_PROMPT_TEMPLATE, "panel prompt legend")
|
| 115 |
+
import json
|
| 116 |
+
payload = json.load(open(Path(config.REVIEWS_DIR) / "ICSAC-SUB-TEST-FC_citations.json"))
|
| 117 |
+
check(payload.get("confirmed_count") == 1 and payload.get("unconfirmed_count") == 2, "JSON counts")
|
| 118 |
+
print("[8] compaction: a manuscript anonymized at submission goes on; anything else still fails closed")
|
| 119 |
+
import review_compaction as rc
|
| 120 |
+
paper = ("Ripples on sand: a field study\n\nAbstract. We measure ripples.\n\n1. Introduction\n"
|
| 121 |
+
"Ripples form under flow (Holm 1999).\n\nReferences\nHolm, P. (1999). Ripples on sand.\n")
|
| 122 |
+
rc._claude_call = lambda manuscript, **k: ('{"author_names": [], "affiliations": [], "emails": [], "orcids": []}', "", 0)
|
| 123 |
+
who = {"names": ["Quill Ashdown"], "other": ["0000-0002-1825-0097", "q.ashdown@example.org", "Example Institute of Sand"]}
|
| 124 |
+
txt, man = rc.compact_paper(paper, log=lambda m: None, known_identity=who)
|
| 125 |
+
check(txt and not man.get("_failure") and man.get("anonymized_at_submission") is True,
|
| 126 |
+
"no spans + none of the submission's identifiers in the text -> proceeds, flagged")
|
| 127 |
+
check("anonymized at submission" in rc.render_manifest(man), "the author's disclosure says so")
|
| 128 |
+
for label, text in (("surname", paper + "\nThanks to Ashdown.\n"), ("ORCID", paper + "0000-0002-1825-0097\n"),
|
| 129 |
+
("email", paper + "q.ashdown@example.org\n"), ("affiliation", paper + "Example Institute of Sand\n")):
|
| 130 |
+
txt, man = rc.compact_paper(text, log=lambda m: None, known_identity=who)
|
| 131 |
+
check(txt == "" and man.get("_failure") == "no identity spans identified", f"no spans + the {label} in the text -> fails closed")
|
| 132 |
+
txt, man = rc.compact_paper(paper, log=lambda m: None)
|
| 133 |
+
check(txt == "" and man.get("_failure"), "no spans and no known identity -> fails closed (unchanged)")
|
| 134 |
+
txt, man = rc.compact_paper(paper, log=lambda m: None, known_identity={"names": [], "other": ["x@example.org"]})
|
| 135 |
+
check(txt == "" and man.get("_failure"), "no spans and no known NAME -> fails closed")
|
| 136 |
+
import review
|
| 137 |
+
ki = review._known_identity({"creators": [{"name": "Quill Ashdown", "orcid": "0000-0002-1825-0097", "affiliation": "X"}, "Doe, Jane"],
|
| 138 |
+
"raw_metadata": {"submission": {"submitter_email": "q@example.org"}}})
|
| 139 |
+
check(ki == {"names": ["Quill Ashdown", "Doe, Jane"], "other": ["0000-0002-1825-0097", "X", "q@example.org"]}, f"known identity {ki}")
|
| 140 |
+
print("[9] no verification results is said as such, never as a clean result")
|
| 141 |
+
rd = {"title": "T", "creators": ["A"], "full_text": "x" * 300}
|
| 142 |
+
stub = "## Citation verification\n\nCitation verification was unavailable for this submission (X).\n"
|
| 143 |
+
for label, rep in (("no report", ""), ("unavailable stub", stub)):
|
| 144 |
+
pr = review.build_prompt(rd, verification_report=rep)
|
| 145 |
+
check("Nothing about its references is established" in pr
|
| 146 |
+
and pr.count("Do not write that any reference") == 1, f"{label}: the panel is told nothing was checked")
|
| 147 |
+
finally:
|
| 148 |
+
shutil.rmtree(TMP, ignore_errors=True)
|
| 149 |
+
|
| 150 |
+
print()
|
| 151 |
+
if failures:
|
| 152 |
+
print(f"{len(failures)} FAILED:")
|
| 153 |
+
for f in failures:
|
| 154 |
+
print(" - " + f)
|
| 155 |
+
sys.exit(1)
|
| 156 |
+
print("all checks passed")
|
|
@@ -0,0 +1,142 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Checks that a paper's keywords reach its public record (2026-10-02).
|
| 3 |
+
Offline: no network, no mail, no pings; every write lands in one temporary
|
| 4 |
+
directory that is removed at the end, pass or fail. Run from the repo root
|
| 5 |
+
with the venv:
|
| 6 |
+
.venv/bin/python intake/scripts/test_keywords_on_record.py
|
| 7 |
+
|
| 8 |
+
Covers: the cleaner; the registry keeping keywords; each of the four registry
|
| 9 |
+
writers passing them (Zenodo community accept, DOI-route accept, Zenodo-route
|
| 10 |
+
publish, Crossref-route register); the DOI route's optional form field winning
|
| 11 |
+
over the source record; the backfill tool's sources and its write gate.
|
| 12 |
+
"""
|
| 13 |
+
from __future__ import annotations
|
| 14 |
+
|
| 15 |
+
import importlib.util
|
| 16 |
+
import json
|
| 17 |
+
import os
|
| 18 |
+
import shutil
|
| 19 |
+
import sys
|
| 20 |
+
import tempfile
|
| 21 |
+
from pathlib import Path
|
| 22 |
+
|
| 23 |
+
ROOT = Path(__file__).resolve().parents[2]
|
| 24 |
+
sys.path.insert(0, str(ROOT))
|
| 25 |
+
|
| 26 |
+
failures: list[str] = []
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def check(cond: bool, name: str) -> None:
|
| 30 |
+
print((" ok " if cond else " FAIL ") + name)
|
| 31 |
+
if not cond:
|
| 32 |
+
failures.append(name)
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
tmp = Path(tempfile.mkdtemp(prefix="keywords-"))
|
| 36 |
+
try:
|
| 37 |
+
import publications as pub
|
| 38 |
+
pub.READY_DIR = str(tmp / "ready-to-publish") # held entries stay inside the temp dir
|
| 39 |
+
|
| 40 |
+
# ── 1. the cleaner ───────────────────────────────────────────────────────
|
| 41 |
+
print("1. cleaner")
|
| 42 |
+
check(pub.clean_keywords(None) == [], "None -> []")
|
| 43 |
+
check(pub.clean_keywords("alpha, beta; gamma") == ["alpha", "beta", "gamma"], "string split on , and ;")
|
| 44 |
+
check(pub.clean_keywords(["alpha", "Alpha", " beta. ", ""]) == ["alpha", "beta"],
|
| 45 |
+
"trimmed, trailing full stop dropped, case-insensitive duplicate dropped")
|
| 46 |
+
check(pub.clean_keywords(["one, two", "three"]) == ["one", "two", "three"], "a pasted list inside one item is split")
|
| 47 |
+
check(pub.clean_keywords(["x" * 81, "ok"]) == ["ok"], "an item over 80 characters is dropped")
|
| 48 |
+
check(len(pub.clean_keywords([f"k{i}" for i in range(20)])) == pub.KEYWORD_LIMIT, "at most KEYWORD_LIMIT kept")
|
| 49 |
+
check(pub.clean_keywords(["en–dash term", "agent–environment"]) == ["en–dash term", "agent–environment"],
|
| 50 |
+
"non-ASCII punctuation kept as typed")
|
| 51 |
+
|
| 52 |
+
# ── 2. the registry keeps them ───────────────────────────────────────────
|
| 53 |
+
print("2. registry")
|
| 54 |
+
site = tmp / "site"; (site / "src" / "data").mkdir(parents=True)
|
| 55 |
+
reg = site / "src" / "data" / "accepted.json"
|
| 56 |
+
reg.write_text("[]")
|
| 57 |
+
pub.WEBSITE_REPO = str(site); pub.REGISTRY_PATH = str(reg)
|
| 58 |
+
e = pub.upsert_entry(approved=True, proto={"title": "T", "authors": ["A"], "doi": "10.5555/icsac.2026.990",
|
| 59 |
+
"source": "submission-pdf", "keywords": ["alpha", "beta"]})
|
| 60 |
+
check(e.get("keywords") == ["alpha", "beta"], "upsert writes keywords")
|
| 61 |
+
e = pub.upsert_entry(approved=True, proto={"title": "T", "authors": ["A"], "doi": "10.5555/icsac.2026.990",
|
| 62 |
+
"source": "submission-pdf"})
|
| 63 |
+
check(e.get("keywords") == ["alpha", "beta"], "a later upsert without keywords keeps them")
|
| 64 |
+
reg.write_text("[]")
|
| 65 |
+
|
| 66 |
+
# ── 3. the four writers pass them ────────────────────────────────────────
|
| 67 |
+
print("3. writers")
|
| 68 |
+
captured: list[dict] = []
|
| 69 |
+
real_upsert = pub.upsert_entry
|
| 70 |
+
pub.upsert_entry = lambda proto, **kw: (captured.append(proto), {})[1]
|
| 71 |
+
try:
|
| 72 |
+
import action
|
| 73 |
+
proto = action._extract_registry_entry("12345", {"metadata": {
|
| 74 |
+
"title": "T", "creators": [{"name": "Author, Test"}], "doi": "10.5281/zenodo.12345",
|
| 75 |
+
"keywords": ["alpha", "beta."]}})
|
| 76 |
+
check(proto.get("keywords") == ["alpha", "beta"], "Zenodo community accept: proto carries the record's keywords")
|
| 77 |
+
proto = action._extract_registry_entry("12345", {"metadata": {"title": "T", "creators": []}})
|
| 78 |
+
check("keywords" not in proto, "Zenodo community accept: no keywords -> no field")
|
| 79 |
+
|
| 80 |
+
sub_dir = tmp / "ICSAC-SUB-99990"; sub_dir.mkdir()
|
| 81 |
+
(sub_dir / "state.json").write_text("{}")
|
| 82 |
+
sub = {"title": "T", "abstract": "A", "keywords": ["alpha", "beta"],
|
| 83 |
+
"creators": [{"name": "Author, Test"}], "form": {"name": "Test Author"}, "license": "cc-by-4.0"}
|
| 84 |
+
|
| 85 |
+
import crossref_deposit
|
| 86 |
+
captured.clear()
|
| 87 |
+
crossref_deposit._push_publications(sub_dir, sub, "10.5555/icsac.2026.990", log=lambda *a: None)
|
| 88 |
+
check(captured and captured[0].get("keywords") == ["alpha", "beta"], "Crossref-route register: proto carries keywords")
|
| 89 |
+
|
| 90 |
+
import publish_watcher
|
| 91 |
+
captured.clear()
|
| 92 |
+
try:
|
| 93 |
+
publish_watcher._register_published(sub_dir, dict(sub, keywords=[]),
|
| 94 |
+
{"doi": "10.5281/zenodo.12345", "record_id": "12345",
|
| 95 |
+
"metadata": {"title": "T", "keywords": ["from", "record"]}})
|
| 96 |
+
except Exception:
|
| 97 |
+
pass # held entry: the function may stop after the upsert; the proto is what is checked
|
| 98 |
+
check(captured and captured[0].get("keywords") == ["from", "record"],
|
| 99 |
+
"Zenodo-route publish: falls back to the deposit's keywords")
|
| 100 |
+
|
| 101 |
+
from intake import submission_worker as sw
|
| 102 |
+
captured.clear()
|
| 103 |
+
sw._register_doi_accept("ICSAC-SUB-99990", sub_dir, dict(sub, doi="10.5555/icsac.2026.990"))
|
| 104 |
+
check(captured and captured[0].get("keywords") == ["alpha", "beta"], "DOI-route accept: proto carries keywords")
|
| 105 |
+
finally:
|
| 106 |
+
pub.upsert_entry = real_upsert
|
| 107 |
+
|
| 108 |
+
# ── 4. DOI route: the form field wins over the source record ─────────────
|
| 109 |
+
print("4. DOI route form field")
|
| 110 |
+
src = (ROOT / "intake" / "submission_worker.py").read_text()
|
| 111 |
+
check('keywords = submission.get("keywords") or review_meta.get("keywords") or []' in src,
|
| 112 |
+
"worker: author keywords first, source record second")
|
| 113 |
+
srv = (ROOT / "intake" / "intake_server.py").read_text()
|
| 114 |
+
check(srv.count('form.get("keywords") or ""') == 2, "server: both handlers read the optional DOI-route field")
|
| 115 |
+
|
| 116 |
+
# ── 5. the backfill tool ─────────────────────────────────────────────────
|
| 117 |
+
print("5. backfill tool")
|
| 118 |
+
spec = importlib.util.spec_from_file_location("backfill", ROOT / "tools" / "backfill-keywords.py")
|
| 119 |
+
bf = importlib.util.module_from_spec(spec); spec.loader.exec_module(bf)
|
| 120 |
+
subs = tmp / "subs"; (subs / "ICSAC-SUB-99991").mkdir(parents=True)
|
| 121 |
+
(subs / "ICSAC-SUB-99991" / "submission.json").write_text(json.dumps({"keywords": ["one", "two."]}))
|
| 122 |
+
bf.SUBMISSIONS_ROOT = subs
|
| 123 |
+
check(bf.from_submission("ICSAC-SUB-99991") == ["one", "two"], "reads and cleans a submission's keywords")
|
| 124 |
+
check(bf.from_submission("ICSAC-SUB-99999") == [], "a missing submission gives none")
|
| 125 |
+
reg.write_text(json.dumps([{"slug": "t", "sub_id": "ICSAC-SUB-99991", "title": "T", "authors": ["A"],
|
| 126 |
+
"doi": "10.5555/icsac.2026.991", "accepted_date": "2026-10-02",
|
| 127 |
+
"source": "submission-pdf"}]))
|
| 128 |
+
os.environ.pop("ICSAC_PUBLISH_APPROVED", None)
|
| 129 |
+
sys.argv = ["backfill", "--write"]
|
| 130 |
+
check(bf.main() == 2 and "keywords" not in json.loads(reg.read_text())[0], "--write refuses without approval")
|
| 131 |
+
sys.argv = ["backfill"]
|
| 132 |
+
check(bf.main() == 0 and "keywords" not in json.loads(reg.read_text())[0], "dry run writes nothing")
|
| 133 |
+
os.environ["ICSAC_PUBLISH_APPROVED"] = "1"
|
| 134 |
+
sys.argv = ["backfill", "--write"]
|
| 135 |
+
check(bf.main() == 0 and json.loads(reg.read_text())[0].get("keywords") == ["one", "two"],
|
| 136 |
+
"--write with approval fills the entry")
|
| 137 |
+
os.environ.pop("ICSAC_PUBLISH_APPROVED", None)
|
| 138 |
+
finally:
|
| 139 |
+
shutil.rmtree(tmp, ignore_errors=True)
|
| 140 |
+
|
| 141 |
+
print(f"\n{'PASS' if not failures else 'FAILED'}: {len(failures)} failure(s)")
|
| 142 |
+
sys.exit(0 if not failures else 1)
|
|
@@ -248,16 +248,16 @@ def main() -> int:
|
|
| 248 |
real_send_followup(sub_id="ICSAC-SUB-TEST-1", tier=3, **common)
|
| 249 |
real_send_followup(sub_id="ICSAC-SUB-TEST-1", tier=1, **common) # a test id can never go out plain
|
| 250 |
na.send_published(to="a@example.com", sub_id="ICSAC-SUB-00001", title="T", author_name="A",
|
| 251 |
-
deposit_doi="10.
|
| 252 |
publications_url="https://icsacinstitute.org/publications/x", stay_involved=True)
|
| 253 |
na.send_published(to="a@example.com", sub_id="ICSAC-SUB-00001", title="T", author_name="A",
|
| 254 |
-
deposit_doi="10.
|
| 255 |
publications_url="https://icsacinstitute.org/publications/x", stay_involved=False)
|
| 256 |
na.send_published(to="a@example.com", sub_id="ICSAC-SUB-TEST-1", title="T", author_name="A",
|
| 257 |
-
deposit_doi="10.
|
| 258 |
publications_url="https://icsacinstitute.org/publications/x", tier=3)
|
| 259 |
na.send_published(to="a@example.com", sub_id="ICSAC-SUB-TEST-1", title="T", author_name="A",
|
| 260 |
-
deposit_doi="10.
|
| 261 |
publications_url="https://icsacinstitute.org/publications/x") # tier omitted
|
| 262 |
check(len(sent) == 8, f"eight renders reached the transport ({len(sent)})")
|
| 263 |
check(all(not k.get("send") for k in sent), "no render ever asks the transport to SEND")
|
|
|
|
| 248 |
real_send_followup(sub_id="ICSAC-SUB-TEST-1", tier=3, **common)
|
| 249 |
real_send_followup(sub_id="ICSAC-SUB-TEST-1", tier=1, **common) # a test id can never go out plain
|
| 250 |
na.send_published(to="a@example.com", sub_id="ICSAC-SUB-00001", title="T", author_name="A",
|
| 251 |
+
deposit_doi="10.5555/icsac.2026.001", deposit_url="https://doi.org/x",
|
| 252 |
publications_url="https://icsacinstitute.org/publications/x", stay_involved=True)
|
| 253 |
na.send_published(to="a@example.com", sub_id="ICSAC-SUB-00001", title="T", author_name="A",
|
| 254 |
+
deposit_doi="10.5555/icsac.2026.001", deposit_url="https://doi.org/x",
|
| 255 |
publications_url="https://icsacinstitute.org/publications/x", stay_involved=False)
|
| 256 |
na.send_published(to="a@example.com", sub_id="ICSAC-SUB-TEST-1", title="T", author_name="A",
|
| 257 |
+
deposit_doi="10.5555/TEST.x", deposit_url="https://doi.org/x",
|
| 258 |
publications_url="https://icsacinstitute.org/publications/x", tier=3)
|
| 259 |
na.send_published(to="a@example.com", sub_id="ICSAC-SUB-TEST-1", title="T", author_name="A",
|
| 260 |
+
deposit_doi="10.5555/TEST.x", deposit_url="https://doi.org/x",
|
| 261 |
publications_url="https://icsacinstitute.org/publications/x") # tier omitted
|
| 262 |
check(len(sent) == 8, f"eight renders reached the transport ({len(sent)})")
|
| 263 |
check(all(not k.get("send") for k in sent), "no render ever asks the transport to SEND")
|
|
@@ -177,7 +177,7 @@ try:
|
|
| 177 |
import crossref_deposit as cd
|
| 178 |
check(cd.preprint_doi(up_rec) == "10.20944/preprints202607.0102.v1", "the pipeline reads the upload route's preprint")
|
| 179 |
sub = dict(up_rec, sub_id="ICSAC-SUB-00099")
|
| 180 |
-
xml = cd.build_deposit_xml(sub, doi="10.
|
| 181 |
pdf_url=None, content_type="journal-article")
|
| 182 |
cd.validate_xml(xml)
|
| 183 |
check(b'relationship-type="hasPreprint"' in xml and b"10.20944/preprints202607.0102.v1" in xml,
|
|
|
|
| 177 |
import crossref_deposit as cd
|
| 178 |
check(cd.preprint_doi(up_rec) == "10.20944/preprints202607.0102.v1", "the pipeline reads the upload route's preprint")
|
| 179 |
sub = dict(up_rec, sub_id="ICSAC-SUB-00099")
|
| 180 |
+
xml = cd.build_deposit_xml(sub, doi="10.5555/icsac.2026.099", landing_url="https://icsacinstitute.org/publications/t",
|
| 181 |
pdf_url=None, content_type="journal-article")
|
| 182 |
cd.validate_xml(xml)
|
| 183 |
check(b'relationship-type="hasPreprint"' in xml and b"10.20944/preprints202607.0102.v1" in xml,
|
|
@@ -51,7 +51,7 @@ try:
|
|
| 51 |
real_run = subprocess.run
|
| 52 |
pub.subprocess.run = lambda cmd, **k: (git_calls.append(cmd), subprocess.CompletedProcess(cmd, 0, "", ""))[1]
|
| 53 |
|
| 54 |
-
proto = {"title": "A held paper", "authors": ["A. Author"], "doi": "10.
|
| 55 |
"source": "submission-pdf", "sub_id": "ICSAC-SUB-00990"}
|
| 56 |
|
| 57 |
print("1. the writers refuse without approval")
|
|
|
|
| 51 |
real_run = subprocess.run
|
| 52 |
pub.subprocess.run = lambda cmd, **k: (git_calls.append(cmd), subprocess.CompletedProcess(cmd, 0, "", ""))[1]
|
| 53 |
|
| 54 |
+
proto = {"title": "A held paper", "authors": ["A. Author"], "doi": "10.5555/icsac.2026.990",
|
| 55 |
"source": "submission-pdf", "sub_id": "ICSAC-SUB-00990"}
|
| 56 |
|
| 57 |
print("1. the writers refuse without approval")
|
|
@@ -36,7 +36,7 @@ try:
|
|
| 36 |
(sub / "submission.json").write_text(json.dumps({"sub_id": sub.name, "title": "T", "source": "upload"}))
|
| 37 |
(sub / "state.json").write_text(json.dumps({"decision": "accept", "state": "completed"}))
|
| 38 |
(sub / "crossref/deposit.xml").write_bytes(b"<x/>")
|
| 39 |
-
(sub / "crossref/staged.json").write_text(json.dumps({"doi": "10.
|
| 40 |
|
| 41 |
calls = {"deposit": 0, "stage": 0}
|
| 42 |
cd.stage = lambda *a, **k: calls.__setitem__("stage", calls["stage"] + 1)
|
|
|
|
| 36 |
(sub / "submission.json").write_text(json.dumps({"sub_id": sub.name, "title": "T", "source": "upload"}))
|
| 37 |
(sub / "state.json").write_text(json.dumps({"decision": "accept", "state": "completed"}))
|
| 38 |
(sub / "crossref/deposit.xml").write_bytes(b"<x/>")
|
| 39 |
+
(sub / "crossref/staged.json").write_text(json.dumps({"doi": "10.5555/icsac.2026.999", "registered": False}))
|
| 40 |
|
| 41 |
calls = {"deposit": 0, "stage": 0}
|
| 42 |
cd.stage = lambda *a, **k: calls.__setitem__("stage", calls["stage"] + 1)
|
|
@@ -0,0 +1,75 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""A decision letter for a revised paper names the earlier submission it
|
| 3 |
+
revises, in the header under the submission ID; a first submission's letter
|
| 4 |
+
carries no such line. Every decision template carries the slot, and both
|
| 5 |
+
callers of send_decision pass the recorded link. Offline, mail call stubbed,
|
| 6 |
+
no files written. Run from the repo root with the venv:
|
| 7 |
+
.venv/bin/python intake/scripts/test_revision_line.py
|
| 8 |
+
"""
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
import sys
|
| 12 |
+
from pathlib import Path
|
| 13 |
+
|
| 14 |
+
ROOT = Path(__file__).resolve().parents[2]
|
| 15 |
+
sys.path.insert(0, str(ROOT))
|
| 16 |
+
|
| 17 |
+
from intake import notify_author as na # noqa: E402
|
| 18 |
+
|
| 19 |
+
failures: list[str] = []
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def check(cond: bool, msg: str) -> None:
|
| 23 |
+
print(("ok " if cond else "FAIL ") + msg)
|
| 24 |
+
if not cond:
|
| 25 |
+
failures.append(msg)
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
DECISION_TEMPLATES = ("accept_doi", "accept_upload", "accept_upload_pending", "revise_codelink",
|
| 29 |
+
"revise_doi", "revise_upload", "scope_reject_doi", "scope_reject_upload")
|
| 30 |
+
for t in DECISION_TEMPLATES:
|
| 31 |
+
raw = (na.TEMPLATES_DIR / f"submission_{t}.md").read_text()
|
| 32 |
+
check("**Submission ID:** {{icsac_submission_id}}{{revision_line}}\n" in raw,
|
| 33 |
+
f"submission_{t}.md: the revision slot sits on the submission ID line")
|
| 34 |
+
for t in ("received", "intake_failure"):
|
| 35 |
+
raw = (na.TEMPLATES_DIR / f"submission_{t}.md").read_text()
|
| 36 |
+
check("{{revision_line}}" not in raw, f"submission_{t}.md: no slot (its sender fills no such key)")
|
| 37 |
+
|
| 38 |
+
sent: list[dict] = []
|
| 39 |
+
_orig = na.email_send.send_email
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def _stub(**kw):
|
| 43 |
+
sent.append(kw)
|
| 44 |
+
return True, "stub"
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
na.email_send.send_email = _stub
|
| 48 |
+
try:
|
| 49 |
+
common = dict(to="ada@example.com", sub_id="ICSAC-SUB-TEST-2000000002",
|
| 50 |
+
title="A Test Paper", author_name="Ada Example",
|
| 51 |
+
panel_report_md="", curator_findings="1. A finding.",
|
| 52 |
+
compaction_manifest={"author_names": ["Ada Example"]})
|
| 53 |
+
for verdict, source in (("revise", "upload"), ("revise", "doi"), ("scope_reject", "upload")):
|
| 54 |
+
sent.clear()
|
| 55 |
+
na.send_decision(verdict=verdict, source=source, revision_of="ICSAC-SUB-TEST-2000000001",
|
| 56 |
+
**common)
|
| 57 |
+
body = sent[-1]["body_md"] if sent else ""
|
| 58 |
+
check("**Submission ID:** ICSAC-SUB-TEST-2000000002\n**Revision of:** ICSAC-SUB-TEST-2000000001\n**Title:**"
|
| 59 |
+
in body, f"{verdict}/{source}: a revision's letter names the earlier submission under its ID")
|
| 60 |
+
sent.clear()
|
| 61 |
+
na.send_decision(verdict=verdict, source=source, **common)
|
| 62 |
+
body = sent[-1]["body_md"] if sent else ""
|
| 63 |
+
check(bool(body) and "Revision of" not in body
|
| 64 |
+
and "**Submission ID:** ICSAC-SUB-TEST-2000000002\n**Title:**" in body,
|
| 65 |
+
f"{verdict}/{source}: a first submission's letter carries no revision line")
|
| 66 |
+
finally:
|
| 67 |
+
na.email_send.send_email = _orig
|
| 68 |
+
|
| 69 |
+
for caller, expr in (("intake/apply_decision.py", 'revision_of=state_pre.get("revision_of") or ""'),
|
| 70 |
+
("intake/submission_worker.py", 'revision_of=state.get("revision_of") or ""')):
|
| 71 |
+
src = (ROOT / caller).read_text()
|
| 72 |
+
check(src.count("send_decision(") == 1 and expr in src.split("send_decision(", 1)[1].split("\n )")[0],
|
| 73 |
+
f"{caller}: the send_decision call passes the recorded link")
|
| 74 |
+
|
| 75 |
+
sys.exit(1 if failures else 0)
|
|
@@ -57,9 +57,9 @@ try:
|
|
| 57 |
|
| 58 |
print("1. evidence block")
|
| 59 |
ev = rqc.build_evidence_block(rid, str(rdir))
|
| 60 |
-
check("5 references; 1 CONFIRMED by identifier, 1
|
| 61 |
check("A. Nobody et al. 2024 — DOI MISMATCH" in ev and "Moreau 2024 — DOI DEAD" in ev, "the lines the panel had to act on")
|
| 62 |
-
check("Brook 2026 — UNVERIFIABLE" in ev and "Hale 1983 —
|
| 63 |
check("CODE PACKAGE DIGEST" in ev and "a.py: compute_xrm" in ev and "a.py:5: xrm = a * b + c" in ev, "digest definitions + construct lines")
|
| 64 |
check("Files (2)" not in ev and "README head" not in ev, "file list and README are left out (the auditor needs definitions, not the inventory)")
|
| 65 |
ev_b = rqc.build_evidence_block(rid, str(rdir), {"author_names": ["Ada Author"], "affiliations": [], "emails": [], "orcids": []})
|
|
|
|
| 57 |
|
| 58 |
print("1. evidence block")
|
| 59 |
ev = rqc.build_evidence_block(rid, str(rdir))
|
| 60 |
+
check("5 references; 1 CONFIRMED by identifier, 1 UNCONFIRMED (no identifier cited; not evidence), 2 with a DOI that disagrees (mismatch or dead), 1 UNVERIFIABLE" in ev, "counts line")
|
| 61 |
check("A. Nobody et al. 2024 — DOI MISMATCH" in ev and "Moreau 2024 — DOI DEAD" in ev, "the lines the panel had to act on")
|
| 62 |
+
check("Brook 2026 — UNVERIFIABLE" in ev and "Hale 1983 — UNCONFIRMED: no identifier cited" in ev, "unverifiable + matched lines")
|
| 63 |
check("CODE PACKAGE DIGEST" in ev and "a.py: compute_xrm" in ev and "a.py:5: xrm = a * b + c" in ev, "digest definitions + construct lines")
|
| 64 |
check("Files (2)" not in ev and "README head" not in ev, "file list and README are left out (the auditor needs definitions, not the inventory)")
|
| 65 |
ev_b = rqc.build_evidence_block(rid, str(rdir), {"author_names": ["Ada Author"], "affiliations": [], "emails": [], "orcids": []})
|
|
@@ -0,0 +1,151 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Workers AI and Mistral as panel providers, and the usage guard that keeps
|
| 3 |
+
Workers AI inside its free daily allocation and Mistral under its monthly cap. Offline: the HTTP call is stubbed, nothing is sent, and the
|
| 4 |
+
ledger lives in a temporary directory that is removed at the end, pass or fail.
|
| 5 |
+
Run from the repo root: .venv/bin/python intake/scripts/test_workers_ai.py
|
| 6 |
+
"""
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
import io
|
| 10 |
+
import json
|
| 11 |
+
import os
|
| 12 |
+
import shutil
|
| 13 |
+
import sys
|
| 14 |
+
import tempfile
|
| 15 |
+
import urllib.error
|
| 16 |
+
import urllib.request
|
| 17 |
+
from pathlib import Path
|
| 18 |
+
|
| 19 |
+
REPO = Path(__file__).resolve().parents[2]
|
| 20 |
+
sys.path.insert(0, str(REPO))
|
| 21 |
+
|
| 22 |
+
failures: list[str] = []
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def check(cond: bool, msg: str) -> None:
|
| 26 |
+
print((" ok " if cond else " FAIL ") + msg)
|
| 27 |
+
if not cond:
|
| 28 |
+
failures.append(msg)
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
TMP = Path(tempfile.mkdtemp(prefix="icsac-wai-"))
|
| 32 |
+
try:
|
| 33 |
+
os.environ["ICSAC_STATE_DIR"] = str(TMP)
|
| 34 |
+
os.environ["TEST_WAI_KEY"] = "test-key"
|
| 35 |
+
import config
|
| 36 |
+
import review
|
| 37 |
+
import redaction
|
| 38 |
+
|
| 39 |
+
RATES = {"@cf/x/model-a": (31818, 68182)}
|
| 40 |
+
SPEC = {"base_url": "https://example.invalid/ai/v1", "key_env": "TEST_WAI_KEY",
|
| 41 |
+
"daily_neuron_cap": 9000, "neuron_rates": RATES}
|
| 42 |
+
config.OAI_COMPAT_PROVIDERS["cftest"] = SPEC
|
| 43 |
+
ledger_path = Path(review._usage_ledger_path("cftest"))
|
| 44 |
+
|
| 45 |
+
sent = []
|
| 46 |
+
reply = {"usage": {"prompt_tokens": 7000, "completion_tokens": 900},
|
| 47 |
+
"choices": [{"message": {"content": "{}"}}]}
|
| 48 |
+
mode = {"http": None}
|
| 49 |
+
|
| 50 |
+
class _Resp(io.BytesIO):
|
| 51 |
+
def __enter__(self):
|
| 52 |
+
return self
|
| 53 |
+
|
| 54 |
+
def __exit__(self, *a):
|
| 55 |
+
return False
|
| 56 |
+
|
| 57 |
+
def fake_urlopen(req, timeout=None):
|
| 58 |
+
sent.append(req.full_url)
|
| 59 |
+
if mode["http"]:
|
| 60 |
+
raise urllib.error.HTTPError(req.full_url, mode["http"], "err", {}, io.BytesIO(b"{}"))
|
| 61 |
+
return _Resp(json.dumps(reply).encode())
|
| 62 |
+
urllib.request.urlopen = fake_urlopen
|
| 63 |
+
|
| 64 |
+
def ledger_today():
|
| 65 |
+
if not ledger_path.exists():
|
| 66 |
+
return 0
|
| 67 |
+
d = json.loads(ledger_path.read_text())
|
| 68 |
+
return d.get(review.datetime.now(review.timezone.utc).strftime("%Y-%m-%d"), 0)
|
| 69 |
+
|
| 70 |
+
prompt = "x" * 30000 # estimate: 30000/3 tokens in + 4000 out = ceil(318.18 + 272.73) = 591
|
| 71 |
+
|
| 72 |
+
print("[1] a call within the allocation is sent and settled to the reported usage")
|
| 73 |
+
r = review.run_oai_compat_review(prompt, "cftest", "@cf/x/model-a")
|
| 74 |
+
check(len(sent) == 1 and sent[0].endswith("/chat/completions"), "one request sent")
|
| 75 |
+
check(ledger_today() == 285, f"ledger = ceil(7000 in + 900 out at the model's rates) = 285 ({ledger_today()})")
|
| 76 |
+
|
| 77 |
+
print("[2] a call that could cross the cap is refused before anything is sent")
|
| 78 |
+
ledger_path.write_text(json.dumps({review.datetime.now(review.timezone.utc).strftime("%Y-%m-%d"): 8500}))
|
| 79 |
+
sent.clear()
|
| 80 |
+
r = review.run_oai_compat_review(prompt, "cftest", "@cf/x/model-a")
|
| 81 |
+
check(not sent and "free-allocation guard" in r.get("error", "") and "8500 used + 591 estimated" in r["error"],
|
| 82 |
+
f"refused, nothing sent ({r.get('error', '')[:90]})")
|
| 83 |
+
check(ledger_today() == 8500, "a refusal reserves nothing")
|
| 84 |
+
|
| 85 |
+
print("[3] a model with no rate on file is refused")
|
| 86 |
+
ledger_path.unlink()
|
| 87 |
+
r = review.run_oai_compat_review(prompt, "cftest", "@cf/x/unpriced")
|
| 88 |
+
check(not sent and "rate on file" in r.get("error", ""), "refused, nothing sent")
|
| 89 |
+
|
| 90 |
+
print("[4] an HTTP 4xx releases the reservation; a reply without usage keeps it")
|
| 91 |
+
mode["http"] = 401
|
| 92 |
+
r = review.run_oai_compat_review(prompt, "cftest", "@cf/x/model-a")
|
| 93 |
+
check(len(sent) == 1 and "HTTP 401" in r.get("error", "") and ledger_today() == 0, f"401: ledger back to 0 ({ledger_today()})")
|
| 94 |
+
mode["http"] = None
|
| 95 |
+
reply.pop("usage")
|
| 96 |
+
r = review.run_oai_compat_review(prompt, "cftest", "@cf/x/model-a")
|
| 97 |
+
check(ledger_today() == 591, f"no usage figures: the 591 reservation stands ({ledger_today()})")
|
| 98 |
+
|
| 99 |
+
print("[5] a provider without a cap is never metered; the ledger keeps 7 days")
|
| 100 |
+
check(review._usage_reserve("x", {"base_url": "u"}, "m", prompt, 4000) == (None, 0, ""), "no cap, no reservation")
|
| 101 |
+
old = {f"2026-09-{d:02d}": 1 for d in range(1, 21)}
|
| 102 |
+
ledger_path.write_text(json.dumps(old))
|
| 103 |
+
review._usage_ledger_update("cftest", lambda l: None, 7)
|
| 104 |
+
check(len(json.loads(ledger_path.read_text())) == 7, "pruned to the last 7 days")
|
| 105 |
+
|
| 106 |
+
print("[6] configuration")
|
| 107 |
+
import importlib.util
|
| 108 |
+
spec = importlib.util.spec_from_file_location("config_example", REPO / "config.example.py")
|
| 109 |
+
ex = importlib.util.module_from_spec(spec); spec.loader.exec_module(ex)
|
| 110 |
+
cf = ex.OAI_COMPAT_PROVIDERS.get("cloudflare") or {}
|
| 111 |
+
check(cf.get("key_env") == "CLOUDFLARE_AI_TOKEN" and cf.get("daily_neuron_cap") == 9000
|
| 112 |
+
and "/ai/v1" in cf.get("base_url", ""), "config.example: the cloudflare provider, cap 9000 by default")
|
| 113 |
+
roster_cf = [e.split("|", 2)[2] for slot in ex.OPENROUTER_MODELS for e in slot if e.startswith("oai|cloudflare|")]
|
| 114 |
+
check(all(m in cf.get("neuron_rates", {}) for m in roster_cf), f"every cloudflare roster entry has a rate ({roster_cf})")
|
| 115 |
+
mi = ex.OAI_COMPAT_PROVIDERS.get("mistral") or {}
|
| 116 |
+
check(mi.get("monthly_cap_usd") == 8.0 and mi.get("key_env") == "MISTRAL_API_KEY", "config.example: mistral capped at $8/month by default")
|
| 117 |
+
roster_mi = [e.split("|", 2)[2] for slot in ex.OPENROUTER_MODELS for e in slot if e.startswith("oai|mistral|")]
|
| 118 |
+
check(all(m in mi.get("usd_rates", {}) for m in roster_mi), f"every mistral roster entry has a USD rate ({roster_mi})")
|
| 119 |
+
check("@cf/" in redaction.FORBIDDEN_VENDOR_TOKENS, "the route prefix is a forbidden self-identification token")
|
| 120 |
+
|
| 121 |
+
print("[7] a monthly USD cap (Mistral): metered in its own ledger, refused before it can pass the cap")
|
| 122 |
+
reply["usage"] = {"prompt_tokens": 7000, "completion_tokens": 900}
|
| 123 |
+
config.OAI_COMPAT_PROVIDERS["mtest"] = {"base_url": "https://example.invalid/v1", "key_env": "TEST_WAI_KEY",
|
| 124 |
+
"monthly_cap_usd": 8.0, "usd_rates": {"m-medium": (2.0, 10.0)}}
|
| 125 |
+
month = review.datetime.now(review.timezone.utc).strftime("%Y-%m")
|
| 126 |
+
mpath = Path(review._usage_ledger_path("mtest"))
|
| 127 |
+
cf_before = ledger_path.read_text() if ledger_path.exists() else ""
|
| 128 |
+
sent.clear()
|
| 129 |
+
r = review.run_oai_compat_review(prompt, "mtest", "m-medium")
|
| 130 |
+
m = json.loads(mpath.read_text())
|
| 131 |
+
check(len(sent) == 1 and m == {month: 0.023}, f"settled to 7000 in + 900 out at $2/$10 per M = $0.023 ({m})")
|
| 132 |
+
check((ledger_path.read_text() if ledger_path.exists() else "") == cf_before and mpath.name == "mtest_usage.json",
|
| 133 |
+
"its own ledger file; the neuron ledger is untouched")
|
| 134 |
+
mpath.write_text(json.dumps({month: 7.95}))
|
| 135 |
+
sent.clear()
|
| 136 |
+
r = review.run_oai_compat_review(prompt, "mtest", "m-medium")
|
| 137 |
+
check(not sent and "monthly spending guard" in r.get("error", "") and "$7.9500 used + $0.0600 estimated > $8.00" in r["error"],
|
| 138 |
+
f"refused before sending ({r.get('error', '')[:100]})")
|
| 139 |
+
check(json.loads(mpath.read_text()) == {month: 7.95}, "a refusal reserves nothing")
|
| 140 |
+
check(review._model_identity("cloudflare:@cf/openai/gpt-oss-120b") == review._model_identity("hf|openai/gpt-oss-120b:deepinfra"),
|
| 141 |
+
"the same model served by two providers is one reviewer")
|
| 142 |
+
finally:
|
| 143 |
+
shutil.rmtree(TMP, ignore_errors=True)
|
| 144 |
+
|
| 145 |
+
print()
|
| 146 |
+
if failures:
|
| 147 |
+
print(f"{len(failures)} FAILED:")
|
| 148 |
+
for f in failures:
|
| 149 |
+
print(" - " + f)
|
| 150 |
+
sys.exit(1)
|
| 151 |
+
print("all checks passed")
|
|
@@ -75,7 +75,7 @@ try:
|
|
| 75 |
(sub / "submission.json").write_text(json.dumps({"sub_id": sub.name, "title": "T", "source": "upload"}))
|
| 76 |
(sub / "state.json").write_text(json.dumps({"decision": "accept", "state": "completed", "deposit_record_id": "555"}))
|
| 77 |
(sub / "crossref/deposit.xml").write_bytes(b"<x/>")
|
| 78 |
-
(sub / "crossref/staged.json").write_text(json.dumps({"doi": "10.
|
| 79 |
cd.stage = lambda *a, **k: None
|
| 80 |
cd.validate_xml = lambda b: None
|
| 81 |
cd.deposit = lambda *a, **k: "batch-1"
|
|
|
|
| 75 |
(sub / "submission.json").write_text(json.dumps({"sub_id": sub.name, "title": "T", "source": "upload"}))
|
| 76 |
(sub / "state.json").write_text(json.dumps({"decision": "accept", "state": "completed", "deposit_record_id": "555"}))
|
| 77 |
(sub / "crossref/deposit.xml").write_bytes(b"<x/>")
|
| 78 |
+
(sub / "crossref/staged.json").write_text(json.dumps({"doi": "10.5555/icsac.2026.998", "registered": False}))
|
| 79 |
cd.stage = lambda *a, **k: None
|
| 80 |
cd.validate_xml = lambda b: None
|
| 81 |
cd.deposit = lambda *a, **k: "batch-1"
|
|
@@ -45,9 +45,8 @@ from fastapi import HTTPException, Request, UploadFile
|
|
| 45 |
# sessions whose subscription items don't reference this product.
|
| 46 |
SPONSOR_PRODUCT_ID = "prod_UXiXkUln7XbrQz"
|
| 47 |
|
| 48 |
-
# Website repo on
|
| 49 |
-
#
|
| 50 |
-
# the dedicated push key.
|
| 51 |
WEBSITE_REPO = Path.home() / "Desktop" / "icsac" / "icsacinstitute.org"
|
| 52 |
LOGO_DIR = WEBSITE_REPO / "public" / "supporter-logos"
|
| 53 |
SUPPORTERS_YAML = WEBSITE_REPO / "src" / "data" / "supporters.yaml"
|
|
|
|
| 45 |
# sessions whose subscription items don't reference this product.
|
| 46 |
SPONSOR_PRODUCT_ID = "prod_UXiXkUln7XbrQz"
|
| 47 |
|
| 48 |
+
# Website repo checkout on the intake host. Its SSH remote uses a host
|
| 49 |
+
# alias that already carries a dedicated push key.
|
|
|
|
| 50 |
WEBSITE_REPO = Path.home() / "Desktop" / "icsac" / "icsacinstitute.org"
|
| 51 |
LOGO_DIR = WEBSITE_REPO / "public" / "supporter-logos"
|
| 52 |
SUPPORTERS_YAML = WEBSITE_REPO / "src" / "data" / "supporters.yaml"
|
|
@@ -220,7 +220,8 @@ def _resolve_pending_doi(sub_id: str, sub_dir: Path,
|
|
| 220 |
paper_path.unlink()
|
| 221 |
src.rename(paper_path)
|
| 222 |
|
| 223 |
-
|
|
|
|
| 224 |
if not isinstance(keywords, list):
|
| 225 |
keywords = [str(keywords)]
|
| 226 |
license_id = ""
|
|
@@ -707,6 +708,7 @@ def _email_decision(sub_id: str, sub_dir: Path, verdict: str,
|
|
| 707 |
deposit_doi=deposit_doi, deposit_url=deposit_url,
|
| 708 |
publications_url=publications_url,
|
| 709 |
tier=tier,
|
|
|
|
| 710 |
)
|
| 711 |
if ok and tier == 1:
|
| 712 |
# Decision emails go to Gmail Drafts (per 2026-05-07 policy).
|
|
@@ -791,6 +793,9 @@ def _register_doi_accept(sub_id: str, sub_dir: Path,
|
|
| 791 |
cdu = publications.code_data_url(submission)
|
| 792 |
if cdu:
|
| 793 |
proto["code_data_url"] = cdu
|
|
|
|
|
|
|
|
|
|
| 794 |
|
| 795 |
entry = publications.upsert_entry(proto)
|
| 796 |
if not entry:
|
|
|
|
| 220 |
paper_path.unlink()
|
| 221 |
src.rename(paper_path)
|
| 222 |
|
| 223 |
+
# Keywords the author typed on the DOI form win; the source record's fill in.
|
| 224 |
+
keywords = submission.get("keywords") or review_meta.get("keywords") or []
|
| 225 |
if not isinstance(keywords, list):
|
| 226 |
keywords = [str(keywords)]
|
| 227 |
license_id = ""
|
|
|
|
| 708 |
deposit_doi=deposit_doi, deposit_url=deposit_url,
|
| 709 |
publications_url=publications_url,
|
| 710 |
tier=tier,
|
| 711 |
+
revision_of=state.get("revision_of") or "",
|
| 712 |
)
|
| 713 |
if ok and tier == 1:
|
| 714 |
# Decision emails go to Gmail Drafts (per 2026-05-07 policy).
|
|
|
|
| 793 |
cdu = publications.code_data_url(submission)
|
| 794 |
if cdu:
|
| 795 |
proto["code_data_url"] = cdu
|
| 796 |
+
kw = publications.clean_keywords(submission.get("keywords"))
|
| 797 |
+
if kw:
|
| 798 |
+
proto["keywords"] = kw
|
| 799 |
|
| 800 |
entry = publications.upsert_entry(proto)
|
| 801 |
if not entry:
|
|
@@ -4,7 +4,7 @@ Subject: ICSAC review complete — recommend — {{icsac_submission_id}}
|
|
| 4 |
|
| 5 |
The ICSAC reviewer panel has completed its evaluation of your submission.
|
| 6 |
|
| 7 |
-
**Submission ID:** {{icsac_submission_id}}
|
| 8 |
**Title:** {{title}}
|
| 9 |
**Decision:** Recommend for publication
|
| 10 |
|
|
|
|
| 4 |
|
| 5 |
The ICSAC reviewer panel has completed its evaluation of your submission.
|
| 6 |
|
| 7 |
+
**Submission ID:** {{icsac_submission_id}}{{revision_line}}
|
| 8 |
**Title:** {{title}}
|
| 9 |
**Decision:** Recommend for publication
|
| 10 |
|
|
@@ -4,7 +4,7 @@ Subject: ICSAC review complete — recommend — {{icsac_submission_id}}
|
|
| 4 |
|
| 5 |
The ICSAC reviewer panel has completed its evaluation of your submission.
|
| 6 |
|
| 7 |
-
**Submission ID:** {{icsac_submission_id}}
|
| 8 |
**Title:** {{title}}
|
| 9 |
**Decision:** Recommend for publication
|
| 10 |
|
|
|
|
| 4 |
|
| 5 |
The ICSAC reviewer panel has completed its evaluation of your submission.
|
| 6 |
|
| 7 |
+
**Submission ID:** {{icsac_submission_id}}{{revision_line}}
|
| 8 |
**Title:** {{title}}
|
| 9 |
**Decision:** Recommend for publication
|
| 10 |
|
|
@@ -4,7 +4,7 @@ Subject: ICSAC review complete — accepted — {{icsac_submission_id}}
|
|
| 4 |
|
| 5 |
Your manuscript has been accepted for publication by the Institute for Complexity Science and Advanced Computing.
|
| 6 |
|
| 7 |
-
**Submission ID:** {{icsac_submission_id}}
|
| 8 |
**Title:** {{title}}
|
| 9 |
**Received:** {{received_date}}
|
| 10 |
**Decision:** {{decided_date}} — accepted
|
|
|
|
| 4 |
|
| 5 |
Your manuscript has been accepted for publication by the Institute for Complexity Science and Advanced Computing.
|
| 6 |
|
| 7 |
+
**Submission ID:** {{icsac_submission_id}}{{revision_line}}
|
| 8 |
**Title:** {{title}}
|
| 9 |
**Received:** {{received_date}}
|
| 10 |
**Decision:** {{decided_date}} — accepted
|
|
@@ -4,7 +4,7 @@ Subject: ICSAC submission — revise and resubmit — {{icsac_submission_id}}
|
|
| 4 |
|
| 5 |
Thank you for submitting your manuscript to the Institute for Complexity Science and Advanced Computing.
|
| 6 |
|
| 7 |
-
**Submission ID:** {{icsac_submission_id}}
|
| 8 |
**Title:** {{title}}
|
| 9 |
**Decision:** Revise and resubmit
|
| 10 |
|
|
|
|
| 4 |
|
| 5 |
Thank you for submitting your manuscript to the Institute for Complexity Science and Advanced Computing.
|
| 6 |
|
| 7 |
+
**Submission ID:** {{icsac_submission_id}}{{revision_line}}
|
| 8 |
**Title:** {{title}}
|
| 9 |
**Decision:** Revise and resubmit
|
| 10 |
|
|
@@ -4,7 +4,7 @@ Subject: ICSAC review complete — revise and resubmit — {{icsac_submission_id
|
|
| 4 |
|
| 5 |
The review of your submission is complete.
|
| 6 |
|
| 7 |
-
**Submission ID:** {{icsac_submission_id}}
|
| 8 |
**Title:** {{title}}
|
| 9 |
**Decision:** Revise and resubmit
|
| 10 |
|
|
@@ -22,7 +22,7 @@ Revise the manuscript and submit the new version through the same form at [icsac
|
|
| 22 |
|
| 23 |
If you think a finding above is mistaken, leave the paper as it was on that point and say so in an email to [help@icsacinstitute.org](mailto:help@icsacinstitute.org) with this submission ID in the subject line. The curation team reads every such message, and a finding we got wrong is withdrawn.
|
| 24 |
|
| 25 |
-
|
| 26 |
|
| 27 |
There is no limit on the number of revise-and-resubmit rounds. ICSAC does not charge submission, review, or article-processing fees at any stage.
|
| 28 |
|
|
|
|
| 4 |
|
| 5 |
The review of your submission is complete.
|
| 6 |
|
| 7 |
+
**Submission ID:** {{icsac_submission_id}}{{revision_line}}
|
| 8 |
**Title:** {{title}}
|
| 9 |
**Decision:** Revise and resubmit
|
| 10 |
|
|
|
|
| 22 |
|
| 23 |
If you think a finding above is mistaken, leave the paper as it was on that point and say so in an email to [help@icsacinstitute.org](mailto:help@icsacinstitute.org) with this submission ID in the subject line. The curation team reads every such message, and a finding we got wrong is withdrawn.
|
| 24 |
|
| 25 |
+
A resubmission gets a fresh panel run: the panel does not see this letter or the earlier review. The curation team does, and checks the new version against the findings in this letter.
|
| 26 |
|
| 27 |
There is no limit on the number of revise-and-resubmit rounds. ICSAC does not charge submission, review, or article-processing fees at any stage.
|
| 28 |
|
|
@@ -4,7 +4,7 @@ Subject: ICSAC review complete — revise and resubmit — {{icsac_submission_id
|
|
| 4 |
|
| 5 |
The review of your submission is complete.
|
| 6 |
|
| 7 |
-
**Submission ID:** {{icsac_submission_id}}
|
| 8 |
**Title:** {{title}}
|
| 9 |
**Decision:** Revise and resubmit
|
| 10 |
|
|
@@ -22,7 +22,7 @@ Revise the manuscript and submit the new version through the same form at [icsac
|
|
| 22 |
|
| 23 |
If you think a finding above is mistaken, leave the paper as it was on that point and say so in an email to [help@icsacinstitute.org](mailto:help@icsacinstitute.org) with this submission ID in the subject line. The curation team reads every such message, and a finding we got wrong is withdrawn.
|
| 24 |
|
| 25 |
-
|
| 26 |
|
| 27 |
There is no limit on the number of revise-and-resubmit rounds. ICSAC does not charge submission, review, or article-processing fees at any stage.
|
| 28 |
|
|
|
|
| 4 |
|
| 5 |
The review of your submission is complete.
|
| 6 |
|
| 7 |
+
**Submission ID:** {{icsac_submission_id}}{{revision_line}}
|
| 8 |
**Title:** {{title}}
|
| 9 |
**Decision:** Revise and resubmit
|
| 10 |
|
|
|
|
| 22 |
|
| 23 |
If you think a finding above is mistaken, leave the paper as it was on that point and say so in an email to [help@icsacinstitute.org](mailto:help@icsacinstitute.org) with this submission ID in the subject line. The curation team reads every such message, and a finding we got wrong is withdrawn.
|
| 24 |
|
| 25 |
+
A resubmission gets a fresh panel run: the panel does not see this letter or the earlier review. The curation team does, and checks the new version against the findings in this letter.
|
| 26 |
|
| 27 |
There is no limit on the number of revise-and-resubmit rounds. ICSAC does not charge submission, review, or article-processing fees at any stage.
|
| 28 |
|
|
@@ -4,7 +4,7 @@ Subject: ICSAC review complete — out of scope — {{icsac_submission_id}}
|
|
| 4 |
|
| 5 |
The ICSAC reviewer panel has completed its evaluation of your submission.
|
| 6 |
|
| 7 |
-
**Submission ID:** {{icsac_submission_id}}
|
| 8 |
**Title:** {{title}}
|
| 9 |
**Decision:** Scope reject
|
| 10 |
|
|
|
|
| 4 |
|
| 5 |
The ICSAC reviewer panel has completed its evaluation of your submission.
|
| 6 |
|
| 7 |
+
**Submission ID:** {{icsac_submission_id}}{{revision_line}}
|
| 8 |
**Title:** {{title}}
|
| 9 |
**Decision:** Scope reject
|
| 10 |
|
|
@@ -4,7 +4,7 @@ Subject: ICSAC review complete — out of scope — {{icsac_submission_id}}
|
|
| 4 |
|
| 5 |
The ICSAC reviewer panel has completed its evaluation of your submission.
|
| 6 |
|
| 7 |
-
**Submission ID:** {{icsac_submission_id}}
|
| 8 |
**Title:** {{title}}
|
| 9 |
**Decision:** Scope reject
|
| 10 |
|
|
|
|
| 4 |
|
| 5 |
The ICSAC reviewer panel has completed its evaluation of your submission.
|
| 6 |
|
| 7 |
+
**Submission ID:** {{icsac_submission_id}}{{revision_line}}
|
| 8 |
**Title:** {{title}}
|
| 9 |
**Decision:** Scope reject
|
| 10 |
|
|
@@ -51,6 +51,10 @@ def send_telegram(message: str, parse_mode: str | None = "Markdown",
|
|
| 51 |
payload = {
|
| 52 |
"chat_id": chat_id,
|
| 53 |
"text": message,
|
|
|
|
|
|
|
|
|
|
|
|
|
| 54 |
}
|
| 55 |
if parse_mode:
|
| 56 |
payload["parse_mode"] = parse_mode
|
|
|
|
| 51 |
payload = {
|
| 52 |
"chat_id": chat_id,
|
| 53 |
"text": message,
|
| 54 |
+
# No link previews: Telegram fetches every link in a message to build one,
|
| 55 |
+
# and a doi.org link fetched before Crossref registers the DOI counts as a
|
| 56 |
+
# failed resolution in the prefix's monthly report.
|
| 57 |
+
"link_preview_options": {"is_disabled": True},
|
| 58 |
}
|
| 59 |
if parse_mode:
|
| 60 |
payload["parse_mode"] = parse_mode
|
|
@@ -86,6 +86,32 @@ _OPTIONAL_KEYS = (
|
|
| 86 |
)
|
| 87 |
|
| 88 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 89 |
def code_data_url(submission: dict) -> Optional[str]:
|
| 90 |
"""The author's public code and data link from the submission form, if
|
| 91 |
any. The paper page prints it as hosted by the author; ICSAC never hosts it."""
|
|
|
|
| 86 |
)
|
| 87 |
|
| 88 |
|
| 89 |
+
KEYWORD_LIMIT = 12
|
| 90 |
+
KEYWORD_MAX_CHARS = 80
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
def clean_keywords(raw: Any) -> list[str]:
|
| 94 |
+
"""Keywords as the paper page shows them: a list or a comma/semicolon
|
| 95 |
+
string in; each item split on commas and semicolons, trimmed, trailing
|
| 96 |
+
full stops dropped, case-insensitive duplicates and over-long items
|
| 97 |
+
removed, at most KEYWORD_LIMIT kept in the author's order."""
|
| 98 |
+
if raw is None:
|
| 99 |
+
return []
|
| 100 |
+
items = raw if isinstance(raw, (list, tuple)) else [raw]
|
| 101 |
+
out: list[str] = []
|
| 102 |
+
seen: set[str] = set()
|
| 103 |
+
for item in items:
|
| 104 |
+
for part in re.split(r"[,;]+", str(item)):
|
| 105 |
+
k = re.sub(r"\s+", " ", part).strip().rstrip(".").strip()
|
| 106 |
+
if not k or len(k) > KEYWORD_MAX_CHARS or k.casefold() in seen:
|
| 107 |
+
continue
|
| 108 |
+
seen.add(k.casefold())
|
| 109 |
+
out.append(k)
|
| 110 |
+
if len(out) >= KEYWORD_LIMIT:
|
| 111 |
+
return out
|
| 112 |
+
return out
|
| 113 |
+
|
| 114 |
+
|
| 115 |
def code_data_url(submission: dict) -> Optional[str]:
|
| 116 |
"""The author's public code and data link from the submission form, if
|
| 117 |
any. The paper page prints it as hosted by the author; ICSAC never hosts it."""
|
|
@@ -185,6 +185,9 @@ def _register_published(sub_dir: Path, submission: dict, deposit: dict) -> dict:
|
|
| 185 |
cdu = publications.code_data_url(submission)
|
| 186 |
if cdu:
|
| 187 |
proto["code_data_url"] = cdu
|
|
|
|
|
|
|
|
|
|
| 188 |
|
| 189 |
entry = publications.upsert_entry(proto)
|
| 190 |
if not entry:
|
|
|
|
| 185 |
cdu = publications.code_data_url(submission)
|
| 186 |
if cdu:
|
| 187 |
proto["code_data_url"] = cdu
|
| 188 |
+
kw = publications.clean_keywords(submission.get("keywords") or metadata.get("keywords"))
|
| 189 |
+
if kw:
|
| 190 |
+
proto["keywords"] = kw
|
| 191 |
|
| 192 |
entry = publications.upsert_entry(proto)
|
| 193 |
if not entry:
|
|
@@ -36,6 +36,7 @@ FORBIDDEN_VENDOR_TOKENS: tuple[str, ...] = (
|
|
| 36 |
# direct-provider labels from `oai|` panel entries (2026-09-28)
|
| 37 |
"sambanova",
|
| 38 |
"deepinfra",
|
|
|
|
| 39 |
"mistral-large",
|
| 40 |
"mistral-medium",
|
| 41 |
"anthropic",
|
|
|
|
| 36 |
# direct-provider labels from `oai|` panel entries (2026-09-28)
|
| 37 |
"sambanova",
|
| 38 |
"deepinfra",
|
| 39 |
+
"@cf/",
|
| 40 |
"mistral-large",
|
| 41 |
"mistral-medium",
|
| 42 |
"anthropic",
|
|
@@ -161,13 +161,14 @@ REVIEW_PROMPT_TEMPLATE = textwrap.dedent("""\
|
|
| 161 |
(c) IDENTIFIER DISAGREEMENT. The citation verification (its legend above these
|
| 162 |
instructions, its per-reference CITATION VERIFICATION RESULTS inside the
|
| 163 |
submission block) checked every reference and marks each one CONFIRMED (identifier resolves
|
| 164 |
-
and matches),
|
| 165 |
DOI MISMATCH / DOI DEAD (the cited DOI points to a different work, or
|
| 166 |
nowhere), or UNVERIFIABLE. A MISMATCH or DEAD line is a citation-integrity
|
| 167 |
finding to score and to name in the justification: it shows a wrong
|
| 168 |
identifier, wrong authors, or a reference assembled from several works,
|
| 169 |
and it bears on the paper's statement about how its references were
|
| 170 |
-
checked.
|
|
|
|
| 171 |
|
| 172 |
Score the dimension based on (a)+(b)+(c) combined. If you cannot verify (a) one
|
| 173 |
way or the other, weight (b) more heavily and explicitly say so in the
|
|
@@ -234,6 +235,16 @@ def _creator_display_names(creators) -> list[str]:
|
|
| 234 |
return out or ["Unknown"]
|
| 235 |
|
| 236 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 237 |
def split_verification_report(report: str) -> tuple[str, str]:
|
| 238 |
"""(legend, per-reference data). The legend is the fixed text before the
|
| 239 |
first '### ' section and carries nothing from the submission; everything
|
|
@@ -278,7 +289,7 @@ def build_prompt(review_data: dict, verification_report: str = "") -> str:
|
|
| 278 |
full_text=full_text,
|
| 279 |
related_identifiers=related_str,
|
| 280 |
code_digest=(review_data.get("code_digest") or "").strip() or "(none)",
|
| 281 |
-
verification_data=verification_data or
|
| 282 |
)
|
| 283 |
head = legend
|
| 284 |
if rubric_context:
|
|
@@ -637,6 +648,107 @@ def run_hf_router_review(prompt: str, hf_model: str, capture_path: str = None) -
|
|
| 637 |
return result
|
| 638 |
|
| 639 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 640 |
def run_oai_compat_review(prompt: str, provider: str, model: str, capture_path: str = None) -> dict:
|
| 641 |
"""Run review against an OpenAI-compatible provider called DIRECTLY with our
|
| 642 |
own key (2026-09-28): `oai|<provider>|<model>` panel entries. Providers and
|
|
@@ -660,6 +772,12 @@ def run_oai_compat_review(prompt: str, provider: str, model: str, capture_path:
|
|
| 660 |
"temperature": 0.3,
|
| 661 |
"max_tokens": 4000,
|
| 662 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 663 |
req = urllib.request.Request(spec["base_url"].rstrip("/") + "/chat/completions",
|
| 664 |
data=_json.dumps(payload).encode())
|
| 665 |
req.add_header("Authorization", f"Bearer {api_key}")
|
|
@@ -682,12 +800,16 @@ def run_oai_compat_review(prompt: str, provider: str, model: str, capture_path:
|
|
| 682 |
except urllib.error.HTTPError as e:
|
| 683 |
ex.shutdown(wait=False)
|
| 684 |
body = e.read()[:300].decode(errors="replace")
|
|
|
|
|
|
|
| 685 |
return {"error": f"HTTP {e.code}: {body}", "model": label}
|
| 686 |
except Exception as e:
|
| 687 |
ex.shutdown(wait=False)
|
| 688 |
return {"error": str(e), "model": label}
|
| 689 |
ex.shutdown(wait=False)
|
| 690 |
|
|
|
|
|
|
|
| 691 |
if not isinstance(data, dict):
|
| 692 |
return {"error": f"unexpected {provider} response shape", "model": label}
|
| 693 |
if data.get("error"):
|
|
@@ -1605,8 +1727,14 @@ def _citation_header_line(record_id) -> str | None:
|
|
| 1605 |
parts = []
|
| 1606 |
cits = payload.get("citations") or []
|
| 1607 |
if cits:
|
| 1608 |
-
|
| 1609 |
import citation_verify
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1610 |
idc = citation_verify.identity_counts(cits)
|
| 1611 |
if idc["mismatch"] or idc["dead"]:
|
| 1612 |
parts.append(f"DOI identity: {idc['mismatch']} mismatch, {idc['dead']} dead")
|
|
@@ -1720,7 +1848,8 @@ def _append_citation_verify_audit(record_id: str, citations: list[dict], error)
|
|
| 1720 |
try:
|
| 1721 |
import datetime, json as _json
|
| 1722 |
verified = sum(1 for c in citations if c.get("verified"))
|
| 1723 |
-
|
|
|
|
| 1724 |
entry = {
|
| 1725 |
"timestamp": datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
| 1726 |
"event": "citation_verify_completed",
|
|
@@ -1728,6 +1857,7 @@ def _append_citation_verify_audit(record_id: str, citations: list[dict], error)
|
|
| 1728 |
"citation_count": len(citations),
|
| 1729 |
"verified_count": verified,
|
| 1730 |
"unverifiable_count": unverifiable,
|
|
|
|
| 1731 |
"extraction_error": (
|
| 1732 |
None if not error else f"{type(error).__name__}: {error}"
|
| 1733 |
),
|
|
@@ -1771,6 +1901,25 @@ def _fire_compaction_pain(review_data: dict, reason: str) -> None:
|
|
| 1771 |
pass
|
| 1772 |
|
| 1773 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1774 |
def review_paper(review_data: dict) -> tuple[str, dict]:
|
| 1775 |
"""Run full multi-model review with self-heal + multi-pass aggregation.
|
| 1776 |
|
|
@@ -1803,6 +1952,7 @@ def review_paper(review_data: dict) -> tuple[str, dict]:
|
|
| 1803 |
redacted_text, compaction_manifest = review_compaction.compact_paper(
|
| 1804 |
original_text,
|
| 1805 |
log=lambda m: print(m, file=__import__("sys").stderr),
|
|
|
|
| 1806 |
)
|
| 1807 |
if compaction_manifest.get("_failure"):
|
| 1808 |
# FAIL CLOSED. Blind-review compaction failed, so the manuscript
|
|
|
|
| 161 |
(c) IDENTIFIER DISAGREEMENT. The citation verification (its legend above these
|
| 162 |
instructions, its per-reference CITATION VERIFICATION RESULTS inside the
|
| 163 |
submission block) checked every reference and marks each one CONFIRMED (identifier resolves
|
| 164 |
+
and matches), UNCONFIRMED (no identifier cited; identity not checked, no record shown),
|
| 165 |
DOI MISMATCH / DOI DEAD (the cited DOI points to a different work, or
|
| 166 |
nowhere), or UNVERIFIABLE. A MISMATCH or DEAD line is a citation-integrity
|
| 167 |
finding to score and to name in the justification: it shows a wrong
|
| 168 |
identifier, wrong authors, or a reference assembled from several works,
|
| 169 |
and it bears on the paper's statement about how its references were
|
| 170 |
+
checked. An UNCONFIRMED line is not evidence of anything: it is neither a
|
| 171 |
+
finding against the paper nor a credit to it.
|
| 172 |
|
| 173 |
Score the dimension based on (a)+(b)+(c) combined. If you cannot verify (a) one
|
| 174 |
way or the other, weight (b) more heavily and explicitly say so in the
|
|
|
|
| 235 |
return out or ["Unknown"]
|
| 236 |
|
| 237 |
|
| 238 |
+
# What the panel reads when there are no per-reference results. A bare "(none)"
|
| 239 |
+
# was read by some models as "no problems found" and reported as a clean
|
| 240 |
+
# verification; the empty case has to say that nothing was checked.
|
| 241 |
+
NO_VERIFICATION_DATA = (
|
| 242 |
+
"(No per-reference citation results: verification was not run, or failed, for "
|
| 243 |
+
"this submission. Nothing about its references is established -- neither that "
|
| 244 |
+
"they exist nor that they are the works cited. Do not write that any reference "
|
| 245 |
+
"was confirmed, matched or found free of problems; call them unverified.)")
|
| 246 |
+
|
| 247 |
+
|
| 248 |
def split_verification_report(report: str) -> tuple[str, str]:
|
| 249 |
"""(legend, per-reference data). The legend is the fixed text before the
|
| 250 |
first '### ' section and carries nothing from the submission; everything
|
|
|
|
| 289 |
full_text=full_text,
|
| 290 |
related_identifiers=related_str,
|
| 291 |
code_digest=(review_data.get("code_digest") or "").strip() or "(none)",
|
| 292 |
+
verification_data=verification_data or NO_VERIFICATION_DATA,
|
| 293 |
)
|
| 294 |
head = legend
|
| 295 |
if rubric_context:
|
|
|
|
| 648 |
return result
|
| 649 |
|
| 650 |
|
| 651 |
+
def _usage_terms(spec: dict):
|
| 652 |
+
"""The usage cap a provider carries, or None. Workers AI: a daily neuron cap
|
| 653 |
+
(a free daily allocation, billed past it on a paid account). Mistral: a
|
| 654 |
+
monthly USD cap (a card-backed account that must never pass its allowance).
|
| 655 |
+
Rates are units per million input / output tokens."""
|
| 656 |
+
if spec.get("daily_neuron_cap"):
|
| 657 |
+
return {"period": "day", "cap": spec["daily_neuron_cap"], "rates": spec.get("neuron_rates") or {},
|
| 658 |
+
"unit": "neurons", "keep": 7}
|
| 659 |
+
if spec.get("monthly_cap_usd"):
|
| 660 |
+
return {"period": "month", "cap": spec["monthly_cap_usd"], "rates": spec.get("usd_rates") or {},
|
| 661 |
+
"unit": "USD", "keep": 13}
|
| 662 |
+
return None
|
| 663 |
+
|
| 664 |
+
|
| 665 |
+
def _usage_ledger_path(provider: str) -> str:
|
| 666 |
+
d = os.environ.get("ICSAC_STATE_DIR") or os.path.join(os.path.dirname(os.path.abspath(__file__)), "state")
|
| 667 |
+
return os.path.join(d, "cf_neurons.json" if provider == "cloudflare" else f"{provider}_usage.json")
|
| 668 |
+
|
| 669 |
+
|
| 670 |
+
def _usage_ledger_update(provider: str, fn, keep: int = 7):
|
| 671 |
+
"""Read-modify-write one provider's usage ledger under an exclusive lock
|
| 672 |
+
(panel slots call in parallel). `fn(ledger) -> result`; the ledger keeps the
|
| 673 |
+
last `keep` periods."""
|
| 674 |
+
import fcntl
|
| 675 |
+
path = _usage_ledger_path(provider)
|
| 676 |
+
os.makedirs(os.path.dirname(path), exist_ok=True)
|
| 677 |
+
with open(path + ".lock", "a") as lk:
|
| 678 |
+
fcntl.flock(lk, fcntl.LOCK_EX)
|
| 679 |
+
try:
|
| 680 |
+
with open(path) as f:
|
| 681 |
+
ledger = json.load(f)
|
| 682 |
+
except (OSError, ValueError):
|
| 683 |
+
ledger = {}
|
| 684 |
+
result = fn(ledger)
|
| 685 |
+
for key in sorted(ledger)[:-keep]:
|
| 686 |
+
ledger.pop(key, None)
|
| 687 |
+
tmp = path + ".tmp"
|
| 688 |
+
with open(tmp, "w") as f:
|
| 689 |
+
json.dump(ledger, f)
|
| 690 |
+
os.replace(tmp, path)
|
| 691 |
+
return result
|
| 692 |
+
|
| 693 |
+
|
| 694 |
+
def _usage_amount(terms: dict, tokens_in: float, tokens_out: float, rate) -> float:
|
| 695 |
+
import math
|
| 696 |
+
x = tokens_in * rate[0] / 1e6 + tokens_out * rate[1] / 1e6
|
| 697 |
+
return math.ceil(x) if terms["unit"] == "neurons" else math.ceil(x * 1e4) / 1e4
|
| 698 |
+
|
| 699 |
+
|
| 700 |
+
def _usage_reserve(provider: str, spec: dict, model: str, prompt: str, max_tokens: int):
|
| 701 |
+
"""A provider with a usage cap is metered BEFORE the call. Reserves a
|
| 702 |
+
deliberately high estimate (a token per 3 characters of prompt, every output
|
| 703 |
+
token used) and refuses the call when the period's total (UTC day or UTC
|
| 704 |
+
month) could cross the cap, or when the model has no rate on file.
|
| 705 |
+
Returns (period_key, reserved, error)."""
|
| 706 |
+
terms = _usage_terms(spec)
|
| 707 |
+
if not terms:
|
| 708 |
+
return None, 0, ""
|
| 709 |
+
rate = terms["rates"].get(model)
|
| 710 |
+
if not rate:
|
| 711 |
+
return None, 0, f"no {terms['unit']} rate on file for {model}; refused so the cap cannot be overrun"
|
| 712 |
+
est = _usage_amount(terms, len(prompt) / 3, max_tokens, rate)
|
| 713 |
+
now = datetime.now(timezone.utc)
|
| 714 |
+
key = now.strftime("%Y-%m-%d") if terms["period"] == "day" else now.strftime("%Y-%m")
|
| 715 |
+
cap = terms["cap"]
|
| 716 |
+
|
| 717 |
+
def _take(ledger):
|
| 718 |
+
used = ledger.get(key, 0)
|
| 719 |
+
if used + est > cap:
|
| 720 |
+
if terms["unit"] == "USD":
|
| 721 |
+
return (f"monthly spending guard: ${used:.4f} used + ${est:.4f} estimated > ${cap:.2f} "
|
| 722 |
+
f"(resets on the 1st, UTC)")
|
| 723 |
+
return (f"daily free-allocation guard: {used} used + {est} estimated > {cap} neurons "
|
| 724 |
+
f"(resets 00:00 UTC)")
|
| 725 |
+
ledger[key] = round(used + est, 4)
|
| 726 |
+
return ""
|
| 727 |
+
err = _usage_ledger_update(provider, _take, terms["keep"])
|
| 728 |
+
return (None, 0, err) if err else (key, est, "")
|
| 729 |
+
|
| 730 |
+
|
| 731 |
+
def _usage_settle(provider: str, spec: dict, model: str, key, reserved, usage=None) -> None:
|
| 732 |
+
"""Replace a reservation with what the reported token counts imply (or with
|
| 733 |
+
0 when the call was refused before inference). Without usage figures the
|
| 734 |
+
reservation stands."""
|
| 735 |
+
terms = _usage_terms(spec)
|
| 736 |
+
if not terms or not key or not reserved:
|
| 737 |
+
return
|
| 738 |
+
if usage == 0:
|
| 739 |
+
actual = 0
|
| 740 |
+
elif isinstance(usage, dict) and usage.get("prompt_tokens") is not None:
|
| 741 |
+
rate = terms["rates"].get(model) or (0, 0)
|
| 742 |
+
actual = _usage_amount(terms, int(usage.get("prompt_tokens") or 0),
|
| 743 |
+
int(usage.get("completion_tokens") or 0), rate)
|
| 744 |
+
else:
|
| 745 |
+
return
|
| 746 |
+
|
| 747 |
+
def _fix(ledger):
|
| 748 |
+
ledger[key] = round(max(0, ledger.get(key, 0) - reserved + actual), 4)
|
| 749 |
+
_usage_ledger_update(provider, _fix, terms["keep"])
|
| 750 |
+
|
| 751 |
+
|
| 752 |
def run_oai_compat_review(prompt: str, provider: str, model: str, capture_path: str = None) -> dict:
|
| 753 |
"""Run review against an OpenAI-compatible provider called DIRECTLY with our
|
| 754 |
own key (2026-09-28): `oai|<provider>|<model>` panel entries. Providers and
|
|
|
|
| 772 |
"temperature": 0.3,
|
| 773 |
"max_tokens": 4000,
|
| 774 |
}
|
| 775 |
+
# Per-model request overrides (config: the provider's model_params), e.g. a
|
| 776 |
+
# larger max_tokens where hidden reasoning counts inside it, or JSON mode.
|
| 777 |
+
payload.update((spec.get("model_params") or {}).get(model) or {})
|
| 778 |
+
day, reserved, guard_err = _usage_reserve(provider, spec, model, prompt, payload["max_tokens"])
|
| 779 |
+
if guard_err:
|
| 780 |
+
return {"error": f"{provider}: {guard_err}", "model": label}
|
| 781 |
req = urllib.request.Request(spec["base_url"].rstrip("/") + "/chat/completions",
|
| 782 |
data=_json.dumps(payload).encode())
|
| 783 |
req.add_header("Authorization", f"Bearer {api_key}")
|
|
|
|
| 800 |
except urllib.error.HTTPError as e:
|
| 801 |
ex.shutdown(wait=False)
|
| 802 |
body = e.read()[:300].decode(errors="replace")
|
| 803 |
+
if 400 <= e.code < 500 and e.code != 408:
|
| 804 |
+
_usage_settle(provider, spec, model, day, reserved, 0) # refused before inference
|
| 805 |
return {"error": f"HTTP {e.code}: {body}", "model": label}
|
| 806 |
except Exception as e:
|
| 807 |
ex.shutdown(wait=False)
|
| 808 |
return {"error": str(e), "model": label}
|
| 809 |
ex.shutdown(wait=False)
|
| 810 |
|
| 811 |
+
if isinstance(data, dict):
|
| 812 |
+
_usage_settle(provider, spec, model, day, reserved, data.get("usage"))
|
| 813 |
if not isinstance(data, dict):
|
| 814 |
return {"error": f"unexpected {provider} response shape", "model": label}
|
| 815 |
if data.get("error"):
|
|
|
|
| 1727 |
parts = []
|
| 1728 |
cits = payload.get("citations") or []
|
| 1729 |
if cits:
|
| 1730 |
+
web = sum(1 for c in cits if c.get("confidence") == "web-source")
|
| 1731 |
import citation_verify
|
| 1732 |
+
parts.append(f"{sum(1 for c in cits if citation_verify.is_identity_confirmed(c))}/{len(cits) - web} resolved")
|
| 1733 |
+
unconfirmed = sum(1 for c in cits if citation_verify.is_unconfirmed_match(c))
|
| 1734 |
+
if unconfirmed:
|
| 1735 |
+
parts.append(f"{unconfirmed} unconfirmed")
|
| 1736 |
+
if web:
|
| 1737 |
+
parts.append(f"{web} cited by URL")
|
| 1738 |
idc = citation_verify.identity_counts(cits)
|
| 1739 |
if idc["mismatch"] or idc["dead"]:
|
| 1740 |
parts.append(f"DOI identity: {idc['mismatch']} mismatch, {idc['dead']} dead")
|
|
|
|
| 1848 |
try:
|
| 1849 |
import datetime, json as _json
|
| 1850 |
verified = sum(1 for c in citations if c.get("verified"))
|
| 1851 |
+
web = sum(1 for c in citations if c.get("confidence") == "web-source")
|
| 1852 |
+
unverifiable = sum(1 for c in citations if not c.get("verified")) - web
|
| 1853 |
entry = {
|
| 1854 |
"timestamp": datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
| 1855 |
"event": "citation_verify_completed",
|
|
|
|
| 1857 |
"citation_count": len(citations),
|
| 1858 |
"verified_count": verified,
|
| 1859 |
"unverifiable_count": unverifiable,
|
| 1860 |
+
"web_source_count": web,
|
| 1861 |
"extraction_error": (
|
| 1862 |
None if not error else f"{type(error).__name__}: {error}"
|
| 1863 |
),
|
|
|
|
| 1901 |
pass
|
| 1902 |
|
| 1903 |
|
| 1904 |
+
def _known_identity(review_data: dict) -> dict:
|
| 1905 |
+
"""The submission's own identifiers, for compaction's anonymized-manuscript
|
| 1906 |
+
check: creator names; creator ORCID iDs and affiliations and the submitter's
|
| 1907 |
+
email and ORCID ("other", matched whole). No names -> compaction fails closed
|
| 1908 |
+
on an empty extraction, as before."""
|
| 1909 |
+
names, other = [], []
|
| 1910 |
+
for c in review_data.get("creators") or []:
|
| 1911 |
+
if isinstance(c, str):
|
| 1912 |
+
names.append(c)
|
| 1913 |
+
elif isinstance(c, dict):
|
| 1914 |
+
if c.get("name"):
|
| 1915 |
+
names.append(c["name"])
|
| 1916 |
+
other += [c.get(k) for k in ("orcid", "affiliation") if c.get(k)]
|
| 1917 |
+
sub = ((review_data.get("raw_metadata") or {}).get("submission") or {})
|
| 1918 |
+
other += [sub.get(k) for k in ("submitter_email", "submitter_orcid") if sub.get(k)]
|
| 1919 |
+
names = [n for n in names if "withheld" not in str(n)]
|
| 1920 |
+
return {"names": names, "other": [o for o in other if str(o).strip()]}
|
| 1921 |
+
|
| 1922 |
+
|
| 1923 |
def review_paper(review_data: dict) -> tuple[str, dict]:
|
| 1924 |
"""Run full multi-model review with self-heal + multi-pass aggregation.
|
| 1925 |
|
|
|
|
| 1952 |
redacted_text, compaction_manifest = review_compaction.compact_paper(
|
| 1953 |
original_text,
|
| 1954 |
log=lambda m: print(m, file=__import__("sys").stderr),
|
| 1955 |
+
known_identity=_known_identity(review_data),
|
| 1956 |
)
|
| 1957 |
if compaction_manifest.get("_failure"):
|
| 1958 |
# FAIL CLOSED. Blind-review compaction failed, so the manuscript
|