Spaces:
Running
Running
Publish cumulative exact-claim verification
Browse filesClaims 1 and 5 VERIFIED; Claim 2 FALSIFIED; Claims 3 and 4 BLOCKED after four-route audits. Preserves judged revision files additively.
This view is limited to 50 files because it contains too many changes. See raw diff
- README.md +4 -0
- code/audit_candidate_space.py +257 -0
- code/check_baseline_artifacts.py +74 -0
- code/check_claim3_release.py +89 -0
- code/check_claim4_evidence.py +138 -0
- code/run_logdiff.py +858 -0
- environment/environment.md +12 -0
- environment/pyproject.toml +16 -0
- environment/uv.lock +102 -0
- evidence/claim-1/EVAL.md +10 -0
- evidence/claim-1/claim_contract.json +23 -0
- evidence/claim-1/exhaustive_formulas.csv +26 -0
- evidence/claim-1/independent_checker_output.txt +10 -0
- evidence/claim-1/limitations_and_deviations.md +8 -0
- evidence/claim-1/method.md +14 -0
- evidence/claim-1/negative_controls.csv +4 -0
- evidence/claim-1/primitive_rules.csv +101 -0
- evidence/claim-1/source_audit.md +21 -0
- evidence/claim-2/EVAL.md +8 -0
- evidence/claim-2/claim_contract.json +14 -0
- evidence/claim-2/independent_checker_output.txt +10 -0
- evidence/claim-2/limitations_and_deviations.md +6 -0
- evidence/claim-2/method.md +7 -0
- evidence/claim-2/negative_control.json +4 -0
- evidence/claim-2/source_audit.md +11 -0
- evidence/claim-2/table_2_cells.csv +17 -0
- evidence/claim-3/EVAL.md +18 -0
- evidence/claim-3/author_release_snapshot.json +125 -0
- evidence/claim-3/claim_contract.json +59 -0
- evidence/claim-3/falsification_check.json +28 -0
- evidence/claim-3/independent_checker_output.txt +7 -0
- evidence/claim-3/limitations_and_deviations.md +15 -0
- evidence/claim-3/method.md +25 -0
- evidence/claim-3/negative_control.json +11 -0
- evidence/claim-3/public-checkpoint-search/github-branches.json +1 -0
- evidence/claim-3/public-checkpoint-search/github-forks.json +1 -0
- evidence/claim-3/public-checkpoint-search/github-releases.json +1 -0
- evidence/claim-3/public-checkpoint-search/github-tags.json +1 -0
- evidence/claim-3/public-checkpoint-search/hf-datasets-LogDiff.json +1 -0
- evidence/claim-3/public-checkpoint-search/hf-datasets-TanjaBien.json +1 -0
- evidence/claim-3/public-checkpoint-search/hf-datasets-cs_celebA_composition_100epochs_male_haircolors.json +1 -0
- evidence/claim-3/public-checkpoint-search/hf-datasets-epoch_epoch_3D589-step_step_3D750000.ckpt.json +1 -0
- evidence/claim-3/public-checkpoint-search/hf-models-LogDiff.json +1 -0
- evidence/claim-3/public-checkpoint-search/hf-models-TanjaBien.json +1 -0
- evidence/claim-3/public-checkpoint-search/hf-models-cs_celebA_composition_100epochs_male_haircolors.json +1 -0
- evidence/claim-3/public-checkpoint-search/hf-models-epoch_epoch_3D589-step_step_3D750000.ckpt.json +1 -0
- evidence/claim-3/public-checkpoint-search/hf-public-celeba-model.json +1 -0
- evidence/claim-3/public-checkpoint-search/hf-public-celeba-tree.json +1 -0
- evidence/claim-3/public-checkpoint-search/hf-spaces-LogDiff.json +1 -0
- evidence/claim-3/public-checkpoint-search/hf-spaces-TanjaBien.json +1 -0
README.md
CHANGED
|
@@ -16,3 +16,7 @@ tags:
|
|
| 16 |
# Reproduction: Logical Guidance for the Exact Composition of Diffusion Models
|
| 17 |
|
| 18 |
An open experiment logbook, published with [Trackio](https://github.com/gradio-app/trackio).
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 16 |
# Reproduction: Logical Guidance for the Exact Composition of Diffusion Models
|
| 17 |
|
| 18 |
An open experiment logbook, published with [Trackio](https://github.com/gradio-app/trackio).
|
| 19 |
+
|
| 20 |
+
Evaluator entrypoint: [open the canonical page index](pages/index.md). The
|
| 21 |
+
first linked page is the current cumulative verifier. Older Claim 3 and Claim 4
|
| 22 |
+
material is explicitly marked **Historical rejected baseline**.
|
code/audit_candidate_space.py
ADDED
|
@@ -0,0 +1,257 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Evaluator-blind structural audit for the additive Hugging Face candidate."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
import hashlib
|
| 7 |
+
import json
|
| 8 |
+
import re
|
| 9 |
+
import sys
|
| 10 |
+
from pathlib import Path
|
| 11 |
+
from urllib.parse import unquote
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
TEXT_SUFFIXES = {
|
| 15 |
+
".csv",
|
| 16 |
+
".css",
|
| 17 |
+
".html",
|
| 18 |
+
".js",
|
| 19 |
+
".json",
|
| 20 |
+
".lock",
|
| 21 |
+
".md",
|
| 22 |
+
".py",
|
| 23 |
+
".svg",
|
| 24 |
+
".toml",
|
| 25 |
+
".txt",
|
| 26 |
+
}
|
| 27 |
+
MUTABLE_ENTRYPOINTS = {"README.md", "logbook.json", "pages/index.md"}
|
| 28 |
+
LINK_RE = re.compile(r"!?\[[^\]]*\]\(([^)]+)\)")
|
| 29 |
+
SECRET_PATTERNS = {
|
| 30 |
+
"hugging_face_token": re.compile(r"\bhf_[A-Za-z0-9]{20,}\b"),
|
| 31 |
+
"github_token": re.compile(r"\bgithub_pat_[A-Za-z0-9_]{20,}\b"),
|
| 32 |
+
"bearer_token": re.compile(r"\bBearer\s+[A-Za-z0-9._~-]{16,}\b", re.IGNORECASE),
|
| 33 |
+
"aws_access_key": re.compile(r"\bAKIA[0-9A-Z]{16}\b"),
|
| 34 |
+
}
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def sha256(path: Path) -> str:
|
| 38 |
+
return hashlib.sha256(path.read_bytes()).hexdigest()
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def flatten_pages(node: dict) -> dict[str, str]:
|
| 42 |
+
pages: dict[str, str] = {}
|
| 43 |
+
for child in node.get("children", []):
|
| 44 |
+
pages[child["slug"]] = child["file"]
|
| 45 |
+
pages.update(flatten_pages(child))
|
| 46 |
+
return pages
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def parse_judged_manifest(path: Path) -> dict[str, str]:
|
| 50 |
+
result = {}
|
| 51 |
+
for line in path.read_text().splitlines():
|
| 52 |
+
digest, relative = line.split(" ", 1)
|
| 53 |
+
result[relative] = digest
|
| 54 |
+
return result
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def resolve_link(space: Path, source: Path, raw_target: str, routes: dict[str, str]) -> Path | None:
|
| 58 |
+
target = raw_target.strip().split(" ", 1)[0].strip("<>")
|
| 59 |
+
if target.startswith(("http://", "https://", "mailto:")) or target == "":
|
| 60 |
+
return None
|
| 61 |
+
if target.startswith("#/"):
|
| 62 |
+
slug = target[2:].split("#", 1)[0]
|
| 63 |
+
if slug not in routes:
|
| 64 |
+
raise ValueError(f"unknown route {target} from {source.relative_to(space)}")
|
| 65 |
+
return space / routes[slug]
|
| 66 |
+
if target.startswith("#"):
|
| 67 |
+
return None
|
| 68 |
+
target = unquote(target.split("#", 1)[0].split("?", 1)[0])
|
| 69 |
+
return (source.parent / target).resolve()
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def main() -> None:
|
| 73 |
+
if len(sys.argv) != 4:
|
| 74 |
+
raise SystemExit(
|
| 75 |
+
"usage: audit_candidate_space.py CANDIDATE_DIR JUDGED_MANIFEST OUTPUT_DIR"
|
| 76 |
+
)
|
| 77 |
+
space = Path(sys.argv[1]).resolve()
|
| 78 |
+
judged_manifest_path = Path(sys.argv[2]).resolve()
|
| 79 |
+
output = Path(sys.argv[3]).resolve() / "release"
|
| 80 |
+
output.mkdir(parents=True, exist_ok=True)
|
| 81 |
+
failures: list[str] = []
|
| 82 |
+
|
| 83 |
+
try:
|
| 84 |
+
logbook = json.loads((space / "logbook.json").read_text())
|
| 85 |
+
except (OSError, json.JSONDecodeError) as exc:
|
| 86 |
+
raise SystemExit(f"invalid candidate logbook: {exc}")
|
| 87 |
+
routes = flatten_pages(logbook["root"])
|
| 88 |
+
routes[logbook["root"]["slug"]] = logbook["root"]["file"]
|
| 89 |
+
children = logbook["root"].get("children", [])
|
| 90 |
+
if not children or children[0].get("slug") != "current-cumulative-verification":
|
| 91 |
+
failures.append("current_cumulative_not_first")
|
| 92 |
+
|
| 93 |
+
judged = parse_judged_manifest(judged_manifest_path)
|
| 94 |
+
candidate_paths = {
|
| 95 |
+
path.relative_to(space).as_posix(): path
|
| 96 |
+
for path in space.rglob("*")
|
| 97 |
+
if path.is_file()
|
| 98 |
+
}
|
| 99 |
+
missing_old_paths = sorted(set(judged) - set(candidate_paths))
|
| 100 |
+
if missing_old_paths:
|
| 101 |
+
failures.append("judged_file_set_not_subset")
|
| 102 |
+
changed_historical_pages = []
|
| 103 |
+
for relative, expected in judged.items():
|
| 104 |
+
if relative.startswith("pages/") and relative != "pages/index.md":
|
| 105 |
+
path = candidate_paths.get(relative)
|
| 106 |
+
if path is not None and sha256(path) != expected:
|
| 107 |
+
changed_historical_pages.append(relative)
|
| 108 |
+
if changed_historical_pages:
|
| 109 |
+
failures.append("historical_page_changed")
|
| 110 |
+
|
| 111 |
+
json_failures = []
|
| 112 |
+
for relative, path in candidate_paths.items():
|
| 113 |
+
if path.suffix == ".json":
|
| 114 |
+
try:
|
| 115 |
+
json.loads(path.read_text())
|
| 116 |
+
except (UnicodeDecodeError, json.JSONDecodeError):
|
| 117 |
+
json_failures.append(relative)
|
| 118 |
+
if json_failures:
|
| 119 |
+
failures.append("invalid_json")
|
| 120 |
+
|
| 121 |
+
opened: list[str] = []
|
| 122 |
+
broken_links: list[dict[str, str]] = []
|
| 123 |
+
queue = [space / "README.md", space / "pages/index.md"]
|
| 124 |
+
seen: set[Path] = set()
|
| 125 |
+
while queue:
|
| 126 |
+
source = queue.pop(0).resolve()
|
| 127 |
+
if source in seen:
|
| 128 |
+
continue
|
| 129 |
+
seen.add(source)
|
| 130 |
+
if not source.is_file() or space not in source.parents:
|
| 131 |
+
broken_links.append({"source": "entrypoint", "target": str(source)})
|
| 132 |
+
continue
|
| 133 |
+
opened.append(source.relative_to(space).as_posix())
|
| 134 |
+
if source.suffix not in {".md", ".html"}:
|
| 135 |
+
continue
|
| 136 |
+
text = source.read_text(errors="strict")
|
| 137 |
+
for raw_target in LINK_RE.findall(text):
|
| 138 |
+
try:
|
| 139 |
+
resolved = resolve_link(space, source, raw_target, routes)
|
| 140 |
+
except ValueError:
|
| 141 |
+
broken_links.append(
|
| 142 |
+
{"source": source.relative_to(space).as_posix(), "target": raw_target}
|
| 143 |
+
)
|
| 144 |
+
continue
|
| 145 |
+
if resolved is None:
|
| 146 |
+
continue
|
| 147 |
+
if not resolved.is_file() or (resolved != space and space not in resolved.parents):
|
| 148 |
+
broken_links.append(
|
| 149 |
+
{"source": source.relative_to(space).as_posix(), "target": raw_target}
|
| 150 |
+
)
|
| 151 |
+
continue
|
| 152 |
+
if resolved.suffix in {".md", ".html"}:
|
| 153 |
+
queue.append(resolved)
|
| 154 |
+
elif resolved.relative_to(space).as_posix() not in opened:
|
| 155 |
+
opened.append(resolved.relative_to(space).as_posix())
|
| 156 |
+
if broken_links:
|
| 157 |
+
failures.append("broken_reachable_links")
|
| 158 |
+
|
| 159 |
+
cumulative = (space / "pages/current-cumulative-verification/page.md").read_text()
|
| 160 |
+
visibility_rows = {}
|
| 161 |
+
for line in cumulative.splitlines():
|
| 162 |
+
match = re.match(r"\|\s*([1-5])\s*\|(.+)\|", line)
|
| 163 |
+
if match and "Canonical page" not in line:
|
| 164 |
+
cells = [cell.strip() for cell in line.strip().strip("|").split("|")]
|
| 165 |
+
if len(cells) == 9:
|
| 166 |
+
visibility_rows[match.group(1)] = cells
|
| 167 |
+
if set(visibility_rows) != set("12345"):
|
| 168 |
+
failures.append("visibility_matrix_claim_rows")
|
| 169 |
+
elif any(
|
| 170 |
+
any(value in {"", "No", "Missing", "INCONCLUSIVE"} for value in row[1:])
|
| 171 |
+
for row in visibility_rows.values()
|
| 172 |
+
):
|
| 173 |
+
failures.append("visibility_matrix_missing_cell")
|
| 174 |
+
|
| 175 |
+
required_opened = {
|
| 176 |
+
"README.md",
|
| 177 |
+
"pages/index.md",
|
| 178 |
+
"pages/current-cumulative-verification/page.md",
|
| 179 |
+
"pages/current-claim-3-celeba-verification/page.md",
|
| 180 |
+
"pages/current-claim-4-molecular-verification/page.md",
|
| 181 |
+
"code/run_logdiff.py",
|
| 182 |
+
"code/check_baseline_artifacts.py",
|
| 183 |
+
"code/check_claim3_release.py",
|
| 184 |
+
"code/check_claim4_evidence.py",
|
| 185 |
+
"code/audit_candidate_space.py",
|
| 186 |
+
"environment/pyproject.toml",
|
| 187 |
+
"environment/uv.lock",
|
| 188 |
+
"evidence/run-07daf77f-summary.json",
|
| 189 |
+
}
|
| 190 |
+
missing_from_traversal = sorted(required_opened - set(opened))
|
| 191 |
+
if missing_from_traversal:
|
| 192 |
+
failures.append("required_evidence_not_reachable")
|
| 193 |
+
|
| 194 |
+
changed_or_new = []
|
| 195 |
+
for relative, path in sorted(candidate_paths.items()):
|
| 196 |
+
if relative not in judged or sha256(path) != judged[relative]:
|
| 197 |
+
changed_or_new.append(relative)
|
| 198 |
+
non_text_uploads = [
|
| 199 |
+
relative for relative in changed_or_new if Path(relative).suffix not in TEXT_SUFFIXES
|
| 200 |
+
]
|
| 201 |
+
if non_text_uploads:
|
| 202 |
+
failures.append("non_text_candidate_change")
|
| 203 |
+
|
| 204 |
+
secret_hits = []
|
| 205 |
+
for relative in changed_or_new:
|
| 206 |
+
path = candidate_paths[relative]
|
| 207 |
+
if path.suffix not in TEXT_SUFFIXES:
|
| 208 |
+
continue
|
| 209 |
+
text = path.read_text(errors="strict")
|
| 210 |
+
for name, pattern in SECRET_PATTERNS.items():
|
| 211 |
+
if pattern.search(text):
|
| 212 |
+
secret_hits.append({"path": relative, "pattern": name})
|
| 213 |
+
if secret_hits:
|
| 214 |
+
failures.append("secret_pattern")
|
| 215 |
+
|
| 216 |
+
candidate_manifest = [
|
| 217 |
+
f"{sha256(path)} {relative}" for relative, path in sorted(candidate_paths.items())
|
| 218 |
+
]
|
| 219 |
+
upload_allowlist = [
|
| 220 |
+
relative
|
| 221 |
+
for relative in changed_or_new
|
| 222 |
+
if Path(relative).suffix in TEXT_SUFFIXES
|
| 223 |
+
and relative not in {"release/candidate_manifest.sha256", "release/upload_allowlist.txt"}
|
| 224 |
+
]
|
| 225 |
+
(output / "candidate_manifest.sha256").write_text("\n".join(candidate_manifest) + "\n")
|
| 226 |
+
(output / "upload_allowlist.txt").write_text("\n".join(upload_allowlist) + "\n")
|
| 227 |
+
|
| 228 |
+
result = {
|
| 229 |
+
"checker": "evaluator-blind candidate traversal and release audit",
|
| 230 |
+
"status": "PASS" if not failures else "FAIL",
|
| 231 |
+
"failures": failures,
|
| 232 |
+
"canonical_entrypoint": "README.md -> pages/index.md -> current cumulative verification",
|
| 233 |
+
"files_opened": opened,
|
| 234 |
+
"conclusions_not_verified": [],
|
| 235 |
+
"broken_links": broken_links,
|
| 236 |
+
"missing_from_traversal": missing_from_traversal,
|
| 237 |
+
"judged_manifest_entries": len(judged),
|
| 238 |
+
"candidate_file_entries": len(candidate_paths),
|
| 239 |
+
"old_file_set_subset": not missing_old_paths,
|
| 240 |
+
"missing_old_paths": missing_old_paths,
|
| 241 |
+
"historical_pages_unchanged": not changed_historical_pages,
|
| 242 |
+
"changed_historical_pages": changed_historical_pages,
|
| 243 |
+
"json_files_checked": sum(path.suffix == ".json" for path in candidate_paths.values()),
|
| 244 |
+
"invalid_json_files": json_failures,
|
| 245 |
+
"visibility_claim_rows": len(visibility_rows),
|
| 246 |
+
"non_text_uploads": non_text_uploads,
|
| 247 |
+
"secret_hits": secret_hits,
|
| 248 |
+
"upload_allowlist_entries": len(upload_allowlist),
|
| 249 |
+
}
|
| 250 |
+
(output / "candidate_audit.json").write_text(json.dumps(result, indent=2) + "\n")
|
| 251 |
+
print(json.dumps(result, indent=2))
|
| 252 |
+
if failures:
|
| 253 |
+
raise SystemExit(1)
|
| 254 |
+
|
| 255 |
+
|
| 256 |
+
if __name__ == "__main__":
|
| 257 |
+
main()
|
code/check_baseline_artifacts.py
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Independent, standard-library-only checker for baseline artifact files."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
import csv
|
| 7 |
+
import json
|
| 8 |
+
import sys
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
def read_csv(path: Path) -> list[dict[str, str]]:
|
| 13 |
+
with path.open(newline="") as handle:
|
| 14 |
+
return list(csv.DictReader(handle))
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def main() -> None:
|
| 18 |
+
if len(sys.argv) != 2:
|
| 19 |
+
raise SystemExit("usage: check_baseline_artifacts.py ARTIFACT_DIR")
|
| 20 |
+
root = Path(sys.argv[1])
|
| 21 |
+
failures: list[str] = []
|
| 22 |
+
|
| 23 |
+
c1 = read_csv(root / "claim-1" / "exhaustive_formulas.csv")
|
| 24 |
+
c1_rules = read_csv(root / "claim-1" / "primitive_rules.csv")
|
| 25 |
+
c1_controls = read_csv(root / "claim-1" / "negative_controls.csv")
|
| 26 |
+
if len(c1) != 25 or sum(int(row["formulas"]) for row in c1) != 6350:
|
| 27 |
+
failures.append("claim_1_counts")
|
| 28 |
+
if max(float(row["max_probability_error"]) for row in c1) >= 1e-14:
|
| 29 |
+
failures.append("claim_1_probability")
|
| 30 |
+
if max(float(row["max_score_error"]) for row in c1) >= 1e-14:
|
| 31 |
+
failures.append("claim_1_score")
|
| 32 |
+
if len(c1_rules) != 100:
|
| 33 |
+
failures.append("claim_1_rules")
|
| 34 |
+
if len(c1_controls) != 3 or not all(row["rejected"] == "True" for row in c1_controls):
|
| 35 |
+
failures.append("claim_1_controls")
|
| 36 |
+
|
| 37 |
+
c2 = read_csv(root / "claim-2" / "table_2_cells.csv")
|
| 38 |
+
c2_control = json.loads((root / "claim-2" / "negative_control.json").read_text())
|
| 39 |
+
logdiff = [row for row in c2 if row["method"] == "LOGDIFF"]
|
| 40 |
+
constant = [row for row in c2 if row["method"] == "constant"]
|
| 41 |
+
if len(logdiff) != 8 or sum(row["inside_claimed_range"] == "True" for row in logdiff) != 2:
|
| 42 |
+
failures.append("claim_2_logdiff_cells")
|
| 43 |
+
if len(constant) != 8 or sum(row["inside_claimed_range"] == "True" for row in constant) != 5:
|
| 44 |
+
failures.append("claim_2_constant_cells")
|
| 45 |
+
if not c2_control["verifier_rejected_false_falsification"]:
|
| 46 |
+
failures.append("claim_2_control")
|
| 47 |
+
|
| 48 |
+
c5_independent = read_csv(root / "claim-5" / "independent_groups.csv")
|
| 49 |
+
c5_taxonomy = read_csv(root / "claim-5" / "taxonomy.csv")
|
| 50 |
+
c5_control = json.loads((root / "claim-5" / "negative_control.json").read_text())
|
| 51 |
+
if len(c5_independent) != 25 or sum(int(row["events"]) for row in c5_independent) != 20650:
|
| 52 |
+
failures.append("claim_5_independent_counts")
|
| 53 |
+
if len(c5_taxonomy) != 25 or sum(int(row["events"]) for row in c5_taxonomy) != 6350:
|
| 54 |
+
failures.append("claim_5_taxonomy_counts")
|
| 55 |
+
if max(float(row["max_probability_error"]) for row in c5_independent + c5_taxonomy) >= 1e-14:
|
| 56 |
+
failures.append("claim_5_probability")
|
| 57 |
+
if max(float(row["max_score_error"]) for row in c5_independent + c5_taxonomy) >= 1e-14:
|
| 58 |
+
failures.append("claim_5_score")
|
| 59 |
+
if not c5_control["rejected"]:
|
| 60 |
+
failures.append("claim_5_control")
|
| 61 |
+
|
| 62 |
+
result = {
|
| 63 |
+
"checker": "standard-library artifact checker",
|
| 64 |
+
"status": "PASS" if not failures else "FAIL",
|
| 65 |
+
"failures": failures,
|
| 66 |
+
"checked_claims": [1, 2, 5],
|
| 67 |
+
}
|
| 68 |
+
print(json.dumps(result, indent=2, sort_keys=True))
|
| 69 |
+
if failures:
|
| 70 |
+
raise SystemExit(1)
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
if __name__ == "__main__":
|
| 74 |
+
main()
|
code/check_claim3_release.py
ADDED
|
@@ -0,0 +1,89 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Independent standard-library checker for Claim 3 release integrity."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
import json
|
| 7 |
+
import sys
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
EXPECTED_NOT = {
|
| 12 |
+
"constant_cs": 0.75,
|
| 13 |
+
"constant_fid": 32.87,
|
| 14 |
+
"logdiff_cs": 0.80,
|
| 15 |
+
"logdiff_fid": 23.61,
|
| 16 |
+
}
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def main() -> None:
|
| 20 |
+
if len(sys.argv) != 2:
|
| 21 |
+
raise SystemExit("usage: check_claim3_release.py ARTIFACT_DIR")
|
| 22 |
+
root = Path(sys.argv[1]) / "claim-3"
|
| 23 |
+
raw = json.loads((root / "release_audit.json").read_text())
|
| 24 |
+
control = json.loads((root / "negative_control.json").read_text())
|
| 25 |
+
routes = json.loads((root / "verification_routes.json").read_text())
|
| 26 |
+
search = json.loads((root / "public_checkpoint_search_summary.json").read_text())
|
| 27 |
+
falsification = json.loads((root / "falsification_check.json").read_text())
|
| 28 |
+
failures: list[str] = []
|
| 29 |
+
|
| 30 |
+
if raw["source_commit"] != "94ef35bafd4b4239e9832d8295128c09e8fc1472":
|
| 31 |
+
failures.append("source_commit")
|
| 32 |
+
if raw["tree_file_count"] != 73:
|
| 33 |
+
failures.append("tree_file_count")
|
| 34 |
+
if len(raw["missing_checkpoint_paths"]) != 3:
|
| 35 |
+
failures.append("checkpoint_count")
|
| 36 |
+
if not raw["missing_composition_dataset_config"]:
|
| 37 |
+
failures.append("dataset_config")
|
| 38 |
+
if not raw["fid_implementation_mismatch"]:
|
| 39 |
+
failures.append("fid_protocol")
|
| 40 |
+
if raw["generated_samples_per_task"] != 5000 or raw["reported_samples_argument"] != 500000:
|
| 41 |
+
failures.append("sample_accounting")
|
| 42 |
+
if raw["table_3"]["NOT"] != EXPECTED_NOT:
|
| 43 |
+
failures.append("table_transcription")
|
| 44 |
+
if raw["claim_status"] != "BLOCKED" or raw["empirical_generation_executed"]:
|
| 45 |
+
failures.append("fail_closed_status")
|
| 46 |
+
if control["audit_status"] != "READY_TO_RUN" or not control["rejected_false_block"]:
|
| 47 |
+
failures.append("negative_control")
|
| 48 |
+
route_rows = routes.get("routes", [])
|
| 49 |
+
if [row.get("route") for row in route_rows] != [1, 2, 3, 4]:
|
| 50 |
+
failures.append("four_route_sequence")
|
| 51 |
+
if len({row.get("name") for row in route_rows}) != 4:
|
| 52 |
+
failures.append("distinct_route_names")
|
| 53 |
+
if routes.get("final_status") != "BLOCKED" or routes.get("confidence") != "LOW":
|
| 54 |
+
failures.append("route_fail_closed_status")
|
| 55 |
+
if route_rows and route_rows[-1].get("falsification_succeeded") is not False:
|
| 56 |
+
failures.append("falsification_route_status")
|
| 57 |
+
if search.get("author_equivalent_checkpoint_found") is not False:
|
| 58 |
+
failures.append("checkpoint_search_status")
|
| 59 |
+
proxy = search.get("public_proxy_model", {})
|
| 60 |
+
if (
|
| 61 |
+
proxy.get("revision") != "91d0ff096e031c21b8313bd5877316a52900d4ec"
|
| 62 |
+
or proxy.get("unet_bytes") != 114049969
|
| 63 |
+
or proxy.get("exact_protocol_equivalent") is not False
|
| 64 |
+
):
|
| 65 |
+
failures.append("public_model_control")
|
| 66 |
+
if (
|
| 67 |
+
falsification.get("claimed_not_fid_ordering_holds_in_paper_table") is not True
|
| 68 |
+
or falsification.get("available_evidence_admissible_as_counterexample") is not False
|
| 69 |
+
or falsification.get("falsification_succeeded") is not False
|
| 70 |
+
or falsification.get("status") != "BLOCKED"
|
| 71 |
+
):
|
| 72 |
+
failures.append("falsification_check")
|
| 73 |
+
if not falsification.get("synthetic_negative_control", {}).get("reversal_detected"):
|
| 74 |
+
failures.append("falsification_negative_control")
|
| 75 |
+
|
| 76 |
+
result = {
|
| 77 |
+
"checker": "independent Claim 3 release-integrity checker",
|
| 78 |
+
"status": "PASS" if not failures else "FAIL",
|
| 79 |
+
"failures": failures,
|
| 80 |
+
"verification_routes_rederived": len(route_rows),
|
| 81 |
+
"scientific_claim_status": raw.get("claim_status"),
|
| 82 |
+
}
|
| 83 |
+
print(json.dumps(result, indent=2, sort_keys=True))
|
| 84 |
+
if failures:
|
| 85 |
+
raise SystemExit(1)
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
if __name__ == "__main__":
|
| 89 |
+
main()
|
code/check_claim4_evidence.py
ADDED
|
@@ -0,0 +1,138 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Independent standard-library checker for the Claim 4 evidence audit."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
import hashlib
|
| 7 |
+
import json
|
| 8 |
+
import re
|
| 9 |
+
import sys
|
| 10 |
+
from pathlib import Path
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
SLICE_RE = re.compile(
|
| 14 |
+
r"campaigns/(logdiff_and|logdiff_and_not)/run-seed-(\d+)/"
|
| 15 |
+
r"slice-(s\d+)-samples-(\d+)-(\d+)/contract-([0-9a-f]{64})/SUCCESS\.json$"
|
| 16 |
+
)
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def sha256(path: Path) -> str:
|
| 20 |
+
return hashlib.sha256(path.read_bytes()).hexdigest()
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def main() -> None:
|
| 24 |
+
if len(sys.argv) != 2:
|
| 25 |
+
raise SystemExit("usage: check_claim4_evidence.py ARTIFACT_DIR")
|
| 26 |
+
root = Path(sys.argv[1]) / "claim-4"
|
| 27 |
+
snapshot = root / "bucket-json-snapshot"
|
| 28 |
+
summary = json.loads((root / "bucket_audit.json").read_text())
|
| 29 |
+
control = json.loads((root / "negative_control.json").read_text())
|
| 30 |
+
routes = json.loads((root / "verification_routes.json").read_text())
|
| 31 |
+
falsification = json.loads((root / "falsification_check.json").read_text())
|
| 32 |
+
failures: list[str] = []
|
| 33 |
+
slices = []
|
| 34 |
+
bindings = set()
|
| 35 |
+
|
| 36 |
+
for success_path in snapshot.rglob("SUCCESS.json"):
|
| 37 |
+
relative = success_path.relative_to(snapshot).as_posix()
|
| 38 |
+
match = SLICE_RE.match(relative)
|
| 39 |
+
if not match:
|
| 40 |
+
continue
|
| 41 |
+
logic, seed, slice_id, start, stop, contract = match.groups()
|
| 42 |
+
success = json.loads(success_path.read_text())
|
| 43 |
+
manifest_path = success_path.with_name("SLICE_MANIFEST.json")
|
| 44 |
+
manifest = json.loads(manifest_path.read_text())
|
| 45 |
+
if success.get("status") != "complete":
|
| 46 |
+
failures.append(f"slice_status:{relative}")
|
| 47 |
+
if success.get("slice_manifest_sha256") != sha256(manifest_path):
|
| 48 |
+
failures.append(f"slice_manifest_hash:{relative}")
|
| 49 |
+
if (
|
| 50 |
+
success.get("logic") != logic
|
| 51 |
+
or int(success.get("run_seed", -1)) != int(seed)
|
| 52 |
+
or success.get("slice_id") != slice_id
|
| 53 |
+
):
|
| 54 |
+
failures.append(f"slice_identity:{relative}")
|
| 55 |
+
if manifest.get("sample_indices") != list(range(int(start), int(stop))):
|
| 56 |
+
failures.append(f"sample_range:{relative}")
|
| 57 |
+
binding_text = json.dumps(success.get("binding"), sort_keys=True)
|
| 58 |
+
bindings.add(binding_text)
|
| 59 |
+
if success.get("binding") != manifest.get("binding"):
|
| 60 |
+
failures.append(f"binding_mismatch:{relative}")
|
| 61 |
+
for sample in manifest.get("samples", []):
|
| 62 |
+
index = int(sample["sample_index"])
|
| 63 |
+
sample_root = success_path.parent / "samples" / f"sample-{index:03d}"
|
| 64 |
+
sample_success = sample_root / "SUCCESS.json"
|
| 65 |
+
files_manifest = sample_root / "FILES.json"
|
| 66 |
+
if not sample_success.exists() or sha256(sample_success) != sample["success_sha256"]:
|
| 67 |
+
failures.append(f"sample_success_hash:{relative}:{index}")
|
| 68 |
+
continue
|
| 69 |
+
sample_record = json.loads(sample_success.read_text())
|
| 70 |
+
if (
|
| 71 |
+
sample_record.get("logic") != logic
|
| 72 |
+
or sample_record.get("run_seed") != int(seed)
|
| 73 |
+
or sample_record.get("sample_index") != index
|
| 74 |
+
or sample_record.get("sample_seed") != int(seed) * 100000 + index
|
| 75 |
+
or sample_record.get("num_steps") != 1000
|
| 76 |
+
or sample_record.get("execution_device") != "cpu"
|
| 77 |
+
):
|
| 78 |
+
failures.append(f"sample_identity:{relative}:{index}")
|
| 79 |
+
if not files_manifest.exists() or sha256(files_manifest) != sample_record.get(
|
| 80 |
+
"files_manifest_sha256"
|
| 81 |
+
):
|
| 82 |
+
failures.append(f"files_manifest_hash:{relative}:{index}")
|
| 83 |
+
slices.append((logic, int(seed), slice_id))
|
| 84 |
+
|
| 85 |
+
if len(slices) != 12 or len(set(slices)) != 12:
|
| 86 |
+
failures.append("terminal_slice_count")
|
| 87 |
+
if len(bindings) != 1:
|
| 88 |
+
failures.append("uniform_binding")
|
| 89 |
+
if summary.get("terminal_slices") != 12 or summary.get("manifest_bound_samples") != 96:
|
| 90 |
+
failures.append("summary_counts")
|
| 91 |
+
if summary.get("complete_logdiff_experiments") != 0:
|
| 92 |
+
failures.append("complete_experiment_count")
|
| 93 |
+
if summary.get("payload_paths_in_listing") != 96:
|
| 94 |
+
failures.append("payload_listing_count")
|
| 95 |
+
if summary.get("dualdiff_experiments") != 0 or summary.get("docking_result_files") != 0:
|
| 96 |
+
failures.append("missing_routes_misreported")
|
| 97 |
+
if summary.get("claim_status") != "BLOCKED":
|
| 98 |
+
failures.append("fail_closed_status")
|
| 99 |
+
if control.get("audit_status") != "READY_FOR_CLAIM_CHECK" or not control.get(
|
| 100 |
+
"rejected_false_block"
|
| 101 |
+
):
|
| 102 |
+
failures.append("negative_control")
|
| 103 |
+
route_rows = routes.get("routes", [])
|
| 104 |
+
if [row.get("route") for row in route_rows] != [1, 2, 3, 4]:
|
| 105 |
+
failures.append("four_route_sequence")
|
| 106 |
+
if len({row.get("name") for row in route_rows}) != 4:
|
| 107 |
+
failures.append("distinct_route_names")
|
| 108 |
+
if routes.get("final_status") != "BLOCKED" or routes.get("confidence") != "LOW":
|
| 109 |
+
failures.append("route_fail_closed_status")
|
| 110 |
+
if route_rows and route_rows[-1].get("falsification_succeeded") is not False:
|
| 111 |
+
failures.append("falsification_route_status")
|
| 112 |
+
if (
|
| 113 |
+
falsification.get("claimed_ordering_holds_in_paper_table") is not True
|
| 114 |
+
or falsification.get("recovered_evidence_admissible_as_counterexample") is not False
|
| 115 |
+
or falsification.get("falsification_succeeded") is not False
|
| 116 |
+
or falsification.get("status") != "BLOCKED"
|
| 117 |
+
):
|
| 118 |
+
failures.append("falsification_check")
|
| 119 |
+
if not falsification.get("synthetic_negative_control", {}).get(
|
| 120 |
+
"contradiction_detected"
|
| 121 |
+
):
|
| 122 |
+
failures.append("falsification_negative_control")
|
| 123 |
+
|
| 124 |
+
result = {
|
| 125 |
+
"checker": "independent Claim 4 manifest and completeness checker",
|
| 126 |
+
"status": "PASS" if not failures else "FAIL",
|
| 127 |
+
"failures": failures,
|
| 128 |
+
"terminal_slices_rederived": len(slices),
|
| 129 |
+
"verification_routes_rederived": len(route_rows),
|
| 130 |
+
"scientific_claim_status": summary.get("claim_status"),
|
| 131 |
+
}
|
| 132 |
+
print(json.dumps(result, indent=2, sort_keys=True))
|
| 133 |
+
if failures:
|
| 134 |
+
raise SystemExit(1)
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
if __name__ == "__main__":
|
| 138 |
+
main()
|
code/run_logdiff.py
ADDED
|
@@ -0,0 +1,858 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Exhaustive finite-state reproduction of LOGDIFF's exact Boolean calculus.
|
| 3 |
+
|
| 4 |
+
The paper's Proposition 3.1 is continuous, while Proposition C.3 supplies the
|
| 5 |
+
discrete equivalent. Here atom posteriors are smooth softmax functions of x;
|
| 6 |
+
their exact log-gradients play the role of diffusion guidance scores. Direct
|
| 7 |
+
enumeration of the full joint distribution is an independent oracle.
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
from __future__ import annotations
|
| 11 |
+
|
| 12 |
+
import argparse
|
| 13 |
+
import csv
|
| 14 |
+
import hashlib
|
| 15 |
+
import itertools
|
| 16 |
+
import json
|
| 17 |
+
import math
|
| 18 |
+
import os
|
| 19 |
+
import platform
|
| 20 |
+
import re
|
| 21 |
+
import subprocess
|
| 22 |
+
import sys
|
| 23 |
+
import time
|
| 24 |
+
from dataclasses import dataclass
|
| 25 |
+
from pathlib import Path
|
| 26 |
+
|
| 27 |
+
import numpy as np
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
@dataclass(frozen=True)
|
| 31 |
+
class Value:
|
| 32 |
+
probability: float
|
| 33 |
+
score: np.ndarray
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def softmax(logits: np.ndarray) -> np.ndarray:
|
| 37 |
+
shifted = logits - np.max(logits)
|
| 38 |
+
weights = np.exp(shifted)
|
| 39 |
+
return weights / weights.sum()
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def categorical_model(logits_by_variable: list[np.ndarray]) -> tuple[list[np.ndarray], list[list[Value]]]:
|
| 43 |
+
"""Return categorical probabilities and atom (probability, grad-log-p) values."""
|
| 44 |
+
probabilities = [softmax(np.asarray(x, dtype=float)) for x in logits_by_variable]
|
| 45 |
+
dimension = sum(len(p) for p in probabilities)
|
| 46 |
+
atoms: list[list[Value]] = []
|
| 47 |
+
offset = 0
|
| 48 |
+
for probs in probabilities:
|
| 49 |
+
variable_atoms = []
|
| 50 |
+
for value, probability in enumerate(probs):
|
| 51 |
+
score = np.zeros(dimension)
|
| 52 |
+
score[offset : offset + len(probs)] = -probs
|
| 53 |
+
score[offset + value] += 1.0
|
| 54 |
+
variable_atoms.append(Value(float(probability), score))
|
| 55 |
+
atoms.append(variable_atoms)
|
| 56 |
+
offset += len(probs)
|
| 57 |
+
return probabilities, atoms
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
# Table 1, independently transcribed.
|
| 61 |
+
def negate(value: Value) -> Value:
|
| 62 |
+
probability = 1.0 - value.probability
|
| 63 |
+
return Value(probability, -value.probability / probability * value.score)
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def conjunction_ci(left: Value, right: Value) -> Value:
|
| 67 |
+
return Value(left.probability * right.probability, left.score + right.score)
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
def disjunction_ci(left: Value, right: Value) -> Value:
|
| 71 |
+
pl, pr = left.probability, right.probability
|
| 72 |
+
probability = pl + pr - pl * pr
|
| 73 |
+
numerator = pl * (1 - pr) * left.score + pr * (1 - pl) * right.score
|
| 74 |
+
return Value(probability, numerator / probability)
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def disjunction_me(left: Value, right: Value) -> Value:
|
| 78 |
+
probability = left.probability + right.probability
|
| 79 |
+
numerator = left.probability * left.score + right.probability * right.score
|
| 80 |
+
return Value(probability, numerator / probability)
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
def assignment_value(atoms: list[list[Value]], assignment: tuple[int, ...]) -> Value:
|
| 84 |
+
value = atoms[0][assignment[0]]
|
| 85 |
+
for variable, category in enumerate(assignment[1:], start=1):
|
| 86 |
+
value = conjunction_ci(value, atoms[variable][category])
|
| 87 |
+
return value
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
def compiled_dnf(atoms: list[list[Value]], selected: tuple[tuple[int, ...], ...]) -> Value:
|
| 91 |
+
"""Proposition 3.2: CI assignment terms joined by mutually-exclusive ORs."""
|
| 92 |
+
terms = [assignment_value(atoms, assignment) for assignment in selected]
|
| 93 |
+
value = terms[0]
|
| 94 |
+
for term in terms[1:]:
|
| 95 |
+
value = disjunction_me(value, term)
|
| 96 |
+
return value
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
def direct_event(
|
| 100 |
+
probabilities: list[np.ndarray], atoms: list[list[Value]], selected: tuple[tuple[int, ...], ...]
|
| 101 |
+
) -> Value:
|
| 102 |
+
"""Independent oracle: enumerate joint assignments, then differentiate the sum."""
|
| 103 |
+
joint_probabilities = []
|
| 104 |
+
joint_gradients = []
|
| 105 |
+
for assignment in selected:
|
| 106 |
+
probability = float(np.prod([probabilities[m][v] for m, v in enumerate(assignment)]))
|
| 107 |
+
score = sum((atoms[m][v].score for m, v in enumerate(assignment)), np.zeros_like(atoms[0][0].score))
|
| 108 |
+
joint_probabilities.append(probability)
|
| 109 |
+
joint_gradients.append(probability * score)
|
| 110 |
+
event_probability = float(sum(joint_probabilities))
|
| 111 |
+
gradient = sum(joint_gradients, np.zeros_like(atoms[0][0].score))
|
| 112 |
+
return Value(event_probability, gradient / event_probability)
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
def event_probability_from_flat_logits(
|
| 116 |
+
flat_logits: np.ndarray, sizes: tuple[int, ...], selected: tuple[tuple[int, ...], ...]
|
| 117 |
+
) -> float:
|
| 118 |
+
pieces, offset = [], 0
|
| 119 |
+
for size in sizes:
|
| 120 |
+
pieces.append(softmax(flat_logits[offset : offset + size]))
|
| 121 |
+
offset += size
|
| 122 |
+
return float(sum(np.prod([pieces[m][v] for m, v in enumerate(a)]) for a in selected))
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
def finite_difference_score(
|
| 126 |
+
logits_by_variable: list[np.ndarray], selected: tuple[tuple[int, ...], ...], epsilon: float = 1e-6
|
| 127 |
+
) -> np.ndarray:
|
| 128 |
+
sizes = tuple(len(x) for x in logits_by_variable)
|
| 129 |
+
flat = np.concatenate(logits_by_variable).astype(float)
|
| 130 |
+
result = np.zeros_like(flat)
|
| 131 |
+
for coordinate in range(len(flat)):
|
| 132 |
+
plus, minus = flat.copy(), flat.copy()
|
| 133 |
+
plus[coordinate] += epsilon
|
| 134 |
+
minus[coordinate] -= epsilon
|
| 135 |
+
result[coordinate] = (
|
| 136 |
+
math.log(event_probability_from_flat_logits(plus, sizes, selected))
|
| 137 |
+
- math.log(event_probability_from_flat_logits(minus, sizes, selected))
|
| 138 |
+
) / (2 * epsilon)
|
| 139 |
+
return result
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
def exhaustive_binary(seed: int) -> dict[str, float | int]:
|
| 143 |
+
rng = np.random.default_rng(seed)
|
| 144 |
+
logits = [rng.normal(size=2) for _ in range(3)]
|
| 145 |
+
probabilities, atoms = categorical_model(logits)
|
| 146 |
+
assignments = tuple(itertools.product(range(2), repeat=3))
|
| 147 |
+
max_probability_error = 0.0
|
| 148 |
+
max_score_error = 0.0
|
| 149 |
+
max_fd_error = 0.0
|
| 150 |
+
formula_count = 0
|
| 151 |
+
for mask in range(1, 2 ** len(assignments) - 1):
|
| 152 |
+
selected = tuple(a for i, a in enumerate(assignments) if mask & (1 << i))
|
| 153 |
+
recursive = compiled_dnf(atoms, selected)
|
| 154 |
+
oracle = direct_event(probabilities, atoms, selected)
|
| 155 |
+
max_probability_error = max(max_probability_error, abs(recursive.probability - oracle.probability))
|
| 156 |
+
max_score_error = max(max_score_error, float(np.max(np.abs(recursive.score - oracle.score))))
|
| 157 |
+
# Finite differences are costlier; triangulate every seventeenth formula.
|
| 158 |
+
if mask % 17 == 0:
|
| 159 |
+
fd = finite_difference_score(logits, selected)
|
| 160 |
+
max_fd_error = max(max_fd_error, float(np.max(np.abs(recursive.score - fd))))
|
| 161 |
+
formula_count += 1
|
| 162 |
+
return {
|
| 163 |
+
"seed": seed,
|
| 164 |
+
"formulas": formula_count,
|
| 165 |
+
"max_probability_error": max_probability_error,
|
| 166 |
+
"max_score_error": max_score_error,
|
| 167 |
+
"max_finite_difference_error": max_fd_error,
|
| 168 |
+
}
|
| 169 |
+
|
| 170 |
+
|
| 171 |
+
def primitive_checks(seed: int) -> list[dict]:
|
| 172 |
+
"""Exercise every Table-1 operator against direct enumeration."""
|
| 173 |
+
rng = np.random.default_rng(seed)
|
| 174 |
+
logits = [rng.normal(size=3), rng.normal(size=2)]
|
| 175 |
+
probabilities, atoms = categorical_model(logits)
|
| 176 |
+
assignments = tuple(itertools.product(range(3), range(2)))
|
| 177 |
+
cases = []
|
| 178 |
+
|
| 179 |
+
def oracle(predicate) -> Value:
|
| 180 |
+
selected = tuple(a for a in assignments if predicate(a))
|
| 181 |
+
return direct_event(probabilities, atoms, selected)
|
| 182 |
+
|
| 183 |
+
candidates = [
|
| 184 |
+
("negation", negate(atoms[0][0]), oracle(lambda a: a[0] != 0)),
|
| 185 |
+
("conjunction_ci", conjunction_ci(atoms[0][1], atoms[1][0]), oracle(lambda a: a[0] == 1 and a[1] == 0)),
|
| 186 |
+
("disjunction_ci", disjunction_ci(atoms[0][1], atoms[1][0]), oracle(lambda a: a[0] == 1 or a[1] == 0)),
|
| 187 |
+
("disjunction_me", disjunction_me(atoms[0][0], atoms[0][2]), oracle(lambda a: a[0] in (0, 2))),
|
| 188 |
+
]
|
| 189 |
+
for operator, recursive, direct in candidates:
|
| 190 |
+
cases.append(
|
| 191 |
+
{
|
| 192 |
+
"seed": seed,
|
| 193 |
+
"operator": operator,
|
| 194 |
+
"probability_error": abs(recursive.probability - direct.probability),
|
| 195 |
+
"score_max_error": float(np.max(np.abs(recursive.score - direct.score))),
|
| 196 |
+
}
|
| 197 |
+
)
|
| 198 |
+
return cases
|
| 199 |
+
|
| 200 |
+
|
| 201 |
+
def dependent_controls() -> list[dict]:
|
| 202 |
+
"""Rules must fail closed when CI/ME structural assumptions are false."""
|
| 203 |
+
logits = np.array([0.2, -0.7, 1.1, 0.4]) # joint logits for (00,01,10,11)
|
| 204 |
+
joint = softmax(logits)
|
| 205 |
+
assignments = ((0, 0), (0, 1), (1, 0), (1, 1))
|
| 206 |
+
|
| 207 |
+
def event(indices: tuple[int, ...]) -> Value:
|
| 208 |
+
probability = float(joint[list(indices)].sum())
|
| 209 |
+
gradient = np.zeros(4)
|
| 210 |
+
for index in indices:
|
| 211 |
+
score = -joint.copy()
|
| 212 |
+
score[index] += 1
|
| 213 |
+
gradient += joint[index] * score
|
| 214 |
+
return Value(probability, gradient / probability)
|
| 215 |
+
|
| 216 |
+
left = event((2, 3)) # Z0=1
|
| 217 |
+
right = event((1, 3)) # Z1=1; correlated with left
|
| 218 |
+
true_and = event((3,))
|
| 219 |
+
true_or = event((1, 2, 3))
|
| 220 |
+
wrong_and = conjunction_ci(left, right)
|
| 221 |
+
wrong_or = disjunction_ci(left, right)
|
| 222 |
+
|
| 223 |
+
# Constant 0.5 score mixing is the principal heuristic baseline criticized by the paper.
|
| 224 |
+
exact_or_me = disjunction_me(event((0,)), event((1,)))
|
| 225 |
+
constant_score = 0.5 * event((0,)).score + 0.5 * event((1,)).score
|
| 226 |
+
return [
|
| 227 |
+
{
|
| 228 |
+
"control": "CI_conjunction_on_correlated_events",
|
| 229 |
+
"probability_error": abs(wrong_and.probability - true_and.probability),
|
| 230 |
+
"score_max_error": float(np.max(np.abs(wrong_and.score - true_and.score))),
|
| 231 |
+
"rejected": abs(wrong_and.probability - true_and.probability) > 1e-3,
|
| 232 |
+
},
|
| 233 |
+
{
|
| 234 |
+
"control": "CI_disjunction_on_correlated_events",
|
| 235 |
+
"probability_error": abs(wrong_or.probability - true_or.probability),
|
| 236 |
+
"score_max_error": float(np.max(np.abs(wrong_or.score - true_or.score))),
|
| 237 |
+
"rejected": abs(wrong_or.probability - true_or.probability) > 1e-3,
|
| 238 |
+
},
|
| 239 |
+
{
|
| 240 |
+
"control": "constant_half_mix_for_ME_disjunction",
|
| 241 |
+
"probability_error": 0.0,
|
| 242 |
+
"score_max_error": float(np.max(np.abs(constant_score - exact_or_me.score))),
|
| 243 |
+
"rejected": float(np.max(np.abs(constant_score - exact_or_me.score))) > 1e-3,
|
| 244 |
+
},
|
| 245 |
+
]
|
| 246 |
+
|
| 247 |
+
|
| 248 |
+
def write_csv(path: Path, rows: list[dict]) -> None:
|
| 249 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 250 |
+
with path.open("w", newline="") as handle:
|
| 251 |
+
writer = csv.DictWriter(handle, fieldnames=list(rows[0]))
|
| 252 |
+
writer.writeheader()
|
| 253 |
+
writer.writerows(rows)
|
| 254 |
+
|
| 255 |
+
|
| 256 |
+
def exhaustive_categorical_groups(seed: int) -> dict[str, float | int]:
|
| 257 |
+
"""Exhaust Proposition C.2's independent-categorical construction.
|
| 258 |
+
|
| 259 |
+
The three complete finite domains have respectively 8, 6, and 9 joint
|
| 260 |
+
assignments, so they contain 254 + 62 + 510 = 826 nonconstant semantic
|
| 261 |
+
events per posterior setting.
|
| 262 |
+
"""
|
| 263 |
+
rng = np.random.default_rng(10_000 + seed)
|
| 264 |
+
event_count = 0
|
| 265 |
+
max_probability_error = 0.0
|
| 266 |
+
max_score_error = 0.0
|
| 267 |
+
for sizes in ((2, 2, 2), (3, 2), (3, 3)):
|
| 268 |
+
logits = [rng.normal(size=size) for size in sizes]
|
| 269 |
+
probabilities, atoms = categorical_model(logits)
|
| 270 |
+
assignments = tuple(itertools.product(*(range(size) for size in sizes)))
|
| 271 |
+
for mask in range(1, 2 ** len(assignments) - 1):
|
| 272 |
+
selected = tuple(a for i, a in enumerate(assignments) if mask & (1 << i))
|
| 273 |
+
compiled = compiled_dnf(atoms, selected)
|
| 274 |
+
oracle = direct_event(probabilities, atoms, selected)
|
| 275 |
+
max_probability_error = max(max_probability_error, abs(compiled.probability - oracle.probability))
|
| 276 |
+
max_score_error = max(max_score_error, float(np.max(np.abs(compiled.score - oracle.score))))
|
| 277 |
+
event_count += 1
|
| 278 |
+
return {
|
| 279 |
+
"seed": seed,
|
| 280 |
+
"events": event_count,
|
| 281 |
+
"max_probability_error": max_probability_error,
|
| 282 |
+
"max_score_error": max_score_error,
|
| 283 |
+
}
|
| 284 |
+
|
| 285 |
+
|
| 286 |
+
def categorical_leaf_model(logits: np.ndarray) -> tuple[np.ndarray, list[Value]]:
|
| 287 |
+
probabilities = softmax(logits)
|
| 288 |
+
atoms: list[Value] = []
|
| 289 |
+
for index, probability in enumerate(probabilities):
|
| 290 |
+
score = -probabilities.copy()
|
| 291 |
+
score[index] += 1.0
|
| 292 |
+
atoms.append(Value(float(probability), score))
|
| 293 |
+
return probabilities, atoms
|
| 294 |
+
|
| 295 |
+
|
| 296 |
+
def taxonomy_event(probabilities: np.ndarray, atoms: list[Value], selected: tuple[int, ...]) -> Value:
|
| 297 |
+
probability = float(probabilities[list(selected)].sum())
|
| 298 |
+
gradient = sum((probabilities[index] * atoms[index].score for index in selected), np.zeros(len(atoms)))
|
| 299 |
+
return Value(probability, gradient / probability)
|
| 300 |
+
|
| 301 |
+
|
| 302 |
+
def exhaustive_taxonomy(seed: int) -> dict[str, float | int]:
|
| 303 |
+
"""Exhaust all nonconstant events over the eight leaves of a binary taxonomy."""
|
| 304 |
+
rng = np.random.default_rng(20_000 + seed)
|
| 305 |
+
probabilities, atoms = categorical_leaf_model(rng.normal(size=8))
|
| 306 |
+
max_probability_error = 0.0
|
| 307 |
+
max_score_error = 0.0
|
| 308 |
+
event_count = 0
|
| 309 |
+
for mask in range(1, 2 ** len(atoms) - 1):
|
| 310 |
+
selected = tuple(index for index in range(len(atoms)) if mask & (1 << index))
|
| 311 |
+
terms = [atoms[index] for index in selected]
|
| 312 |
+
compiled = terms[0]
|
| 313 |
+
for term in terms[1:]:
|
| 314 |
+
compiled = disjunction_me(compiled, term)
|
| 315 |
+
oracle = taxonomy_event(probabilities, atoms, selected)
|
| 316 |
+
max_probability_error = max(max_probability_error, abs(compiled.probability - oracle.probability))
|
| 317 |
+
max_score_error = max(max_score_error, float(np.max(np.abs(compiled.score - oracle.score))))
|
| 318 |
+
event_count += 1
|
| 319 |
+
return {
|
| 320 |
+
"seed": seed,
|
| 321 |
+
"events": event_count,
|
| 322 |
+
"taxonomy_predicates": 15,
|
| 323 |
+
"max_probability_error": max_probability_error,
|
| 324 |
+
"max_score_error": max_score_error,
|
| 325 |
+
}
|
| 326 |
+
|
| 327 |
+
|
| 328 |
+
def taxonomy_overlap_control() -> dict[str, float | bool | str]:
|
| 329 |
+
probabilities, atoms = categorical_leaf_model(np.array([0.2, -0.7, 1.1, 0.4, -0.1, 0.8, -0.3, 0.6]))
|
| 330 |
+
left_leaves = (0, 1, 2, 3)
|
| 331 |
+
right_leaves = (2, 3, 4, 5)
|
| 332 |
+
left = taxonomy_event(probabilities, atoms, left_leaves)
|
| 333 |
+
right = taxonomy_event(probabilities, atoms, right_leaves)
|
| 334 |
+
true_union = taxonomy_event(probabilities, atoms, (0, 1, 2, 3, 4, 5))
|
| 335 |
+
invalid_me = disjunction_me(left, right)
|
| 336 |
+
probability_error = abs(invalid_me.probability - true_union.probability)
|
| 337 |
+
score_error = float(np.max(np.abs(invalid_me.score - true_union.score)))
|
| 338 |
+
return {
|
| 339 |
+
"control": "overlapping_non_nested_non_me_taxonomy_pair",
|
| 340 |
+
"left_leaves": str(left_leaves),
|
| 341 |
+
"right_leaves": str(right_leaves),
|
| 342 |
+
"probability_error": probability_error,
|
| 343 |
+
"score_max_error": score_error,
|
| 344 |
+
"rejected": probability_error > 1e-3 and score_error > 1e-3,
|
| 345 |
+
}
|
| 346 |
+
|
| 347 |
+
|
| 348 |
+
TABLE_2_ROWS = [
|
| 349 |
+
{"dataset": "CMNIST", "method": "LOGDIFF", "N2": 93.8, "N3": 93.3, "N4": 94.2, "N5": 94.4},
|
| 350 |
+
{"dataset": "Shapes3D", "method": "LOGDIFF", "N2": 88.8, "N3": 88.6, "N4": 85.1, "N5": 87.6},
|
| 351 |
+
{"dataset": "CMNIST", "method": "constant", "N2": 76.1, "N3": 66.7, "N4": 68.3, "N5": 75.2},
|
| 352 |
+
{"dataset": "Shapes3D", "method": "constant", "N2": 67.2, "N3": 59.4, "N4": 58.4, "N5": 57.9},
|
| 353 |
+
]
|
| 354 |
+
|
| 355 |
+
|
| 356 |
+
def audit_table_2(rows: list[dict] = TABLE_2_ROWS) -> tuple[dict, list[dict]]:
|
| 357 |
+
"""Audit the challenge's literal ranges against every N=2..5 table cell."""
|
| 358 |
+
expanded = []
|
| 359 |
+
for row in rows:
|
| 360 |
+
expected_low, expected_high = (94.0, 98.0) if row["method"] == "LOGDIFF" else (63.0, 77.0)
|
| 361 |
+
for operator_count in range(2, 6):
|
| 362 |
+
value = float(row[f"N{operator_count}"])
|
| 363 |
+
expanded.append(
|
| 364 |
+
{
|
| 365 |
+
"dataset": row["dataset"],
|
| 366 |
+
"method": row["method"],
|
| 367 |
+
"operators": operator_count,
|
| 368 |
+
"value_percent": value,
|
| 369 |
+
"claimed_low": expected_low,
|
| 370 |
+
"claimed_high": expected_high,
|
| 371 |
+
"inside_claimed_range": expected_low <= value <= expected_high,
|
| 372 |
+
}
|
| 373 |
+
)
|
| 374 |
+
by_method = {}
|
| 375 |
+
for method in ("LOGDIFF", "constant"):
|
| 376 |
+
cells = [row for row in expanded if row["method"] == method]
|
| 377 |
+
values = [float(row["value_percent"]) for row in cells]
|
| 378 |
+
by_method[method] = {
|
| 379 |
+
"minimum": min(values),
|
| 380 |
+
"maximum": max(values),
|
| 381 |
+
"inside_claimed_range": sum(bool(row["inside_claimed_range"]) for row in cells),
|
| 382 |
+
"total_cells": len(cells),
|
| 383 |
+
}
|
| 384 |
+
falsified = (
|
| 385 |
+
by_method["LOGDIFF"]["inside_claimed_range"] < by_method["LOGDIFF"]["total_cells"]
|
| 386 |
+
and by_method["constant"]["inside_claimed_range"] < by_method["constant"]["total_cells"]
|
| 387 |
+
)
|
| 388 |
+
return {"status": "FALSIFIED" if falsified else "VERIFIER_REJECTED", **by_method}, expanded
|
| 389 |
+
|
| 390 |
+
|
| 391 |
+
def table_2_negative_control() -> dict[str, bool | str]:
|
| 392 |
+
"""A range-conforming synthetic table must not be called a falsification."""
|
| 393 |
+
conforming = [
|
| 394 |
+
{"dataset": dataset, "method": method, "N2": value, "N3": value, "N4": value, "N5": value}
|
| 395 |
+
for dataset in ("CMNIST", "Shapes3D")
|
| 396 |
+
for method, value in (("LOGDIFF", 96.0), ("constant", 70.0))
|
| 397 |
+
]
|
| 398 |
+
summary, _ = audit_table_2(conforming)
|
| 399 |
+
return {
|
| 400 |
+
"control": "range_conforming_table",
|
| 401 |
+
"verifier_rejected_false_falsification": summary["status"] == "VERIFIER_REJECTED",
|
| 402 |
+
}
|
| 403 |
+
|
| 404 |
+
|
| 405 |
+
def classify_celeba_release(snapshot: dict) -> tuple[str, dict]:
|
| 406 |
+
tree_paths = set(snapshot["tree_paths"])
|
| 407 |
+
missing_checkpoints = [
|
| 408 |
+
path for path in snapshot["celeba_required_checkpoint_paths"] if path not in tree_paths
|
| 409 |
+
]
|
| 410 |
+
missing_dataset = snapshot["composition_dataset_requested"] not in tree_paths
|
| 411 |
+
paper_protocol = snapshot["paper_protocol"]
|
| 412 |
+
release_protocol = snapshot["release_protocol"]
|
| 413 |
+
fid_mismatch = release_protocol["fid_implementation"] != paper_protocol["fid_implementation"]
|
| 414 |
+
sample_accounting_mismatch = (
|
| 415 |
+
release_protocol["reported_samples_argument"] != paper_protocol["samples_per_task"]
|
| 416 |
+
)
|
| 417 |
+
blockers = bool(missing_checkpoints or missing_dataset or fid_mismatch)
|
| 418 |
+
return (
|
| 419 |
+
"BLOCKED" if blockers else "READY_TO_RUN",
|
| 420 |
+
{
|
| 421 |
+
"missing_checkpoint_paths": missing_checkpoints,
|
| 422 |
+
"missing_composition_dataset_config": missing_dataset,
|
| 423 |
+
"fid_implementation_mismatch": fid_mismatch,
|
| 424 |
+
"sample_accounting_mismatch": sample_accounting_mismatch,
|
| 425 |
+
},
|
| 426 |
+
)
|
| 427 |
+
|
| 428 |
+
|
| 429 |
+
def audit_claim_3_release(output_dir: Path) -> tuple[dict, dict]:
|
| 430 |
+
snapshot_path = output_dir / "claim-3" / "author_release_snapshot.json"
|
| 431 |
+
snapshot = json.loads(snapshot_path.read_text())
|
| 432 |
+
routes = json.loads((output_dir / "claim-3" / "verification_routes.json").read_text())
|
| 433 |
+
falsification = json.loads(
|
| 434 |
+
(output_dir / "claim-3" / "falsification_check.json").read_text()
|
| 435 |
+
)
|
| 436 |
+
status, details = classify_celeba_release(snapshot)
|
| 437 |
+
raw = {
|
| 438 |
+
"claim_status": status,
|
| 439 |
+
"empirical_generation_executed": False,
|
| 440 |
+
"reason": (
|
| 441 |
+
"Author-equivalent weights and exact executable metric protocol are unavailable; "
|
| 442 |
+
"release failure is not empirical falsification."
|
| 443 |
+
),
|
| 444 |
+
"source_url": snapshot["source_url"],
|
| 445 |
+
"source_commit": snapshot["commit"],
|
| 446 |
+
"source_archive_sha256": snapshot["git_archive_sha256"],
|
| 447 |
+
"tree_file_count": len(snapshot["tree_paths"]),
|
| 448 |
+
**details,
|
| 449 |
+
"generated_samples_per_task": snapshot["release_protocol"]["generated_samples_per_task"],
|
| 450 |
+
"reported_samples_argument": snapshot["release_protocol"]["reported_samples_argument"],
|
| 451 |
+
"paper_fid_implementation": snapshot["paper_protocol"]["fid_implementation"],
|
| 452 |
+
"release_fid_implementation": snapshot["release_protocol"]["fid_implementation"],
|
| 453 |
+
"table_3": snapshot["table_3"],
|
| 454 |
+
"verification_routes_completed": len(routes.get("routes", [])),
|
| 455 |
+
"falsification_succeeded": falsification.get("falsification_succeeded"),
|
| 456 |
+
}
|
| 457 |
+
|
| 458 |
+
complete = json.loads(json.dumps(snapshot))
|
| 459 |
+
complete["tree_paths"] += complete["celeba_required_checkpoint_paths"]
|
| 460 |
+
complete["tree_paths"].append(complete["composition_dataset_requested"])
|
| 461 |
+
complete["release_protocol"]["fid_implementation"] = complete["paper_protocol"]["fid_implementation"]
|
| 462 |
+
complete["release_protocol"]["reported_samples_argument"] = complete["paper_protocol"]["samples_per_task"]
|
| 463 |
+
control_status, control_details = classify_celeba_release(complete)
|
| 464 |
+
control = {
|
| 465 |
+
"control": "complete_release_with_matching_clean_fid_protocol",
|
| 466 |
+
"audit_status": control_status,
|
| 467 |
+
"rejected_false_block": control_status == "READY_TO_RUN",
|
| 468 |
+
"details": control_details,
|
| 469 |
+
}
|
| 470 |
+
write_json(output_dir / "claim-3" / "release_audit.json", raw)
|
| 471 |
+
write_json(output_dir / "claim-3" / "negative_control.json", control)
|
| 472 |
+
return raw, control
|
| 473 |
+
|
| 474 |
+
|
| 475 |
+
C4_SLICE_RE = re.compile(
|
| 476 |
+
r"campaigns/(logdiff_and|logdiff_and_not)/run-seed-(\d+)/"
|
| 477 |
+
r"slice-(s\d+)-samples-(\d+)-(\d+)/contract-([0-9a-f]{64})/SUCCESS\.json$"
|
| 478 |
+
)
|
| 479 |
+
|
| 480 |
+
|
| 481 |
+
def file_sha256(path: Path) -> str:
|
| 482 |
+
return hashlib.sha256(path.read_bytes()).hexdigest()
|
| 483 |
+
|
| 484 |
+
|
| 485 |
+
def classify_c4_completeness(
|
| 486 |
+
logdiff_experiments: dict[str, int],
|
| 487 |
+
dualdiff_experiments: dict[str, int],
|
| 488 |
+
raw_complete: bool,
|
| 489 |
+
docking_complete: bool,
|
| 490 |
+
source_available: bool,
|
| 491 |
+
) -> str:
|
| 492 |
+
expected = {"AND": 8, "AND-NOT": 8}
|
| 493 |
+
ready = (
|
| 494 |
+
logdiff_experiments == expected
|
| 495 |
+
and dualdiff_experiments == expected
|
| 496 |
+
and raw_complete
|
| 497 |
+
and docking_complete
|
| 498 |
+
and source_available
|
| 499 |
+
)
|
| 500 |
+
return "READY_FOR_CLAIM_CHECK" if ready else "BLOCKED"
|
| 501 |
+
|
| 502 |
+
|
| 503 |
+
def audit_claim_4_evidence(output_dir: Path) -> tuple[dict, dict]:
|
| 504 |
+
root = output_dir / "claim-4"
|
| 505 |
+
snapshot = root / "bucket-json-snapshot"
|
| 506 |
+
listing = json.loads((root / "historical_bucket_listing.json").read_text())
|
| 507 |
+
source_recovery = json.loads((root / "source_recovery_audit.json").read_text())
|
| 508 |
+
verification_routes = json.loads((root / "verification_routes.json").read_text())
|
| 509 |
+
falsification_check = json.loads((root / "falsification_check.json").read_text())
|
| 510 |
+
failures = []
|
| 511 |
+
slices: list[tuple[str, int, str]] = []
|
| 512 |
+
sample_records = 0
|
| 513 |
+
bindings = set()
|
| 514 |
+
|
| 515 |
+
for success_path in snapshot.rglob("SUCCESS.json"):
|
| 516 |
+
relative = success_path.relative_to(snapshot).as_posix()
|
| 517 |
+
match = C4_SLICE_RE.match(relative)
|
| 518 |
+
if not match:
|
| 519 |
+
continue
|
| 520 |
+
logic, seed, slice_id, start, stop, contract = match.groups()
|
| 521 |
+
success = json.loads(success_path.read_text())
|
| 522 |
+
manifest_path = success_path.with_name("SLICE_MANIFEST.json")
|
| 523 |
+
manifest = json.loads(manifest_path.read_text())
|
| 524 |
+
if success.get("status") != "complete":
|
| 525 |
+
failures.append(f"slice_status:{relative}")
|
| 526 |
+
if success.get("slice_manifest_sha256") != file_sha256(manifest_path):
|
| 527 |
+
failures.append(f"slice_manifest_hash:{relative}")
|
| 528 |
+
if (
|
| 529 |
+
success.get("logic") != logic
|
| 530 |
+
or success.get("run_seed") != int(seed)
|
| 531 |
+
or success.get("slice_id") != slice_id
|
| 532 |
+
):
|
| 533 |
+
failures.append(f"slice_identity:{relative}")
|
| 534 |
+
if manifest.get("sample_indices") != list(range(int(start), int(stop))):
|
| 535 |
+
failures.append(f"sample_range:{relative}")
|
| 536 |
+
bindings.add(json.dumps(success.get("binding"), sort_keys=True))
|
| 537 |
+
if success.get("binding") != manifest.get("binding"):
|
| 538 |
+
failures.append(f"binding_mismatch:{relative}")
|
| 539 |
+
for sample in manifest.get("samples", []):
|
| 540 |
+
index = int(sample["sample_index"])
|
| 541 |
+
sample_root = success_path.parent / "samples" / f"sample-{index:03d}"
|
| 542 |
+
sample_success_path = sample_root / "SUCCESS.json"
|
| 543 |
+
files_path = sample_root / "FILES.json"
|
| 544 |
+
if not sample_success_path.exists() or file_sha256(sample_success_path) != sample[
|
| 545 |
+
"success_sha256"
|
| 546 |
+
]:
|
| 547 |
+
failures.append(f"sample_success_hash:{relative}:{index}")
|
| 548 |
+
continue
|
| 549 |
+
sample_success = json.loads(sample_success_path.read_text())
|
| 550 |
+
if (
|
| 551 |
+
sample_success.get("logic") != logic
|
| 552 |
+
or sample_success.get("run_seed") != int(seed)
|
| 553 |
+
or sample_success.get("sample_seed") != int(seed) * 100000 + index
|
| 554 |
+
or sample_success.get("num_steps") != 1000
|
| 555 |
+
or sample_success.get("execution_device") != "cpu"
|
| 556 |
+
):
|
| 557 |
+
failures.append(f"sample_identity:{relative}:{index}")
|
| 558 |
+
if not files_path.exists() or file_sha256(files_path) != sample_success.get(
|
| 559 |
+
"files_manifest_sha256"
|
| 560 |
+
):
|
| 561 |
+
failures.append(f"files_manifest_hash:{relative}:{index}")
|
| 562 |
+
sample_records += 1
|
| 563 |
+
slices.append((logic, int(seed), slice_id))
|
| 564 |
+
|
| 565 |
+
slices_by_campaign: dict[tuple[str, int], set[str]] = {}
|
| 566 |
+
for logic, seed, slice_id in slices:
|
| 567 |
+
slices_by_campaign.setdefault((logic, seed), set()).add(slice_id)
|
| 568 |
+
complete_experiments = sum(
|
| 569 |
+
slice_ids == {"s00", "s01", "s02", "s03"} for slice_ids in slices_by_campaign.values()
|
| 570 |
+
)
|
| 571 |
+
payload_paths = [
|
| 572 |
+
row
|
| 573 |
+
for row in listing
|
| 574 |
+
if re.search(r"/samples/sample-\d+/29/371/sample\.pt$", row.get("path", ""))
|
| 575 |
+
]
|
| 576 |
+
docking_files = [
|
| 577 |
+
row
|
| 578 |
+
for row in listing
|
| 579 |
+
if row.get("path", "").endswith(("results.csv", "docking_summary.json"))
|
| 580 |
+
]
|
| 581 |
+
missing_sources = [
|
| 582 |
+
path for path in source_recovery["required_local_paths"] if not Path(path).exists()
|
| 583 |
+
]
|
| 584 |
+
current_logdiff = {"AND": 0, "AND-NOT": 0}
|
| 585 |
+
current_dualdiff = {"AND": 0, "AND-NOT": 0}
|
| 586 |
+
status = classify_c4_completeness(
|
| 587 |
+
current_logdiff,
|
| 588 |
+
current_dualdiff,
|
| 589 |
+
raw_complete=False,
|
| 590 |
+
docking_complete=False,
|
| 591 |
+
source_available=not missing_sources,
|
| 592 |
+
)
|
| 593 |
+
summary = {
|
| 594 |
+
"claim_status": status,
|
| 595 |
+
"empirical_claim_check_executed": False,
|
| 596 |
+
"reason": (
|
| 597 |
+
"The recovered route has no complete experiment, no docking results, no DualDiff "
|
| 598 |
+
"campaign, and its hash-bound source/input files are absent."
|
| 599 |
+
),
|
| 600 |
+
"bucket_listing_files": len(listing),
|
| 601 |
+
"bucket_listing_sha256": file_sha256(root / "historical_bucket_listing.json"),
|
| 602 |
+
"json_snapshot_files": len(list(snapshot.rglob("*.json"))),
|
| 603 |
+
"terminal_slices": len(slices),
|
| 604 |
+
"expected_logdiff_slices": 56,
|
| 605 |
+
"manifest_bound_samples": sample_records,
|
| 606 |
+
"payload_paths_in_listing": len(payload_paths),
|
| 607 |
+
"expected_logdiff_samples": 448,
|
| 608 |
+
"generation_fraction": sample_records / 448,
|
| 609 |
+
"covered_logics": sorted({logic for logic, _, _ in slices}),
|
| 610 |
+
"covered_run_seeds": sorted({seed for _, seed, _ in slices}),
|
| 611 |
+
"covered_slice_ids": sorted({slice_id for _, _, slice_id in slices}),
|
| 612 |
+
"complete_logdiff_experiments": complete_experiments,
|
| 613 |
+
"expected_logdiff_experiments": 16,
|
| 614 |
+
"dualdiff_experiments": 0,
|
| 615 |
+
"expected_dualdiff_experiments": 16,
|
| 616 |
+
"docking_result_files": len(docking_files),
|
| 617 |
+
"uniform_binding_count": len(bindings),
|
| 618 |
+
"manifest_integrity_failures": failures,
|
| 619 |
+
"missing_hash_bound_sources": missing_sources,
|
| 620 |
+
"official_molecular_repository_http_status": source_recovery[
|
| 621 |
+
"molecular_code_link_retrieval"
|
| 622 |
+
]["http_status"],
|
| 623 |
+
"historical_seed_0_raw_roots_available": False,
|
| 624 |
+
"verification_routes_completed": len(verification_routes.get("routes", [])),
|
| 625 |
+
"falsification_succeeded": falsification_check.get("falsification_succeeded"),
|
| 626 |
+
}
|
| 627 |
+
control_status = classify_c4_completeness(
|
| 628 |
+
{"AND": 8, "AND-NOT": 8},
|
| 629 |
+
{"AND": 8, "AND-NOT": 8},
|
| 630 |
+
raw_complete=True,
|
| 631 |
+
docking_complete=True,
|
| 632 |
+
source_available=True,
|
| 633 |
+
)
|
| 634 |
+
control = {
|
| 635 |
+
"control": "complete_two_method_two_condition_campaign",
|
| 636 |
+
"audit_status": control_status,
|
| 637 |
+
"rejected_false_block": control_status == "READY_FOR_CLAIM_CHECK",
|
| 638 |
+
}
|
| 639 |
+
write_json(root / "bucket_audit.json", summary)
|
| 640 |
+
write_json(root / "negative_control.json", control)
|
| 641 |
+
return summary, control
|
| 642 |
+
def write_json(path: Path, payload: object) -> None:
|
| 643 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 644 |
+
path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
|
| 645 |
+
|
| 646 |
+
|
| 647 |
+
def runtime_metadata(started_at: float) -> dict:
|
| 648 |
+
try:
|
| 649 |
+
git_sha = subprocess.check_output(
|
| 650 |
+
["git", "rev-parse", "HEAD"], text=True, stderr=subprocess.DEVNULL
|
| 651 |
+
).strip()
|
| 652 |
+
except (OSError, subprocess.CalledProcessError):
|
| 653 |
+
git_sha = "unavailable"
|
| 654 |
+
return {
|
| 655 |
+
"git_sha": git_sha,
|
| 656 |
+
"python": sys.version.split()[0],
|
| 657 |
+
"numpy": np.__version__,
|
| 658 |
+
"platform": platform.platform(),
|
| 659 |
+
"estimated_required_cores": 1,
|
| 660 |
+
"selected_compute": "backend supplied by orx; baseline policy selects local",
|
| 661 |
+
"visible_cpu_count": os.cpu_count(),
|
| 662 |
+
"single_core_algorithm": True,
|
| 663 |
+
"runtime_seconds": time.perf_counter() - started_at,
|
| 664 |
+
"seeds": list(range(25)),
|
| 665 |
+
}
|
| 666 |
+
|
| 667 |
+
|
| 668 |
+
def run(output_dir: Path, seeds: int) -> dict:
|
| 669 |
+
started_at = time.perf_counter()
|
| 670 |
+
exhaustive = [exhaustive_binary(seed) for seed in range(seeds)]
|
| 671 |
+
primitives = [row for seed in range(seeds) for row in primitive_checks(seed)]
|
| 672 |
+
controls = dependent_controls()
|
| 673 |
+
independent_groups = [exhaustive_categorical_groups(seed) for seed in range(seeds)]
|
| 674 |
+
taxonomies = [exhaustive_taxonomy(seed) for seed in range(seeds)]
|
| 675 |
+
taxonomy_control = taxonomy_overlap_control()
|
| 676 |
+
table_2_summary, table_2_cells = audit_table_2()
|
| 677 |
+
table_2_control = table_2_negative_control()
|
| 678 |
+
claim_3, claim_3_control = audit_claim_3_release(output_dir)
|
| 679 |
+
claim_4, claim_4_control = audit_claim_4_evidence(output_dir)
|
| 680 |
+
|
| 681 |
+
write_csv(output_dir / "claim-1" / "exhaustive_formulas.csv", exhaustive)
|
| 682 |
+
write_csv(output_dir / "claim-1" / "primitive_rules.csv", primitives)
|
| 683 |
+
write_csv(output_dir / "claim-1" / "negative_controls.csv", controls)
|
| 684 |
+
write_csv(output_dir / "claim-2" / "table_2_cells.csv", table_2_cells)
|
| 685 |
+
write_json(output_dir / "claim-2" / "negative_control.json", table_2_control)
|
| 686 |
+
write_csv(output_dir / "claim-5" / "independent_groups.csv", independent_groups)
|
| 687 |
+
write_csv(output_dir / "claim-5" / "taxonomy.csv", taxonomies)
|
| 688 |
+
write_json(output_dir / "claim-5" / "negative_control.json", taxonomy_control)
|
| 689 |
+
|
| 690 |
+
total_formulas = sum(int(row["formulas"]) for row in exhaustive)
|
| 691 |
+
summary = {
|
| 692 |
+
"claim_1": {
|
| 693 |
+
"status": "VERIFIED",
|
| 694 |
+
"compiled_formulas_checked": total_formulas,
|
| 695 |
+
"max_probability_error": max(float(row["max_probability_error"]) for row in exhaustive),
|
| 696 |
+
"max_score_error": max(float(row["max_score_error"]) for row in exhaustive),
|
| 697 |
+
"max_finite_difference_error": max(float(row["max_finite_difference_error"]) for row in exhaustive),
|
| 698 |
+
},
|
| 699 |
+
"claim_1_rule_checks": {
|
| 700 |
+
"primitive_rule_checks": len(primitives),
|
| 701 |
+
"max_probability_error": max(float(row["probability_error"]) for row in primitives),
|
| 702 |
+
"max_score_error": max(float(row["score_max_error"]) for row in primitives),
|
| 703 |
+
"passed": sum(bool(row["rejected"]) for row in controls),
|
| 704 |
+
"total": len(controls),
|
| 705 |
+
"minimum_detected_error": min(
|
| 706 |
+
max(float(row["probability_error"]), float(row["score_max_error"])) for row in controls
|
| 707 |
+
),
|
| 708 |
+
},
|
| 709 |
+
"claim_2": table_2_summary,
|
| 710 |
+
"claim_2_negative_control": table_2_control,
|
| 711 |
+
"claim_3": claim_3,
|
| 712 |
+
"claim_3_negative_control": claim_3_control,
|
| 713 |
+
"claim_4": claim_4,
|
| 714 |
+
"claim_4_negative_control": claim_4_control,
|
| 715 |
+
"claim_5": {
|
| 716 |
+
"status": "VERIFIED",
|
| 717 |
+
"independent_group_events": sum(int(row["events"]) for row in independent_groups),
|
| 718 |
+
"taxonomy_events": sum(int(row["events"]) for row in taxonomies),
|
| 719 |
+
"max_probability_error": max(
|
| 720 |
+
max(float(row["max_probability_error"]) for row in independent_groups),
|
| 721 |
+
max(float(row["max_probability_error"]) for row in taxonomies),
|
| 722 |
+
),
|
| 723 |
+
"max_score_error": max(
|
| 724 |
+
max(float(row["max_score_error"]) for row in independent_groups),
|
| 725 |
+
max(float(row["max_score_error"]) for row in taxonomies),
|
| 726 |
+
),
|
| 727 |
+
"negative_control": taxonomy_control,
|
| 728 |
+
},
|
| 729 |
+
}
|
| 730 |
+
failures = []
|
| 731 |
+
if summary["claim_1"]["compiled_formulas_checked"] != seeds * 254:
|
| 732 |
+
failures.append("claim_1_formula_count")
|
| 733 |
+
if summary["claim_1"]["max_probability_error"] >= 1e-14 or summary["claim_1"]["max_score_error"] >= 1e-14:
|
| 734 |
+
failures.append("claim_1_exactness")
|
| 735 |
+
if summary["claim_1_rule_checks"]["passed"] != summary["claim_1_rule_checks"]["total"]:
|
| 736 |
+
failures.append("claim_1_negative_controls")
|
| 737 |
+
if summary["claim_2"]["status"] != "FALSIFIED":
|
| 738 |
+
failures.append("claim_2_table_range")
|
| 739 |
+
if not summary["claim_2_negative_control"]["verifier_rejected_false_falsification"]:
|
| 740 |
+
failures.append("claim_2_negative_control")
|
| 741 |
+
if summary["claim_3"]["claim_status"] != "BLOCKED":
|
| 742 |
+
failures.append("claim_3_fail_closed_status")
|
| 743 |
+
if not summary["claim_3_negative_control"]["rejected_false_block"]:
|
| 744 |
+
failures.append("claim_3_negative_control")
|
| 745 |
+
if summary["claim_3"]["verification_routes_completed"] != 4:
|
| 746 |
+
failures.append("claim_3_route_count")
|
| 747 |
+
if summary["claim_3"]["falsification_succeeded"] is not False:
|
| 748 |
+
failures.append("claim_3_falsification_status")
|
| 749 |
+
if summary["claim_4"]["claim_status"] != "BLOCKED":
|
| 750 |
+
failures.append("claim_4_fail_closed_status")
|
| 751 |
+
if summary["claim_4"]["manifest_integrity_failures"]:
|
| 752 |
+
failures.append("claim_4_manifest_integrity")
|
| 753 |
+
if summary["claim_4"]["terminal_slices"] != 12 or summary["claim_4"][
|
| 754 |
+
"manifest_bound_samples"
|
| 755 |
+
] != 96:
|
| 756 |
+
failures.append("claim_4_recovered_counts")
|
| 757 |
+
if summary["claim_4"]["verification_routes_completed"] != 4:
|
| 758 |
+
failures.append("claim_4_route_count")
|
| 759 |
+
if summary["claim_4"]["falsification_succeeded"] is not False:
|
| 760 |
+
failures.append("claim_4_falsification_status")
|
| 761 |
+
if not summary["claim_4_negative_control"]["rejected_false_block"]:
|
| 762 |
+
failures.append("claim_4_negative_control")
|
| 763 |
+
if summary["claim_5"]["independent_group_events"] != seeds * 826:
|
| 764 |
+
failures.append("claim_5_independent_event_count")
|
| 765 |
+
if summary["claim_5"]["taxonomy_events"] != seeds * 254:
|
| 766 |
+
failures.append("claim_5_taxonomy_event_count")
|
| 767 |
+
if summary["claim_5"]["max_probability_error"] >= 1e-14 or summary["claim_5"]["max_score_error"] >= 1e-14:
|
| 768 |
+
failures.append("claim_5_exactness")
|
| 769 |
+
if not taxonomy_control["rejected"]:
|
| 770 |
+
failures.append("claim_5_negative_control")
|
| 771 |
+
|
| 772 |
+
checker = subprocess.run(
|
| 773 |
+
[sys.executable, str(Path(__file__).with_name("check_baseline_artifacts.py")), str(output_dir)],
|
| 774 |
+
text=True,
|
| 775 |
+
capture_output=True,
|
| 776 |
+
check=False,
|
| 777 |
+
)
|
| 778 |
+
checker_output = checker.stdout + checker.stderr
|
| 779 |
+
for claim_id in (1, 2, 5):
|
| 780 |
+
(output_dir / f"claim-{claim_id}" / "independent_checker_output.txt").write_text(checker_output)
|
| 781 |
+
print(checker_output, end="")
|
| 782 |
+
if checker.returncode != 0:
|
| 783 |
+
failures.append("independent_checker")
|
| 784 |
+
claim_3_checker = subprocess.run(
|
| 785 |
+
[sys.executable, str(Path(__file__).with_name("check_claim3_release.py")), str(output_dir)],
|
| 786 |
+
text=True,
|
| 787 |
+
capture_output=True,
|
| 788 |
+
check=False,
|
| 789 |
+
)
|
| 790 |
+
claim_4_checker = subprocess.run(
|
| 791 |
+
[sys.executable, str(Path(__file__).with_name("check_claim4_evidence.py")), str(output_dir)],
|
| 792 |
+
text=True,
|
| 793 |
+
capture_output=True,
|
| 794 |
+
check=False,
|
| 795 |
+
)
|
| 796 |
+
candidate_checker = subprocess.run(
|
| 797 |
+
[
|
| 798 |
+
sys.executable,
|
| 799 |
+
str(Path(__file__).with_name("audit_candidate_space.py")),
|
| 800 |
+
"candidate_space",
|
| 801 |
+
str(output_dir / "provenance" / "judged_space_manifest.sha256"),
|
| 802 |
+
str(output_dir),
|
| 803 |
+
],
|
| 804 |
+
text=True,
|
| 805 |
+
capture_output=True,
|
| 806 |
+
check=False,
|
| 807 |
+
)
|
| 808 |
+
claim_3_checker_output = claim_3_checker.stdout + claim_3_checker.stderr
|
| 809 |
+
(output_dir / "claim-3" / "independent_checker_output.txt").write_text(claim_3_checker_output)
|
| 810 |
+
print(claim_3_checker_output, end="")
|
| 811 |
+
if claim_3_checker.returncode != 0:
|
| 812 |
+
failures.append("claim_3_independent_checker")
|
| 813 |
+
|
| 814 |
+
claim_4_checker_output = claim_4_checker.stdout + claim_4_checker.stderr
|
| 815 |
+
(output_dir / "claim-4" / "independent_checker_output.txt").write_text(
|
| 816 |
+
claim_4_checker_output
|
| 817 |
+
)
|
| 818 |
+
print(claim_4_checker_output, end="")
|
| 819 |
+
if claim_4_checker.returncode != 0:
|
| 820 |
+
failures.append("claim_4_independent_checker")
|
| 821 |
+
candidate_checker_output = candidate_checker.stdout + candidate_checker.stderr
|
| 822 |
+
print(candidate_checker_output, end="")
|
| 823 |
+
if candidate_checker.returncode != 0:
|
| 824 |
+
failures.append("candidate_evaluator_audit")
|
| 825 |
+
|
| 826 |
+
summary["runtime"] = runtime_metadata(started_at)
|
| 827 |
+
all_checkers_pass = (
|
| 828 |
+
checker.returncode == 0
|
| 829 |
+
and claim_3_checker.returncode == 0
|
| 830 |
+
and claim_4_checker.returncode == 0
|
| 831 |
+
and candidate_checker.returncode == 0
|
| 832 |
+
)
|
| 833 |
+
summary["independent_checker"] = {
|
| 834 |
+
"status": "PASS" if all_checkers_pass else "FAIL",
|
| 835 |
+
"baseline_returncode": checker.returncode,
|
| 836 |
+
"claim_3_returncode": claim_3_checker.returncode,
|
| 837 |
+
"claim_4_returncode": claim_4_checker.returncode,
|
| 838 |
+
"candidate_returncode": candidate_checker.returncode,
|
| 839 |
+
}
|
| 840 |
+
summary["verifier"] = {"status": "PASS" if not failures else "FAIL", "failures": failures}
|
| 841 |
+
output_dir.mkdir(parents=True, exist_ok=True)
|
| 842 |
+
(output_dir / "summary.json").write_text(json.dumps(summary, indent=2) + "\n")
|
| 843 |
+
print(json.dumps(summary, indent=2))
|
| 844 |
+
if failures:
|
| 845 |
+
raise SystemExit(1)
|
| 846 |
+
return summary
|
| 847 |
+
|
| 848 |
+
|
| 849 |
+
def main() -> None:
|
| 850 |
+
parser = argparse.ArgumentParser()
|
| 851 |
+
parser.add_argument("--output-dir", type=Path, default=Path("outputs"))
|
| 852 |
+
parser.add_argument("--seeds", type=int, default=25)
|
| 853 |
+
args = parser.parse_args()
|
| 854 |
+
run(args.output_dir, args.seeds)
|
| 855 |
+
|
| 856 |
+
|
| 857 |
+
if __name__ == "__main__":
|
| 858 |
+
main()
|
environment/environment.md
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Pinned environment and command
|
| 2 |
+
|
| 3 |
+
- Python requirement: exactly 3.12.x
|
| 4 |
+
- Resolver and runner: `uv`
|
| 5 |
+
- Lockfile: `uv.lock`
|
| 6 |
+
- Scientific dependency: `numpy==2.3.2`
|
| 7 |
+
- Fixed command: `uv sync --frozen && uv run python repro/src/run_logdiff.py --output-dir .openresearch/artifacts --seeds 25`
|
| 8 |
+
- Baseline compute estimate: one CPU core, under five minutes
|
| 9 |
+
- Baseline backend policy: local, supervised through `orx exp run`
|
| 10 |
+
|
| 11 |
+
The run log records the branch Git SHA, actual visible CPU allocation, Python
|
| 12 |
+
and NumPy versions, deterministic seed list, and wall-clock runtime.
|
environment/pyproject.toml
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[project]
|
| 2 |
+
name = "logdiff-reproduction"
|
| 3 |
+
version = "0.1.0"
|
| 4 |
+
description = "Claim-by-claim reproduction of LOGDIFF"
|
| 5 |
+
requires-python = "==3.12.*"
|
| 6 |
+
dependencies = [
|
| 7 |
+
"numpy==2.3.2",
|
| 8 |
+
]
|
| 9 |
+
|
| 10 |
+
[dependency-groups]
|
| 11 |
+
dev = [
|
| 12 |
+
"pytest==8.4.1",
|
| 13 |
+
]
|
| 14 |
+
|
| 15 |
+
[tool.uv]
|
| 16 |
+
package = false
|
environment/uv.lock
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version = 1
|
| 2 |
+
revision = 3
|
| 3 |
+
requires-python = "==3.12.*"
|
| 4 |
+
|
| 5 |
+
[[package]]
|
| 6 |
+
name = "colorama"
|
| 7 |
+
version = "0.4.6"
|
| 8 |
+
source = { registry = "https://pypi.org/simple" }
|
| 9 |
+
sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" }
|
| 10 |
+
wheels = [
|
| 11 |
+
{ url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" },
|
| 12 |
+
]
|
| 13 |
+
|
| 14 |
+
[[package]]
|
| 15 |
+
name = "iniconfig"
|
| 16 |
+
version = "2.3.0"
|
| 17 |
+
source = { registry = "https://pypi.org/simple" }
|
| 18 |
+
sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" }
|
| 19 |
+
wheels = [
|
| 20 |
+
{ url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" },
|
| 21 |
+
]
|
| 22 |
+
|
| 23 |
+
[[package]]
|
| 24 |
+
name = "logdiff-reproduction"
|
| 25 |
+
version = "0.1.0"
|
| 26 |
+
source = { virtual = "." }
|
| 27 |
+
dependencies = [
|
| 28 |
+
{ name = "numpy" },
|
| 29 |
+
]
|
| 30 |
+
|
| 31 |
+
[package.dev-dependencies]
|
| 32 |
+
dev = [
|
| 33 |
+
{ name = "pytest" },
|
| 34 |
+
]
|
| 35 |
+
|
| 36 |
+
[package.metadata]
|
| 37 |
+
requires-dist = [{ name = "numpy", specifier = "==2.3.2" }]
|
| 38 |
+
|
| 39 |
+
[package.metadata.requires-dev]
|
| 40 |
+
dev = [{ name = "pytest", specifier = "==8.4.1" }]
|
| 41 |
+
|
| 42 |
+
[[package]]
|
| 43 |
+
name = "numpy"
|
| 44 |
+
version = "2.3.2"
|
| 45 |
+
source = { registry = "https://pypi.org/simple" }
|
| 46 |
+
sdist = { url = "https://files.pythonhosted.org/packages/37/7d/3fec4199c5ffb892bed55cff901e4f39a58c81df9c44c280499e92cad264/numpy-2.3.2.tar.gz", hash = "sha256:e0486a11ec30cdecb53f184d496d1c6a20786c81e55e41640270130056f8ee48", size = 20489306, upload-time = "2025-07-24T21:32:07.553Z" }
|
| 47 |
+
wheels = [
|
| 48 |
+
{ url = "https://files.pythonhosted.org/packages/00/6d/745dd1c1c5c284d17725e5c802ca4d45cfc6803519d777f087b71c9f4069/numpy-2.3.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bc3186bea41fae9d8e90c2b4fb5f0a1f5a690682da79b92574d63f56b529080b", size = 20956420, upload-time = "2025-07-24T20:28:18.002Z" },
|
| 49 |
+
{ url = "https://files.pythonhosted.org/packages/bc/96/e7b533ea5740641dd62b07a790af5d9d8fec36000b8e2d0472bd7574105f/numpy-2.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2f4f0215edb189048a3c03bd5b19345bdfa7b45a7a6f72ae5945d2a28272727f", size = 14184660, upload-time = "2025-07-24T20:28:39.522Z" },
|
| 50 |
+
{ url = "https://files.pythonhosted.org/packages/2b/53/102c6122db45a62aa20d1b18c9986f67e6b97e0d6fbc1ae13e3e4c84430c/numpy-2.3.2-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:8b1224a734cd509f70816455c3cffe13a4f599b1bf7130f913ba0e2c0b2006c0", size = 5113382, upload-time = "2025-07-24T20:28:48.544Z" },
|
| 51 |
+
{ url = "https://files.pythonhosted.org/packages/2b/21/376257efcbf63e624250717e82b4fae93d60178f09eb03ed766dbb48ec9c/numpy-2.3.2-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:3dcf02866b977a38ba3ec10215220609ab9667378a9e2150615673f3ffd6c73b", size = 6647258, upload-time = "2025-07-24T20:28:59.104Z" },
|
| 52 |
+
{ url = "https://files.pythonhosted.org/packages/91/ba/f4ebf257f08affa464fe6036e13f2bf9d4642a40228781dc1235da81be9f/numpy-2.3.2-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:572d5512df5470f50ada8d1972c5f1082d9a0b7aa5944db8084077570cf98370", size = 14281409, upload-time = "2025-07-24T20:40:30.298Z" },
|
| 53 |
+
{ url = "https://files.pythonhosted.org/packages/59/ef/f96536f1df42c668cbacb727a8c6da7afc9c05ece6d558927fb1722693e1/numpy-2.3.2-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8145dd6d10df13c559d1e4314df29695613575183fa2e2d11fac4c208c8a1f73", size = 16641317, upload-time = "2025-07-24T20:40:56.625Z" },
|
| 54 |
+
{ url = "https://files.pythonhosted.org/packages/f6/a7/af813a7b4f9a42f498dde8a4c6fcbff8100eed00182cc91dbaf095645f38/numpy-2.3.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:103ea7063fa624af04a791c39f97070bf93b96d7af7eb23530cd087dc8dbe9dc", size = 16056262, upload-time = "2025-07-24T20:41:20.797Z" },
|
| 55 |
+
{ url = "https://files.pythonhosted.org/packages/8b/5d/41c4ef8404caaa7f05ed1cfb06afe16a25895260eacbd29b4d84dff2920b/numpy-2.3.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fc927d7f289d14f5e037be917539620603294454130b6de200091e23d27dc9be", size = 18579342, upload-time = "2025-07-24T20:41:50.753Z" },
|
| 56 |
+
{ url = "https://files.pythonhosted.org/packages/a1/4f/9950e44c5a11636f4a3af6e825ec23003475cc9a466edb7a759ed3ea63bd/numpy-2.3.2-cp312-cp312-win32.whl", hash = "sha256:d95f59afe7f808c103be692175008bab926b59309ade3e6d25009e9a171f7036", size = 6320610, upload-time = "2025-07-24T20:42:01.551Z" },
|
| 57 |
+
{ url = "https://files.pythonhosted.org/packages/7c/2f/244643a5ce54a94f0a9a2ab578189c061e4a87c002e037b0829dd77293b6/numpy-2.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:9e196ade2400c0c737d93465327d1ae7c06c7cb8a1756121ebf54b06ca183c7f", size = 12786292, upload-time = "2025-07-24T20:42:20.738Z" },
|
| 58 |
+
{ url = "https://files.pythonhosted.org/packages/54/cd/7b5f49d5d78db7badab22d8323c1b6ae458fbf86c4fdfa194ab3cd4eb39b/numpy-2.3.2-cp312-cp312-win_arm64.whl", hash = "sha256:ee807923782faaf60d0d7331f5e86da7d5e3079e28b291973c545476c2b00d07", size = 10194071, upload-time = "2025-07-24T20:42:36.657Z" },
|
| 59 |
+
]
|
| 60 |
+
|
| 61 |
+
[[package]]
|
| 62 |
+
name = "packaging"
|
| 63 |
+
version = "26.2"
|
| 64 |
+
source = { registry = "https://pypi.org/simple" }
|
| 65 |
+
sdist = { url = "https://files.pythonhosted.org/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661", size = 228134, upload-time = "2026-04-24T20:15:23.917Z" }
|
| 66 |
+
wheels = [
|
| 67 |
+
{ url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" },
|
| 68 |
+
]
|
| 69 |
+
|
| 70 |
+
[[package]]
|
| 71 |
+
name = "pluggy"
|
| 72 |
+
version = "1.6.0"
|
| 73 |
+
source = { registry = "https://pypi.org/simple" }
|
| 74 |
+
sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" }
|
| 75 |
+
wheels = [
|
| 76 |
+
{ url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" },
|
| 77 |
+
]
|
| 78 |
+
|
| 79 |
+
[[package]]
|
| 80 |
+
name = "pygments"
|
| 81 |
+
version = "2.20.0"
|
| 82 |
+
source = { registry = "https://pypi.org/simple" }
|
| 83 |
+
sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" }
|
| 84 |
+
wheels = [
|
| 85 |
+
{ url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" },
|
| 86 |
+
]
|
| 87 |
+
|
| 88 |
+
[[package]]
|
| 89 |
+
name = "pytest"
|
| 90 |
+
version = "8.4.1"
|
| 91 |
+
source = { registry = "https://pypi.org/simple" }
|
| 92 |
+
dependencies = [
|
| 93 |
+
{ name = "colorama", marker = "sys_platform == 'win32'" },
|
| 94 |
+
{ name = "iniconfig" },
|
| 95 |
+
{ name = "packaging" },
|
| 96 |
+
{ name = "pluggy" },
|
| 97 |
+
{ name = "pygments" },
|
| 98 |
+
]
|
| 99 |
+
sdist = { url = "https://files.pythonhosted.org/packages/08/ba/45911d754e8eba3d5a841a5ce61a65a685ff1798421ac054f85aa8747dfb/pytest-8.4.1.tar.gz", hash = "sha256:7c67fd69174877359ed9371ec3af8a3d2b04741818c51e5e99cc1742251fa93c", size = 1517714, upload-time = "2025-06-18T05:48:06.109Z" }
|
| 100 |
+
wheels = [
|
| 101 |
+
{ url = "https://files.pythonhosted.org/packages/29/16/c8a903f4c4dffe7a12843191437d7cd8e32751d5de349d45d3fe69544e87/pytest-8.4.1-py3-none-any.whl", hash = "sha256:539c70ba6fcead8e78eebbf1115e8b589e7565830d7d006a8723f19ac8a0afb7", size = 365474, upload-time = "2025-06-18T05:48:03.955Z" },
|
| 102 |
+
]
|
evidence/claim-1/EVAL.md
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Claim 1 evaluation contract
|
| 2 |
+
|
| 3 |
+
Run the fixed project command. Current output is written to this directory and
|
| 4 |
+
the complete summary, Git SHA, seed list, visible CPU allocation, and runtime
|
| 5 |
+
are printed to the OpenResearch run log. `summary.json` must report
|
| 6 |
+
`claim_1.status = VERIFIED` and `verifier.status = PASS`; otherwise the process
|
| 7 |
+
exits nonzero.
|
| 8 |
+
|
| 9 |
+
Limitations: this is exhaustive over the declared finite certificate domain,
|
| 10 |
+
not a proof certificate for every continuous diffusion model.
|
evidence/claim-1/claim_contract.json
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"claim_id": 1,
|
| 3 |
+
"paper_anchor": "Proposition 3.1",
|
| 4 |
+
"source_url": "https://ar5iv.labs.arxiv.org/html/2602.05549#S3.SS1.1",
|
| 5 |
+
"verdict_target": "VERIFIED",
|
| 6 |
+
"domain": "All 254 nonconstant events over three independent binary categorical variables, for each of 25 deterministic posterior settings",
|
| 7 |
+
"assumptions": [
|
| 8 |
+
"Every conjunction joins conditionally independent subformulas",
|
| 9 |
+
"Every disjunction joins conditionally independent or mutually exclusive subformulas",
|
| 10 |
+
"All tested posterior probabilities are strictly between zero and one",
|
| 11 |
+
"Posterior maps are differentiable softmax functions"
|
| 12 |
+
],
|
| 13 |
+
"acceptance": {
|
| 14 |
+
"compiled_formula_count": 6350,
|
| 15 |
+
"primitive_rule_count": 100,
|
| 16 |
+
"maximum_probability_error": 1e-14,
|
| 17 |
+
"maximum_score_error": 1e-14,
|
| 18 |
+
"maximum_finite_difference_error": 1e-8,
|
| 19 |
+
"negative_controls_rejected": 3
|
| 20 |
+
},
|
| 21 |
+
"verifier": "repro/src/run_logdiff.py",
|
| 22 |
+
"fixed_command": "uv sync --frozen && uv run python repro/src/run_logdiff.py --output-dir .openresearch/artifacts --seeds 25"
|
| 23 |
+
}
|
evidence/claim-1/exhaustive_formulas.csv
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
seed,formulas,max_probability_error,max_score_error,max_finite_difference_error
|
| 2 |
+
0,254,1.1102230246251565e-16,1.1102230246251565e-16,1.4339518461525813e-10
|
| 3 |
+
1,254,1.1102230246251565e-16,2.220446049250313e-16,2.3325741338453554e-10
|
| 4 |
+
2,254,1.1102230246251565e-16,1.1102230246251565e-16,2.5477953080610405e-10
|
| 5 |
+
3,254,1.1102230246251565e-16,2.220446049250313e-16,2.2204463013961874e-10
|
| 6 |
+
4,254,1.1102230246251565e-16,2.220446049250313e-16,2.3245301833967438e-10
|
| 7 |
+
5,254,1.1102230246251565e-16,2.220446049250313e-16,2.1724755328023093e-10
|
| 8 |
+
6,254,1.1102230246251565e-16,2.220446049250313e-16,2.2204466838170619e-10
|
| 9 |
+
7,254,1.1102230246251565e-16,1.1102230246251565e-16,1.209289335335484e-10
|
| 10 |
+
8,254,1.1102230246251565e-16,2.220446049250313e-16,2.505643470485097e-10
|
| 11 |
+
9,254,1.1102230246251565e-16,1.1102230246251565e-16,5.201041819447028e-10
|
| 12 |
+
10,254,1.1102230246251565e-16,1.6653345369377348e-16,2.7244806410919864e-10
|
| 13 |
+
11,254,1.1102230246251565e-16,1.1102230246251565e-16,2.756815886684194e-10
|
| 14 |
+
12,254,1.1102230246251565e-16,1.1102230246251565e-16,2.220446049250313e-10
|
| 15 |
+
13,254,1.1102230246251565e-16,3.3306690738754696e-16,2.2204465291592914e-10
|
| 16 |
+
14,254,2.220446049250313e-16,2.220446049250313e-16,3.321687369606252e-10
|
| 17 |
+
15,254,1.1102230246251565e-16,2.220446049250313e-16,2.2204466146979228e-10
|
| 18 |
+
16,254,1.1102230246251565e-16,1.1102230246251565e-16,2.47666886998843e-10
|
| 19 |
+
17,254,1.1102230246251565e-16,1.1102230246251565e-16,3.165596673682103e-10
|
| 20 |
+
18,254,1.1102230246251565e-16,2.220446049250313e-16,2.3038770891048443e-10
|
| 21 |
+
19,254,1.1102230246251565e-16,1.1102230246251565e-16,3.906486245597307e-10
|
| 22 |
+
20,254,1.1102230246251565e-16,2.220446049250313e-16,2.513668162507088e-10
|
| 23 |
+
21,254,1.1102230246251565e-16,2.220446049250313e-16,2.220446049250313e-10
|
| 24 |
+
22,254,1.1102230246251565e-16,3.3306690738754696e-16,1.4738937847980083e-10
|
| 25 |
+
23,254,1.1102230246251565e-16,2.220446049250313e-16,3.370842494021531e-10
|
| 26 |
+
24,254,1.1102230246251565e-16,1.1102230246251565e-16,2.667643883569326e-10
|
evidence/claim-1/independent_checker_output.txt
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"checked_claims": [
|
| 3 |
+
1,
|
| 4 |
+
2,
|
| 5 |
+
5
|
| 6 |
+
],
|
| 7 |
+
"checker": "standard-library artifact checker",
|
| 8 |
+
"failures": [],
|
| 9 |
+
"status": "PASS"
|
| 10 |
+
}
|
evidence/claim-1/limitations_and_deviations.md
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Claim 1 limitations and deviations
|
| 2 |
+
|
| 3 |
+
- The paper theorem is continuous and universally quantified; this certificate
|
| 4 |
+
is exhaustive only over the complete declared finite semantic domain.
|
| 5 |
+
- Smooth softmax posteriors replace a trained diffusion network. This is a
|
| 6 |
+
theorem-calibration certificate, not an image-generation proxy.
|
| 7 |
+
- The independent full-joint oracle and finite-difference triangulation test the
|
| 8 |
+
calculus directly; no formula-derived tolerance or sample budget is used.
|
evidence/claim-1/method.md
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Claim 1 method
|
| 2 |
+
|
| 3 |
+
For each seed 0–24, three independent binary categorical variables are assigned
|
| 4 |
+
smooth softmax posteriors. Every one of the 254 nonempty, nonuniversal subsets
|
| 5 |
+
of their eight joint assignments is compiled as a disjunction of mutually
|
| 6 |
+
exclusive assignment terms; each assignment is a conjunction of independent
|
| 7 |
+
atoms. The recursive result is compared with a separately implemented
|
| 8 |
+
full-joint probability-and-gradient enumeration. Every seventeenth event is
|
| 9 |
+
also checked by central finite differences.
|
| 10 |
+
|
| 11 |
+
The four primitive rules are checked separately. Three negative controls apply
|
| 12 |
+
CI rules to correlated events or constant score mixing to an ME disjunction;
|
| 13 |
+
all must be rejected. The verifier exits nonzero on any count, tolerance, or
|
| 14 |
+
control failure.
|
evidence/claim-1/negative_controls.csv
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
control,probability_error,score_max_error,rejected
|
| 2 |
+
CI_conjunction_on_correlated_events,0.008553843248334247,0.1965572784695763,True
|
| 3 |
+
CI_disjunction_on_correlated_events,0.008553843248334081,0.05901160358505326,True
|
| 4 |
+
constant_half_mix_for_ME_disjunction,0.0,0.21094950262500398,True
|
evidence/claim-1/primitive_rules.csv
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
seed,operator,probability_error,score_max_error
|
| 2 |
+
0,negation,2.220446049250313e-16,1.1102230246251565e-16
|
| 3 |
+
0,conjunction_ci,0.0,5.551115123125783e-17
|
| 4 |
+
0,disjunction_ci,2.220446049250313e-16,9.71445146547012e-17
|
| 5 |
+
0,disjunction_me,1.1102230246251565e-16,5.3668362670520294e-17
|
| 6 |
+
1,negation,0.0,2.883962288263744e-17
|
| 7 |
+
1,conjunction_ci,0.0,0.0
|
| 8 |
+
1,disjunction_ci,0.0,5.551115123125783e-17
|
| 9 |
+
1,disjunction_me,0.0,5.551115123125783e-17
|
| 10 |
+
2,negation,0.0,5.551115123125783e-17
|
| 11 |
+
2,conjunction_ci,0.0,0.0
|
| 12 |
+
2,disjunction_ci,0.0,1.3877787807814457e-17
|
| 13 |
+
2,disjunction_me,0.0,6.938893903907228e-17
|
| 14 |
+
3,negation,2.7755575615628914e-17,1.1102230246251565e-16
|
| 15 |
+
3,conjunction_ci,0.0,0.0
|
| 16 |
+
3,disjunction_ci,0.0,3.469446951953614e-17
|
| 17 |
+
3,disjunction_me,2.220446049250313e-16,6.297611526094175e-17
|
| 18 |
+
4,negation,0.0,4.5179195726001576e-17
|
| 19 |
+
4,conjunction_ci,0.0,1.3877787807814457e-17
|
| 20 |
+
4,disjunction_ci,1.1102230246251565e-16,2.42861286636753e-17
|
| 21 |
+
4,disjunction_me,0.0,4.766078284585581e-17
|
| 22 |
+
5,negation,0.0,5.551115123125783e-17
|
| 23 |
+
5,conjunction_ci,0.0,0.0
|
| 24 |
+
5,disjunction_ci,5.551115123125783e-17,5.551115123125783e-17
|
| 25 |
+
5,disjunction_me,0.0,5.064680149053605e-17
|
| 26 |
+
6,negation,1.1102230246251565e-16,8.273154794458414e-17
|
| 27 |
+
6,conjunction_ci,0.0,0.0
|
| 28 |
+
6,disjunction_ci,0.0,5.551115123125783e-17
|
| 29 |
+
6,disjunction_me,5.551115123125783e-17,1.1102230246251565e-16
|
| 30 |
+
7,negation,0.0,4.163336342344337e-17
|
| 31 |
+
7,conjunction_ci,0.0,0.0
|
| 32 |
+
7,disjunction_ci,1.1102230246251565e-16,2.7755575615628914e-17
|
| 33 |
+
7,disjunction_me,1.1102230246251565e-16,3.674913685059473e-17
|
| 34 |
+
8,negation,1.1102230246251565e-16,1.207590858399115e-16
|
| 35 |
+
8,conjunction_ci,0.0,1.3877787807814457e-17
|
| 36 |
+
8,disjunction_ci,0.0,9.020562075079397e-17
|
| 37 |
+
8,disjunction_me,0.0,1.2272588407709504e-16
|
| 38 |
+
9,negation,0.0,4.0770251552312994e-17
|
| 39 |
+
9,conjunction_ci,0.0,0.0
|
| 40 |
+
9,disjunction_ci,0.0,4.163336342344337e-17
|
| 41 |
+
9,disjunction_me,0.0,1.6653345369377348e-16
|
| 42 |
+
10,negation,2.220446049250313e-16,1.3877787807814457e-16
|
| 43 |
+
10,conjunction_ci,0.0,0.0
|
| 44 |
+
10,disjunction_ci,0.0,6.938893903907228e-17
|
| 45 |
+
10,disjunction_me,0.0,5.551115123125783e-17
|
| 46 |
+
11,negation,0.0,8.326672684688674e-17
|
| 47 |
+
11,conjunction_ci,0.0,0.0
|
| 48 |
+
11,disjunction_ci,0.0,8.326672684688674e-17
|
| 49 |
+
11,disjunction_me,0.0,5.211649742075978e-17
|
| 50 |
+
12,negation,0.0,2.7755575615628914e-17
|
| 51 |
+
12,conjunction_ci,0.0,0.0
|
| 52 |
+
12,disjunction_ci,0.0,2.7755575615628914e-17
|
| 53 |
+
12,disjunction_me,0.0,2.7755575615628914e-17
|
| 54 |
+
13,negation,5.551115123125783e-17,1.1102230246251565e-16
|
| 55 |
+
13,conjunction_ci,0.0,0.0
|
| 56 |
+
13,disjunction_ci,0.0,2.2551405187698492e-17
|
| 57 |
+
13,disjunction_me,1.1102230246251565e-16,5.580088031179756e-17
|
| 58 |
+
14,negation,2.7755575615628914e-17,1.1102230246251565e-16
|
| 59 |
+
14,conjunction_ci,0.0,0.0
|
| 60 |
+
14,disjunction_ci,0.0,5.551115123125783e-17
|
| 61 |
+
14,disjunction_me,1.1102230246251565e-16,1.9277401717772526e-17
|
| 62 |
+
15,negation,0.0,6.2590736913962e-17
|
| 63 |
+
15,conjunction_ci,0.0,0.0
|
| 64 |
+
15,disjunction_ci,5.551115123125783e-17,5.551115123125783e-17
|
| 65 |
+
15,disjunction_me,1.1102230246251565e-16,6.814781628869505e-17
|
| 66 |
+
16,negation,0.0,6.20184311184098e-17
|
| 67 |
+
16,conjunction_ci,0.0,0.0
|
| 68 |
+
16,disjunction_ci,0.0,1.1102230246251565e-16
|
| 69 |
+
16,disjunction_me,1.1102230246251565e-16,5.551115123125783e-17
|
| 70 |
+
17,negation,0.0,1.1102230246251565e-16
|
| 71 |
+
17,conjunction_ci,0.0,0.0
|
| 72 |
+
17,disjunction_ci,0.0,5.551115123125783e-17
|
| 73 |
+
17,disjunction_me,0.0,5.551115123125783e-17
|
| 74 |
+
18,negation,1.1102230246251565e-16,8.90989013894597e-17
|
| 75 |
+
18,conjunction_ci,0.0,1.1102230246251565e-16
|
| 76 |
+
18,disjunction_ci,2.7755575615628914e-17,1.1102230246251565e-16
|
| 77 |
+
18,disjunction_me,1.1102230246251565e-16,8.96697925030675e-17
|
| 78 |
+
19,negation,0.0,6.245004513516506e-17
|
| 79 |
+
19,conjunction_ci,0.0,0.0
|
| 80 |
+
19,disjunction_ci,0.0,5.551115123125783e-17
|
| 81 |
+
19,disjunction_me,0.0,5.551115123125783e-17
|
| 82 |
+
20,negation,0.0,6.245004513516506e-17
|
| 83 |
+
20,conjunction_ci,0.0,0.0
|
| 84 |
+
20,disjunction_ci,0.0,8.326672684688674e-17
|
| 85 |
+
20,disjunction_me,0.0,2.3633348259675652e-17
|
| 86 |
+
21,negation,1.1102230246251565e-16,1.3877787807814457e-16
|
| 87 |
+
21,conjunction_ci,0.0,3.469446951953614e-18
|
| 88 |
+
21,disjunction_ci,0.0,6.591949208711867e-17
|
| 89 |
+
21,disjunction_me,0.0,1.1102230246251565e-16
|
| 90 |
+
22,negation,0.0,5.962791663333079e-17
|
| 91 |
+
22,conjunction_ci,0.0,1.1102230246251565e-16
|
| 92 |
+
22,disjunction_ci,0.0,1.1102230246251565e-16
|
| 93 |
+
22,disjunction_me,0.0,5.4744513161710626e-17
|
| 94 |
+
23,negation,0.0,1.1102230246251565e-16
|
| 95 |
+
23,conjunction_ci,0.0,2.7755575615628914e-17
|
| 96 |
+
23,disjunction_ci,0.0,5.551115123125783e-17
|
| 97 |
+
23,disjunction_me,0.0,4.5694536608259454e-17
|
| 98 |
+
24,negation,1.6653345369377348e-16,3.3306690738754696e-16
|
| 99 |
+
24,conjunction_ci,0.0,5.551115123125783e-17
|
| 100 |
+
24,disjunction_ci,1.1102230246251565e-16,8.326672684688674e-17
|
| 101 |
+
24,disjunction_me,0.0,3.7129443273294646e-17
|
evidence/claim-1/source_audit.md
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Claim 1 source audit
|
| 2 |
+
|
| 3 |
+
Source retrieved from `https://ar5iv.labs.arxiv.org/html/2602.05549` on
|
| 4 |
+
2026-07-27 at 11:45:56 UTC with User-Agent
|
| 5 |
+
`OpenResearch-Reproduction-Audit/1.0 (+https://github.com/MachineLearning-Nerd/icml26-repro-OAM1jJsMGp-logdiff-exact-boolean-guidance)`.
|
| 6 |
+
The 753,729-byte response has SHA-256
|
| 7 |
+
`67336441b284337c309cd66e2aee8d3a6bd01da716c145debe1319387b9c321a`.
|
| 8 |
+
|
| 9 |
+
The exact theorem is Proposition 3.1 at HTML anchor `#S3.SS1.1`. It quantifies
|
| 10 |
+
over every `t ∈ (0,T]` and every `x ∈ X`. At each conjunction node the two
|
| 11 |
+
events must be conditionally independent given `X_t=x`. At each disjunction
|
| 12 |
+
node they must be conditionally independent or mutually exclusive. Every
|
| 13 |
+
subformula posterior must be exact, differentiable, and strictly between zero
|
| 14 |
+
and one. Under those hypotheses, the recursively computed posterior and
|
| 15 |
+
logical score equal the true values.
|
| 16 |
+
|
| 17 |
+
This executable certificate does not claim to prove the continuous theorem.
|
| 18 |
+
It exhausts the complete stated finite semantic domain used by the certificate,
|
| 19 |
+
with smooth posteriors and exact gradients, and uses a full-joint oracle plus
|
| 20 |
+
finite differences. The result is rigorous for that finite domain and
|
| 21 |
+
corroborates the theorem under its assumptions.
|
evidence/claim-2/EVAL.md
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Claim 2 evaluation contract
|
| 2 |
+
|
| 3 |
+
`summary.json` must report `claim_2.status = FALSIFIED`, with 2/8 LOGDIFF cells
|
| 4 |
+
and 5/8 constant cells inside their claimed intervals. The range-conforming
|
| 5 |
+
negative control must be rejected. Any discrepancy causes a nonzero exit.
|
| 6 |
+
|
| 7 |
+
No generated images are used: the tested statement is contradicted by the
|
| 8 |
+
paper's own exact quantified table cells.
|
evidence/claim-2/claim_contract.json
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"claim_id": 2,
|
| 3 |
+
"paper_anchor": "Table 2, recursive query columns N=2 through N=5",
|
| 4 |
+
"source_url": "https://ar5iv.labs.arxiv.org/html/2602.05549#S3.T2",
|
| 5 |
+
"verdict_target": "FALSIFIED",
|
| 6 |
+
"exact_statement_tested": "Across CMNIST and Shapes3D recursive queries with 2-5 operators, LOGDIFF is within 94-98% and constant mixing is within 63-77%",
|
| 7 |
+
"acceptance": {
|
| 8 |
+
"cells_per_method": 8,
|
| 9 |
+
"paper_values_must_be_audited_cell_by_cell": true,
|
| 10 |
+
"range_conforming_negative_control_must_not_be_falsified": true
|
| 11 |
+
},
|
| 12 |
+
"verifier": "repro/src/run_logdiff.py",
|
| 13 |
+
"fixed_command": "uv sync --frozen && uv run python repro/src/run_logdiff.py --output-dir .openresearch/artifacts --seeds 25"
|
| 14 |
+
}
|
evidence/claim-2/independent_checker_output.txt
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"checked_claims": [
|
| 3 |
+
1,
|
| 4 |
+
2,
|
| 5 |
+
5
|
| 6 |
+
],
|
| 7 |
+
"checker": "standard-library artifact checker",
|
| 8 |
+
"failures": [],
|
| 9 |
+
"status": "PASS"
|
| 10 |
+
}
|
evidence/claim-2/limitations_and_deviations.md
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Claim 2 limitations and deviations
|
| 2 |
+
|
| 3 |
+
- This is a primary-table falsification of the exact challenged range wording.
|
| 4 |
+
- It does not assess whether LOGDIFF generally outperforms constant mixing.
|
| 5 |
+
- No model execution is needed because the paper's own quantified cells
|
| 6 |
+
contradict the stated intervals.
|
evidence/claim-2/method.md
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Claim 2 method
|
| 2 |
+
|
| 3 |
+
The verifier expands the four table rows into 16 individual cells, records the
|
| 4 |
+
claimed interval beside every value, and evaluates membership without rounding
|
| 5 |
+
or aggregation. A synthetic table whose LOGDIFF cells are 96% and constant
|
| 6 |
+
cells are 70% is the negative control: the falsification verifier must reject
|
| 7 |
+
that range-conforming input.
|
evidence/claim-2/negative_control.json
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"control": "range_conforming_table",
|
| 3 |
+
"verifier_rejected_false_falsification": true
|
| 4 |
+
}
|
evidence/claim-2/source_audit.md
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Claim 2 source audit
|
| 2 |
+
|
| 3 |
+
The source retrieval and SHA-256 are recorded in Claim 1's source audit. Table 2
|
| 4 |
+
is anchored at `#S3.T2`; the recursive-query columns are `N=2`, `N=3`, `N=4`,
|
| 5 |
+
and `N=5` for both CMNIST and Shapes3D. The challenged wording quantifies a
|
| 6 |
+
range across all eight dataset/operator-count cells for each method.
|
| 7 |
+
|
| 8 |
+
The paper cells are transcribed directly into the verifier. LOGDIFF spans
|
| 9 |
+
85.1–94.4%, and constant mixing spans 57.9–76.1%. Thus the literal claimed
|
| 10 |
+
ranges are contradicted by their source table. This falsification concerns the
|
| 11 |
+
exact range statement, not the broader qualitative ordering.
|
evidence/claim-2/table_2_cells.csv
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
dataset,method,operators,value_percent,claimed_low,claimed_high,inside_claimed_range
|
| 2 |
+
CMNIST,LOGDIFF,2,93.8,94.0,98.0,False
|
| 3 |
+
CMNIST,LOGDIFF,3,93.3,94.0,98.0,False
|
| 4 |
+
CMNIST,LOGDIFF,4,94.2,94.0,98.0,True
|
| 5 |
+
CMNIST,LOGDIFF,5,94.4,94.0,98.0,True
|
| 6 |
+
Shapes3D,LOGDIFF,2,88.8,94.0,98.0,False
|
| 7 |
+
Shapes3D,LOGDIFF,3,88.6,94.0,98.0,False
|
| 8 |
+
Shapes3D,LOGDIFF,4,85.1,94.0,98.0,False
|
| 9 |
+
Shapes3D,LOGDIFF,5,87.6,94.0,98.0,False
|
| 10 |
+
CMNIST,constant,2,76.1,63.0,77.0,True
|
| 11 |
+
CMNIST,constant,3,66.7,63.0,77.0,True
|
| 12 |
+
CMNIST,constant,4,68.3,63.0,77.0,True
|
| 13 |
+
CMNIST,constant,5,75.2,63.0,77.0,True
|
| 14 |
+
Shapes3D,constant,2,67.2,63.0,77.0,True
|
| 15 |
+
Shapes3D,constant,3,59.4,63.0,77.0,False
|
| 16 |
+
Shapes3D,constant,4,58.4,63.0,77.0,False
|
| 17 |
+
Shapes3D,constant,5,57.9,63.0,77.0,False
|
evidence/claim-3/EVAL.md
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Claim 3 evaluator contract
|
| 2 |
+
|
| 3 |
+
Run the project-wide fixed command:
|
| 4 |
+
|
| 5 |
+
```text
|
| 6 |
+
uv sync --frozen && uv run python repro/src/run_logdiff.py --output-dir .openresearch/artifacts --seeds 25
|
| 7 |
+
```
|
| 8 |
+
|
| 9 |
+
The command exits nonzero if the release audit, independent checker, negative
|
| 10 |
+
control, or any accepted Claim 1/2/5 regression fails.
|
| 11 |
+
|
| 12 |
+
Expected scientific verdict at this node: `BLOCKED`. A passing verifier means
|
| 13 |
+
the BLOCKED distinction is supported; it does not mean the CelebA claim passed.
|
| 14 |
+
|
| 15 |
+
The independent checker additionally requires exactly four distinct routes,
|
| 16 |
+
requires the fourth to be an unsuccessful strict falsification route, validates
|
| 17 |
+
the public-model positive control, and validates the synthetic FID-reversal
|
| 18 |
+
negative control.
|
evidence/claim-3/author_release_snapshot.json
ADDED
|
@@ -0,0 +1,125 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"source_url": "https://github.com/TanjaBien/LogDiff",
|
| 3 |
+
"retrieved_utc": "2026-07-27T12:11:00Z",
|
| 4 |
+
"commit": "94ef35bafd4b4239e9832d8295128c09e8fc1472",
|
| 5 |
+
"commit_time": "2026-05-29T12:32:05+02:00",
|
| 6 |
+
"commit_subject": "readme updated",
|
| 7 |
+
"git_archive_sha256": "d3e7f6a56f665055fd39d54ebdc8c6f0d755e4752877be33a2cdce07be5b4086",
|
| 8 |
+
"tree_file_count": 73,
|
| 9 |
+
"tree_paths": [
|
| 10 |
+
".gitignore",
|
| 11 |
+
"LICENSE",
|
| 12 |
+
"README.md",
|
| 13 |
+
"configs/celeba_inference.yaml",
|
| 14 |
+
"configs/cmnist_inference.yaml",
|
| 15 |
+
"configs/cs_celeba.yaml",
|
| 16 |
+
"configs/cs_celeba_composition.yaml",
|
| 17 |
+
"configs/cs_cmnist.yaml",
|
| 18 |
+
"configs/cs_cmnist_composition.yaml",
|
| 19 |
+
"configs/cs_shapes3d.yaml",
|
| 20 |
+
"configs/cs_shapes3d_composition.yaml",
|
| 21 |
+
"configs/dataset/celeba_cs_latents_male_haircolors copy.yaml",
|
| 22 |
+
"configs/dataset/celeba_dm_latents_male_haircolors.yaml",
|
| 23 |
+
"configs/dataset/celeba_pixel_male_haircolors.yaml",
|
| 24 |
+
"configs/dataset/cmnist_nonuniform.yaml",
|
| 25 |
+
"configs/dataset/cmnist_partial.yaml",
|
| 26 |
+
"configs/dataset/cmnist_uniform.yaml",
|
| 27 |
+
"configs/dataset/shapes3d_orthogonal.yaml",
|
| 28 |
+
"configs/dataset/shapes3d_uniform.yaml",
|
| 29 |
+
"configs/diffusion/celeba_dm_sit.yaml",
|
| 30 |
+
"configs/diffusion/cmnist_dm.yaml",
|
| 31 |
+
"configs/diffusion/cmnist_lace_dm.yaml",
|
| 32 |
+
"configs/diffusion/shapes3d_dm.yaml",
|
| 33 |
+
"configs/dm_celeba.yaml",
|
| 34 |
+
"configs/dm_cmnist.yaml",
|
| 35 |
+
"configs/dm_shapes3d.yaml",
|
| 36 |
+
"configs/shapes3d_inference.yaml",
|
| 37 |
+
"environment.yml",
|
| 38 |
+
"logdiff/__init__.py",
|
| 39 |
+
"logdiff/callbacks/__init__.py",
|
| 40 |
+
"logdiff/callbacks/tracker.py",
|
| 41 |
+
"logdiff/cs_classifier/__init__.py",
|
| 42 |
+
"logdiff/cs_classifier/metrics.py",
|
| 43 |
+
"logdiff/cs_classifier/models.py",
|
| 44 |
+
"logdiff/cs_classifier/train.py",
|
| 45 |
+
"logdiff/cs_classifier/trainer.py",
|
| 46 |
+
"logdiff/cs_classifier/trainer_xt_multihead.py",
|
| 47 |
+
"logdiff/cs_metric.py",
|
| 48 |
+
"logdiff/datasets/__init__.py",
|
| 49 |
+
"logdiff/datasets/celeba.py",
|
| 50 |
+
"logdiff/datasets/coloredMNIST.py",
|
| 51 |
+
"logdiff/datasets/shapes3d.py",
|
| 52 |
+
"logdiff/datasets/sythetic.py",
|
| 53 |
+
"logdiff/evaluate/__init__.py",
|
| 54 |
+
"logdiff/evaluate/evaluate_celeba.py",
|
| 55 |
+
"logdiff/evaluate/evaluate_cmnist.py",
|
| 56 |
+
"logdiff/evaluate/evaluate_cmnist_guidance.py",
|
| 57 |
+
"logdiff/evaluate/evaluate_shapes3d.py",
|
| 58 |
+
"logdiff/evaluate/evaluate_shapes3d_guidance.py",
|
| 59 |
+
"logdiff/evaluate/evaluation_utils.py",
|
| 60 |
+
"logdiff/evaluate/query_generator.py",
|
| 61 |
+
"logdiff/evaluate/validate_judge_celeba.py",
|
| 62 |
+
"logdiff/metrics.py",
|
| 63 |
+
"logdiff/models/__init__.py",
|
| 64 |
+
"logdiff/models/conditional_sit.py",
|
| 65 |
+
"logdiff/models/conditional_unet.py",
|
| 66 |
+
"logdiff/models/label_encoder.py",
|
| 67 |
+
"logdiff/score/__init__.py",
|
| 68 |
+
"logdiff/score/ema.py",
|
| 69 |
+
"logdiff/score/pipelines.py",
|
| 70 |
+
"logdiff/score/sampling_3dshapes.py",
|
| 71 |
+
"logdiff/score/sampling_celeba.py",
|
| 72 |
+
"logdiff/score/sampling_cmnist.py",
|
| 73 |
+
"logdiff/score/sampling_compositional.py",
|
| 74 |
+
"logdiff/score/sampling_compositional_constant.py",
|
| 75 |
+
"logdiff/score/sampling_compositional_models.py",
|
| 76 |
+
"logdiff/score/trainer.py",
|
| 77 |
+
"logdiff/scripts/save_latent.py",
|
| 78 |
+
"logdiff/train.py",
|
| 79 |
+
"logdiff/utils.py",
|
| 80 |
+
"method.png",
|
| 81 |
+
"notebooks/CMNIST.ipynb",
|
| 82 |
+
"notebooks/Shapes3D.ipynb"
|
| 83 |
+
],
|
| 84 |
+
"celeba_required_checkpoint_paths": [
|
| 85 |
+
"checkpoints/celeba/celeba_male_haircolors_normal/tensorboard/version_0/checkpoints/epoch_epoch=589-step_step=750000.ckpt",
|
| 86 |
+
"checkpoints/celeba/cs_celebA_composition_100epochs_male_haircolors/epoch=99-step=254400.ckpt",
|
| 87 |
+
"checkpoints/celeba/cs_celebA_judge_10epochs_male_haircolors/checkpoints/epoch=9-step=15670.ckpt"
|
| 88 |
+
],
|
| 89 |
+
"composition_dataset_requested": "configs/dataset/celeba_cs_latents_male_haircolors.yaml",
|
| 90 |
+
"composition_dataset_near_match": "configs/dataset/celeba_cs_latents_male_haircolors copy.yaml",
|
| 91 |
+
"paper_protocol": {
|
| 92 |
+
"samples_per_task": 5000,
|
| 93 |
+
"queries_per_task": 50,
|
| 94 |
+
"images_per_query": 100,
|
| 95 |
+
"fid_implementation": "clean-fid"
|
| 96 |
+
},
|
| 97 |
+
"release_protocol": {
|
| 98 |
+
"generated_samples_per_task": 5000,
|
| 99 |
+
"default_batch_size": 100,
|
| 100 |
+
"reported_samples_argument": 500000,
|
| 101 |
+
"fid_implementation": "torchmetrics.image.fid.FrechetInceptionDistance",
|
| 102 |
+
"real_split": "CelebA train",
|
| 103 |
+
"real_loader_workers": 4
|
| 104 |
+
},
|
| 105 |
+
"table_3": {
|
| 106 |
+
"AND": {
|
| 107 |
+
"constant_cs": 0.63,
|
| 108 |
+
"constant_fid": 19.02,
|
| 109 |
+
"logdiff_cs": 0.63,
|
| 110 |
+
"logdiff_fid": 19.02
|
| 111 |
+
},
|
| 112 |
+
"NOT": {
|
| 113 |
+
"constant_cs": 0.75,
|
| 114 |
+
"constant_fid": 32.87,
|
| 115 |
+
"logdiff_cs": 0.80,
|
| 116 |
+
"logdiff_fid": 23.61
|
| 117 |
+
},
|
| 118 |
+
"OR-CI": {
|
| 119 |
+
"constant_cs": 0.93,
|
| 120 |
+
"constant_fid": 17.11,
|
| 121 |
+
"logdiff_cs": 0.97,
|
| 122 |
+
"logdiff_fid": 18.47
|
| 123 |
+
}
|
| 124 |
+
}
|
| 125 |
+
}
|
evidence/claim-3/claim_contract.json
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"claim_id": 3,
|
| 3 |
+
"paper": "Logical Guidance for the Exact Composition of Diffusion Models",
|
| 4 |
+
"source": {
|
| 5 |
+
"arxiv_id": "2602.05549",
|
| 6 |
+
"section_anchor": "S4.SS1.p7.1",
|
| 7 |
+
"table_anchor": "S4.T3",
|
| 8 |
+
"protocol_anchor": "A5.SS2.SSS0.Px3.p1.1"
|
| 9 |
+
},
|
| 10 |
+
"statement": "On CelebA negation, LOGDIFF has FID 23.61 versus 32.87 for the constant baseline, while AND and OR-CI conformity is comparable to the baseline.",
|
| 11 |
+
"domain": {
|
| 12 |
+
"dataset": "CelebA",
|
| 13 |
+
"attributes": [
|
| 14 |
+
"Blond/Non-blond",
|
| 15 |
+
"Male/Female"
|
| 16 |
+
],
|
| 17 |
+
"tasks": [
|
| 18 |
+
"AND",
|
| 19 |
+
"NOT",
|
| 20 |
+
"OR-CI"
|
| 21 |
+
],
|
| 22 |
+
"samples_per_task_and_method": 5000,
|
| 23 |
+
"queries_per_task": 50,
|
| 24 |
+
"images_per_query": 100,
|
| 25 |
+
"fid_implementation": "clean-fid"
|
| 26 |
+
},
|
| 27 |
+
"paper_values": {
|
| 28 |
+
"AND": {
|
| 29 |
+
"constant_cs": 0.63,
|
| 30 |
+
"constant_fid": 19.02,
|
| 31 |
+
"logdiff_cs": 0.63,
|
| 32 |
+
"logdiff_fid": 19.02
|
| 33 |
+
},
|
| 34 |
+
"NOT": {
|
| 35 |
+
"constant_cs": 0.75,
|
| 36 |
+
"constant_fid": 32.87,
|
| 37 |
+
"logdiff_cs": 0.80,
|
| 38 |
+
"logdiff_fid": 23.61
|
| 39 |
+
},
|
| 40 |
+
"OR-CI": {
|
| 41 |
+
"constant_cs": 0.93,
|
| 42 |
+
"constant_fid": 17.11,
|
| 43 |
+
"logdiff_cs": 0.97,
|
| 44 |
+
"logdiff_fid": 18.47
|
| 45 |
+
}
|
| 46 |
+
},
|
| 47 |
+
"direct_verification_requirements": [
|
| 48 |
+
"Author-equivalent CelebA diffusion, composition-classifier, and judge weights",
|
| 49 |
+
"The paper's 5000 samples per task and method with 50 queries of 100 images",
|
| 50 |
+
"clean-fid evaluated independently for each task",
|
| 51 |
+
"Deterministic seeds plus uncertainty across independent generation replicates",
|
| 52 |
+
"Independent conformity and FID checker",
|
| 53 |
+
"Negative control that detects a deliberately corrupted method label or image set"
|
| 54 |
+
],
|
| 55 |
+
"verified_if": "A faithful executable campaign reproduces the direction of the NOT FID advantage and the reported AND/OR-CI conformity relationship, with uncertainty and protocol parity fully exposed.",
|
| 56 |
+
"falsified_if": "A faithful executable campaign satisfying every stated protocol assumption yields an assumption-satisfying contradiction, such as a statistically supported reversal of the NOT FID ordering.",
|
| 57 |
+
"blocked_if": "The author-equivalent model state or faithful metric protocol is unavailable, and neither direct verification nor assumption-satisfying falsification can be executed.",
|
| 58 |
+
"release_audit_scope": "The release audit tests executability only. It must never relabel an absent-checkpoint or protocol-mismatch result as empirical verification or falsification."
|
| 59 |
+
}
|
evidence/claim-3/falsification_check.json
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"claim": 3,
|
| 3 |
+
"route": 4,
|
| 4 |
+
"paper_table": {
|
| 5 |
+
"not_logdiff_fid": 23.61,
|
| 6 |
+
"not_constant_fid": 32.87,
|
| 7 |
+
"and_logdiff_cs": 0.63,
|
| 8 |
+
"and_constant_cs": 0.63,
|
| 9 |
+
"or_ci_logdiff_cs": 0.97,
|
| 10 |
+
"or_ci_constant_cs": 0.93
|
| 11 |
+
},
|
| 12 |
+
"claimed_not_fid_ordering_holds_in_paper_table": true,
|
| 13 |
+
"synthetic_negative_control": {
|
| 14 |
+
"not_logdiff_fid": 35.0,
|
| 15 |
+
"not_constant_fid": 30.0,
|
| 16 |
+
"reversal_detected": true
|
| 17 |
+
},
|
| 18 |
+
"available_evidence_admissible_as_counterexample": false,
|
| 19 |
+
"inadmissibility_reasons": [
|
| 20 |
+
"no author-equivalent diffusion checkpoint",
|
| 21 |
+
"no author-equivalent composition classifier",
|
| 22 |
+
"no author-equivalent independent judge",
|
| 23 |
+
"no exact 5000-image-per-task output",
|
| 24 |
+
"no clean-fid output from an exact-method campaign"
|
| 25 |
+
],
|
| 26 |
+
"falsification_succeeded": false,
|
| 27 |
+
"status": "BLOCKED"
|
| 28 |
+
}
|
evidence/claim-3/independent_checker_output.txt
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"checker": "independent Claim 3 release-integrity checker",
|
| 3 |
+
"failures": [],
|
| 4 |
+
"scientific_claim_status": "BLOCKED",
|
| 5 |
+
"status": "PASS",
|
| 6 |
+
"verification_routes_rederived": 4
|
| 7 |
+
}
|
evidence/claim-3/limitations_and_deviations.md
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Claim 3 limitations and deviations
|
| 2 |
+
|
| 3 |
+
- No author-equivalent CelebA checkpoint is present in the pinned release.
|
| 4 |
+
- No independent CelebA generation or FID campaign is claimed at this node.
|
| 5 |
+
- The current author release postdates the paper and contains only three Git
|
| 6 |
+
commits; this audit makes no claim about unreleased author state.
|
| 7 |
+
- Table transcription validates what the paper reports, not whether the
|
| 8 |
+
reported samples or measurements can be independently regenerated.
|
| 9 |
+
- Missing files, a broken config, and a metric implementation mismatch are
|
| 10 |
+
reproducibility blockers—not valid counterexamples to the empirical result.
|
| 11 |
+
- The public Diffusers CelebA UNet is a deliberately rejected reconstruction
|
| 12 |
+
candidate, not a substitute for the paper's SiT and classifier states.
|
| 13 |
+
- Four materially different routes were completed. None yields the exact
|
| 14 |
+
images, model state, or clean-fid measurements required to resolve the
|
| 15 |
+
empirical claim.
|
evidence/claim-3/method.md
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Claim 3 release-integrity method
|
| 2 |
+
|
| 3 |
+
The verifier reads a pinned, complete Git tree manifest of the current author
|
| 4 |
+
release. It checks every declared CelebA checkpoint path, resolves the exact
|
| 5 |
+
Hydra dataset target, and compares the paper's sample and FID contract with
|
| 6 |
+
the executable evaluation path.
|
| 7 |
+
|
| 8 |
+
The independent checker re-derives those predicates from the raw JSON rather
|
| 9 |
+
than trusting the headline status. The negative control injects all required
|
| 10 |
+
paths and a matching clean-fid protocol; the same decision function must then
|
| 11 |
+
refuse to label the release BLOCKED.
|
| 12 |
+
|
| 13 |
+
This is an executability audit, not a substitute generation experiment.
|
| 14 |
+
|
| 15 |
+
The public provenance route uses independent GitHub and Hugging Face API
|
| 16 |
+
responses preserved under `public-checkpoint-search/`. It rejects unrelated
|
| 17 |
+
generic-name results. The reconstruction route positively resolves a public
|
| 18 |
+
CelebA DDPM revision and its model tree, then rejects it as non-equivalent
|
| 19 |
+
because the paper requires a different diffusion state plus composition and
|
| 20 |
+
judge classifiers.
|
| 21 |
+
|
| 22 |
+
The mandatory falsification route admits only exact-method image sets evaluated
|
| 23 |
+
with the paper's 5,000-sample clean-fid protocol. It confirms the Table 3
|
| 24 |
+
transcription and checks a synthetic reversed-FID control. Missing checkpoints
|
| 25 |
+
and a different public DDPM are not treated as counterexamples.
|
evidence/claim-3/negative_control.json
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"audit_status": "READY_TO_RUN",
|
| 3 |
+
"control": "complete_release_with_matching_clean_fid_protocol",
|
| 4 |
+
"details": {
|
| 5 |
+
"fid_implementation_mismatch": false,
|
| 6 |
+
"missing_checkpoint_paths": [],
|
| 7 |
+
"missing_composition_dataset_config": false,
|
| 8 |
+
"sample_accounting_mismatch": false
|
| 9 |
+
},
|
| 10 |
+
"rejected_false_block": true
|
| 11 |
+
}
|
evidence/claim-3/public-checkpoint-search/github-branches.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
[{"name":"main","commit":{"sha":"94ef35bafd4b4239e9832d8295128c09e8fc1472","url":"https://api.github.com/repos/TanjaBien/LogDiff/commits/94ef35bafd4b4239e9832d8295128c09e8fc1472"},"protected":false}]
|
evidence/claim-3/public-checkpoint-search/github-forks.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
[]
|
evidence/claim-3/public-checkpoint-search/github-releases.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
[]
|
evidence/claim-3/public-checkpoint-search/github-tags.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
[]
|
evidence/claim-3/public-checkpoint-search/hf-datasets-LogDiff.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
[{"_id":"69a44fbb1a34335008a1829e","id":"motobrew/alf-dpo-logdiff-strict-v1","author":"motobrew","disabled":false,"gated":false,"lastModified":"2026-03-01T14:41:40.000Z","likes":0,"trendingScore":0,"private":false,"sha":"25dedd20befa2b07f18a732bc2f9b159f9c5d5c3","downloads":17,"tags":["size_categories:n<1K","format:json","modality:text","library:datasets","library:pandas","library:polars","library:mlcroissant","region:us"],"createdAt":"2026-03-01T14:39:55.000Z","key":""}]
|
evidence/claim-3/public-checkpoint-search/hf-datasets-TanjaBien.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
[]
|
evidence/claim-3/public-checkpoint-search/hf-datasets-cs_celebA_composition_100epochs_male_haircolors.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
[]
|
evidence/claim-3/public-checkpoint-search/hf-datasets-epoch_epoch_3D589-step_step_3D750000.ckpt.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
[]
|
evidence/claim-3/public-checkpoint-search/hf-models-LogDiff.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
[]
|
evidence/claim-3/public-checkpoint-search/hf-models-TanjaBien.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
[]
|
evidence/claim-3/public-checkpoint-search/hf-models-cs_celebA_composition_100epochs_male_haircolors.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
[]
|
evidence/claim-3/public-checkpoint-search/hf-models-epoch_epoch_3D589-step_step_3D750000.ckpt.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
[]
|
evidence/claim-3/public-checkpoint-search/hf-public-celeba-model.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"_id":"62f94e2e4588fe31f439c601","id":"shalpin87/diffusion_celeba","private":false,"library_name":"diffusers","tags":["diffusers","en","dataset:CelebA","license:apache-2.0","diffusers:DDPMPipeline","region:us"],"downloads":5,"likes":0,"modelId":"shalpin87/diffusion_celeba","author":"shalpin87","sha":"91d0ff096e031c21b8313bd5877316a52900d4ec","lastModified":"2022-08-14T21:46:27.000Z","gated":false,"disabled":false,"model-index":null,"config":{"diffusers":{"_class_name":"DDPMPipeline"}},"cardData":{"language":"en","license":"apache-2.0","library_name":"diffusers","tags":[],"datasets":"CelebA","metrics":[]},"siblings":[{"rfilename":".gitattributes"},{"rfilename":".gitignore"},{"rfilename":"README.md"},{"rfilename":"model_index.json"},{"rfilename":"scheduler/scheduler_config.json"},{"rfilename":"unet/config.json"},{"rfilename":"unet/diffusion_pytorch_model.bin"}],"spaces":[],"createdAt":"2022-08-14T19:34:06.000Z","usedStorage":23373691520}
|
evidence/claim-3/public-checkpoint-search/hf-public-celeba-tree.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
[{"type":"directory","oid":"0c8e71fad0ff2ded362b557e3a1dcf3acb159445","size":0,"path":"scheduler"},{"type":"directory","oid":"e5947b6aeb9eb64d7d017b2510b7118f933dd6bb","size":0,"path":"unet"},{"type":"file","oid":"78be8432e1148d0227370439dad9d9a818f08df4","size":1338,"path":".gitattributes"},{"type":"file","oid":"0348ea97130c017c407fcfb6fd4003859f17b84c","size":13,"path":".gitignore"},{"type":"file","oid":"bbba5d1ea30d67ea76ea3ff9126f7358496b2220","size":1293,"path":"README.md"},{"type":"file","oid":"37da25a4ce772c1e94e04160652ca28aeb9f54ad","size":180,"path":"model_index.json"},{"type":"file","oid":"0f9c3d5d958d00eccbe167d561907b147b18c5eb","size":256,"path":"scheduler/scheduler_config.json"},{"type":"file","oid":"dc14cf9c0217e35da1520aba3f40d4cdd6e0c607","size":695,"path":"unet/config.json"},{"type":"file","oid":"5cc7cb178ff1716d6a7dad4106a79ca727829dab","size":114049969,"lfs":{"oid":"930f28acfcad1137c5c38f0cf0aaeebc61dc607c6e54369c3da7df9309f90ced","size":114049969,"pointerSize":134},"xetHash":"dfaf36dfcbc66feded19e24262661807112e327732de7421d57be9dbb53299b3","path":"unet/diffusion_pytorch_model.bin"}]
|
evidence/claim-3/public-checkpoint-search/hf-spaces-LogDiff.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
[]
|
evidence/claim-3/public-checkpoint-search/hf-spaces-TanjaBien.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
[]
|