Q-TensorFormer / scripts /scientific_integrity_checker.py
Premchandyadav369
feat(neurips-ieee): camera-ready publication artifacts, Theorems 1-4, 50-trial statistical significance, interactive deep-dive tab
6d1f076
Raw History Blame Contribute Delete
6.35 kB
#!/usr/bin/env python3
"""
scripts/scientific_integrity_checker.py
Automated Scientific Integrity & Evidence Classification Auditor (Phase 12).
Verifies:
1. All required reports in reports/ and raw benchmark files in results/raw/ exist and are non-empty.
2. Every benchmark block in results/raw/*.json includes a valid scientific_classification label.
3. Production checkpoint loaders (qtensorformer_to_gguf.py, configuration_qtensorformer.py)
do not use strict=False or ignore missing embedding weights.
4. Documentation (README.md, MODEL_CARD.md, reports/*.md) contains no forbidden promotional
claims and has zero broken Markdown tables or unbalanced LaTeX $$ blocks.
"""
import json
import sys
from pathlib import Path
from typing import List
ROOT_DIR = Path(__file__).resolve().parent.parent
REQUIRED_REPORTS = [
"BASELINE_AUDIT.md",
"REPOSITORY_INVENTORY.md",
"EXISTING_FAILURES.md",
"ARCHITECTURE_FAMILY_MAP.md",
"CLAIM_TO_CODE_MAP.md",
"SOURCE_API_AUDIT.md",
"CONFIG_PROVENANCE.md",
"CHECKPOINT_INTEGRITY.md",
"CANONICAL_MODEL_DECISION.md",
"COLD_START_EVALUATION.md",
"EDGE_INFERENCE_EVALUATION.md",
"ENERGY_MEASUREMENT_METHODOLOGY.md",
"BENCHMARK_PROTOCOL.md",
"BASELINE_FAIRNESS.md",
"BENCHMARK_RESULTS.md",
"TEST_COVERAGE.md",
"SCIENTIFIC_INTEGRITY_AUDIT.md",
"RELATED_WORK_AND_NOVELTY.md",
"FINAL_RESEARCH_AUDIT.md",
"RELEASE_READINESS.md",
"KNOWN_LIMITATIONS.md",
"REPRODUCIBILITY_CHECKLIST.md",
"FRONTIER_RESEARCH_BENCHMARKS.md",
"NEURIPS_CAMERA_READY_PAPER.md",
]
REQUIRED_RAW_JSON = [
"checkpoint_integrity.json",
"cold_start_and_edge.json",
"tt_compression_audit.json",
"controller_audit.json",
"kv_cache_audit.json",
"quantum_pathway_audit.json",
"frontier_research_benchmarks.json",
"neurips_statistical_evaluation.json",
]
ALLOWED_LABELS = {
"MEASURED",
"ESTIMATED",
"SIMULATED",
"THEORETICAL",
"PROJECTED",
"N/A",
"BLOCKED",
"UNVERIFIED",
}
FORBIDDEN_CLAIMS = [
"guaranteed energy savings",
"quantum speedup on cpu",
"quantum advantage in nlp",
"zero runtime overhead",
"first ever adaptive transformer",
"asymptotic pareto optimality guaranteed",
]
def check_markdown_tables_and_math(file_path: Path, errors: List[str]):
text = file_path.read_text(encoding="utf-8")
lines = text.splitlines()
# Check $$ display math balance outside fenced code blocks
in_code = False
dd_count = 0
in_table = False
expected_cols = 0
for idx, line in enumerate(lines, start=1):
stripped = line.strip()
if stripped.startswith("```"):
in_code = not in_code
in_table = False
continue
if in_code:
continue
dd_count += line.count("$$")
if stripped.startswith("|") and stripped.endswith("|"):
cols = len([c for c in stripped.split("|")[1:-1]])
if not in_table:
in_table = True
expected_cols = cols
else:
if cols != expected_cols:
errors.append(
f"[{file_path.name}:{idx}] Table column mismatch: expected {expected_cols} columns, got {cols}"
)
else:
in_table = False
if dd_count % 2 != 0:
errors.append(
f"[{file_path.name}] Unbalanced $$ display math delimiters (count={dd_count})"
)
# Check forbidden promotional phrases (ignoring lines that explicitly list them as forbidden)
for idx, line in enumerate(lines, start=1):
low = line.lower()
if "forbidden" in low or "prohibit" in low or "rejects" in low:
continue
for phrase in FORBIDDEN_CLAIMS:
if phrase in low:
errors.append(
f"[{file_path.name}:{idx}] Forbidden promotional phrase detected: '{phrase}'"
)
def main():
errors: List[str] = []
# 1. Check required reports
reports_dir = ROOT_DIR / "reports"
for rname in REQUIRED_REPORTS:
rpath = reports_dir / rname
if not rpath.exists() or rpath.stat().st_size < 100:
errors.append(f"Missing or empty report: reports/{rname}")
else:
check_markdown_tables_and_math(rpath, errors)
# 2. Check required raw JSON files and scientific classifications
raw_dir = ROOT_DIR / "results" / "raw"
for jname in REQUIRED_RAW_JSON:
jpath = raw_dir / jname
if not jpath.exists():
errors.append(f"Missing raw benchmark file: results/raw/{jname}")
continue
data = json.loads(jpath.read_text(encoding="utf-8"))
raw_str = json.dumps(data)
if "scientific_classification" not in raw_str:
errors.append(f"results/raw/{jname} lacks 'scientific_classification' labels")
# 3. Check production source files for strict=False bypasses
gguf_src = (ROOT_DIR / "qtensorformer_to_gguf.py").read_text(encoding="utf-8")
if "strict=False" in gguf_src:
errors.append("qtensorformer_to_gguf.py contains forbidden 'strict=False'")
cfg_src = (ROOT_DIR / "configuration_qtensorformer.py").read_text(encoding="utf-8")
if 'keys_to_ignore_at_loading = ["model.embed_tokens.weight"]' in cfg_src:
errors.append(
"configuration_qtensorformer.py still ignores model.embed_tokens.weight at loading"
)
# 4. Check README.md and MODEL_CARD.md
for doc_name in ["README.md", "MODEL_CARD.md"]:
doc_path = ROOT_DIR / doc_name
if not doc_path.exists():
errors.append(f"Missing {doc_name}")
else:
check_markdown_tables_and_math(doc_path, errors)
print("=" * 70)
print("Q-TensorFormer — Automated Scientific Integrity Audit")
print("=" * 70)
print(f"Reports Verified: {len(REQUIRED_REPORTS)}")
print(f"Raw JSON Verified: {len(REQUIRED_RAW_JSON)}")
print(f"Integrity Errors: {len(errors)}")
if errors:
for e in errors:
print(f" [FAIL] {e}")
sys.exit(1)
else:
print("[PASS] Zero scientific integrity violations, zero broken tables/equations.")
print("=" * 70)
if __name__ == "__main__":
main()