Spaces:
Sleeping
Sleeping
File size: 5,521 Bytes
31226fd | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 | """Run the complete suite and publish an honest, source-bound test baseline.
Ordinary pytest invocations never update tracked verification artifacts. This
explicit command enables their publication and rejects failed/interrupted runs
or a source tree that changes while verification is running.
"""
from __future__ import annotations
import hashlib
import json
import os
import platform
import re
import subprocess
import sys
import tempfile
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from xml.etree import ElementTree
REPO_ROOT = Path(__file__).resolve().parent.parent
BASELINE_PATH = REPO_ROOT / "tests" / "test_baseline.json"
sys.path.insert(0, str(REPO_ROOT))
def main() -> int:
from src.dashboard.verification import (
SCIENTIFIC_PATHS,
TEST_PATHS,
canonical_source_file_bytes,
compute_source_tree_digest,
current_git_head,
inspect_source_tree,
source_digest_paths,
)
initial_inspection = inspect_source_tree(REPO_ROOT)
if initial_inspection.mismatching_files:
print(
"ERROR: Source freeze must be generated from a clean relevant tree: "
+ ", ".join(initial_inspection.mismatching_files)
)
return 1
digest = compute_source_tree_digest(REPO_ROOT)
if digest is None:
print("ERROR: Cannot compute source digest")
return 1
with tempfile.TemporaryDirectory(prefix="brain5d-baseline-") as directory:
report = Path(directory) / "pytest.xml"
command = [
sys.executable,
"-m",
"pytest",
"tests/",
"-q",
"-ra",
f"--junitxml={report}",
]
environment = {**os.environ, "BRAIN5D_WRITE_VERIFICATION_ARTIFACTS": "1"}
result = subprocess.run(
command,
cwd=REPO_ROOT,
env=environment,
capture_output=True,
text=True,
check=False,
timeout=900,
)
print(result.stdout)
print(result.stderr, file=sys.stderr)
if not report.is_file():
print("ERROR: pytest produced no machine-readable results")
return 1
suites = ElementTree.parse(report).getroot().findall(".//testsuite")
tests = sum(int(suite.get("tests", "0")) for suite in suites)
failed = sum(int(suite.get("failures", "0")) for suite in suites)
errors = sum(int(suite.get("errors", "0")) for suite in suites)
skipped = sum(int(suite.get("skipped", "0")) for suite in suites)
expected_failures = sum(
1
for item in ElementTree.parse(report).getroot().iter("skipped")
if item.get("type") == "pytest.xfail"
)
unexpected_pass_counts = re.findall(r"(\d+) xpassed", result.stdout)
unexpected_passes = (
int(unexpected_pass_counts[-1]) if unexpected_pass_counts else 0
)
passed = tests - failed - errors - skipped - unexpected_passes
final_inspection = inspect_source_tree(REPO_ROOT)
unchanged = (
final_inspection.digest == digest and not final_inspection.mismatching_files
)
success = (
result.returncode == 0
and passed > 0
and errors == 0
and failed == 0
and unchanged
)
baseline: dict[str, Any] = {
"schema_version": 2,
"tested_commit": current_git_head(REPO_ROOT),
"tested_tree_digest": digest,
"tree_digest_paths": [
"src/",
"configs/",
"research/schemas/",
"pyproject.toml",
"tests/",
],
"tree_digest_excludes": ["tests/test_baseline.json"],
"timestamp": datetime.now(timezone.utc).isoformat(),
"python": platform.python_version(),
"platform": platform.platform(),
"source_unchanged_during_run": unchanged,
"full_collection": {
"status": "passed" if success else "failed",
"collected": tests,
"collection_errors": errors,
},
"full_suite": {
"command": "python -m pytest tests/ -q -ra --junitxml=<temporary-report>",
"status": "passed" if success else "failed",
"exit_code": result.returncode,
"passed": passed,
"failed": failed,
"errors": errors,
"skipped": skipped - expected_failures,
"xfailed": expected_failures,
"xpassed": unexpected_passes,
"skipped_reasons": [
element.text or ""
for element in ElementTree.parse(report).getroot().iter("skipped")
],
},
}
baseline["source_files"] = {
relative: hashlib.sha256(
canonical_source_file_bytes(REPO_ROOT, relative)
).hexdigest()
for relative in source_digest_paths(REPO_ROOT, SCIENTIFIC_PATHS + TEST_PATHS)
}
BASELINE_PATH.write_text(
json.dumps(baseline, indent=2, ensure_ascii=False) + "\n", encoding="utf-8"
)
print(
f"Baseline: {passed} passed, {failed} failed, {errors} errors, {skipped} skipped; unchanged={unchanged}"
)
return 0 if success else 1
if __name__ == "__main__":
raise SystemExit(main())
|