Preserve terminal evaluation observer for unattended recovery

scripts/watch_review23_final_eval.py

@@ -0,0 +1,44 @@
+"""Observe the existing E3 evaluator and archive its terminal state; never launch training."""
+import json
+import os
+import subprocess
+import time
+from datetime import datetime, timezone
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+OUT = ROOT / 'results/phase6/review23/e3'
+PID = 621209
+
+def alive():
+    try:
+        return 'evaluate_review23_e3_final.py' in Path(f'/proc/{PID}/cmdline').read_text()
+    except FileNotFoundError:
+        return False
+
+def main():
+    while alive():
+        time.sleep(30)
+    metrics = OUT / 'final_metrics.json'
+    status = json.loads((OUT / 'pipeline_status.json').read_text())
+    status['observed_at_utc'] = datetime.now(timezone.utc).isoformat()
+    if metrics.exists():
+        final = json.loads(metrics.read_text())
+        valid = (final.get('status') == 'complete' and final.get('test_read') is False
+                 and final.get('config_sha256') == status.get('config_sha256')
+                 and 'primary_seed42' in final.get('models', {}))
+        status['status'] = 'complete' if valid else 'failed_final_metrics_integrity'
+        status['final_metrics'] = str(metrics.relative_to(ROOT))
+    else:
+        status['status'] = 'failed_final_evaluation'
+        status['failure_reason'] = 'Observed evaluator terminated without final_metrics.json; inspect final_eval.log. No automatic retry.'
+    status['stage'] = 'terminal_final_evaluation'
+    status['test_read'] = False
+    temp = OUT / 'pipeline_status.completion.tmp'
+    temp.write_text(json.dumps(status, indent=2) + '\n')
+    os.replace(temp, OUT / 'pipeline_status.json')
+    print(json.dumps(status), flush=True)
+    subprocess.run([str(ROOT / '.venv/bin/python'), '-u', str(ROOT / 'scripts/hf_sync_snapshot.py')], cwd=ROOT, check=True)
+
+if __name__ == '__main__':
+    main()