"""Evidence and layout checks. Integration fixtures live only in temporary dirs.""" import hashlib import importlib.util import json import os from pathlib import Path import shutil import subprocess import tempfile import unittest from unittest.mock import patch ROOT = Path(__file__).resolve().parents[1] spec = importlib.util.spec_from_file_location("build_demo", ROOT / "scripts/build_demo.py") demo = importlib.util.module_from_spec(spec) spec.loader.exec_module(demo) FILES = {"project.godot": 'config_version=5\n[application]\nrun/main_scene="res://main.tscn"\n', "main.tscn": '[gd_scene load_steps=2 format=3]\n[ext_resource type="Script" path="res://main.gd" id="1"]\n[node name="Main" type="Node2D"]\nscript = ExtResource("1")\n', "main.gd": "extends Node2D\n\nfunc _ready():\n queue_redraw()\n\nfunc _draw():\n draw_circle(Vector2(200, 180), 40, Color.CYAN)\n"} def write_json(path, value): path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(value), encoding="utf-8") def fixture_evaluation(root): manifest = {"expected_count": 500, "run_seconds": 60, "smoke_test": False, "sample_indices": [0, 200, 400], "generation": {"backend": "unit-test-only"}} write_json(root / "manifest.json", manifest) for index in range(500): item = root / "items" / f"{index:04d}" attempt = {"index": index, "prompt_id": f"test-{index}", "instruction": f"Distinct test fixture instruction number {index}."} write_json(item / "attempt.json", attempt) result = {"index": index, "prompt_id": attempt["prompt_id"], "compile_success": index in (0, 200), "clean_run_60s": index in (0, 200), "run_seconds": 60, "project_sha256": demo.project_sha256(FILES), "prompt_to_launch_seconds": 123.5 if index in (0, 200) else None} write_json(item / "result.json", result) if index in (0, 200, 400): output = json.dumps({"files": FILES}) generation = {"output": output, "output_sha256": hashlib.sha256(output.encode()).hexdigest(), "generation_seconds": 119.0} write_json(item / "generation.json", generation) video = root / "recordings" / "sample-000.mp4" video.parent.mkdir() video.write_bytes(b"unit test placeholder; ffprobe is mocked") write_json(video.with_suffix(".recording.json"), {"capture_success": True, "duration_seconds": 15, "fixed_timestep": True, "player_input": False, "sample_index": 0, "prompt_id": "test-0", "evaluation_manifest_sha256": demo.file_hash(root / "manifest.json"), "generation_sha256": demo.file_hash(root / "items/0000/generation.json"), "project_sha256": demo.project_sha256(FILES), "output_sha256": demo.file_hash(video)}) return video class DemoEvidenceTests(unittest.TestCase): def setUp(self): self.temp = tempfile.TemporaryDirectory() self.root = Path(self.temp.name) self.video = fixture_evaluation(self.root) def tearDown(self): self.temp.cleanup() @patch.object(demo, "probe_video", return_value={"duration_seconds": 15, "width": 960, "height": 540}) def test_all_500_are_counted_and_fixed_failure_slots_retained(self, probe): plan = demo.evaluation_plan(self.root) self.assertEqual(plan["metrics"]["denominator"], 500) self.assertEqual([sample["index"] for sample in plan["samples"]], [0, 200, 400]) self.assertEqual(plan["metrics"]["compile_success_count"], 2) self.assertEqual(plan["metrics"]["clean_run_60s_count"], 2) self.assertEqual(plan["metrics"]["compile_success_rate"], .004) self.assertEqual(plan["metrics"]["prompt_to_launch_median_seconds"], 123.5) self.assertEqual(plan["metrics"]["under_60s_launch_count"], 0) self.assertIsNotNone(plan["samples"][0]["capture"]) self.assertIn("recording is unavailable", plan["samples"][1]["diagnostic"]) self.assertIn("did not import", plan["samples"][2]["diagnostic"]) self.assertTrue(plan["includes_failed_or_unrecorded_samples"]) self.assertIn("119.0s", demo.latency_text(plan["samples"][0])) self.assertIn("123.5s", demo.latency_text(plan["samples"][0])) def test_partial_evaluation_is_rejected(self): (self.root / "items/0499/result.json").unlink() with self.assertRaisesRegex(ValueError, "exactly results"): demo.evaluation_plan(self.root) def test_no_captures_cannot_become_a_gameplay_demo(self): self.video.unlink() with self.assertRaisesRegex(ValueError, "At least one real recording"): demo.evaluation_plan(self.root) def test_wrong_generation_hash_is_rejected(self): sidecar = self.video.with_suffix(".recording.json") metadata = demo.read_json(sidecar) metadata["generation_sha256"] = "different generation" write_json(sidecar, metadata) with self.assertRaisesRegex(ValueError, "generation_sha256"): demo.evaluation_plan(self.root) def test_video_content_hash_is_verified(self): self.video.write_bytes(b"modified video") with self.assertRaisesRegex(ValueError, "content hash"): demo.evaluation_plan(self.root) def test_infrastructure_failure_blocks_final_rates(self): path = self.root / "items/0123/result.json" value = demo.read_json(path); value["infrastructure_failure"] = True write_json(path, value) with self.assertRaisesRegex(ValueError, "Infrastructure failures"): demo.evaluation_plan(self.root) def test_summary_cannot_inflate_successes(self): write_json(self.root / "summary.json", {"status": "complete", "recorded_prompts": 500, "compile_success_count": 500, "clean_run_60s_count": 500}) with self.assertRaisesRegex(ValueError, "summary disagrees"): demo.evaluation_plan(self.root) def test_changed_generation_cannot_use_original_capture(self): path = self.root / "items/0000/generation.json" generation = demo.read_json(path); generation["output"] += " " write_json(path, generation) with self.assertRaisesRegex(ValueError, "output hash mismatch"): demo.evaluation_plan(self.root) def preview_fixture_plan(video: Path) -> dict: return {"kind": "unit-test-fixture-not-a-project-demo", "scaffold_preview": True, "metrics": None, "includes_failed_or_unrecorded_samples": False, "samples": [{"index": index, "prompt_id": f"test-{index}", "instruction": "This is a synthetic unit-test title card, not model performance evidence.", "files": FILES, "generation_seconds": None, "prompt_to_launch_seconds": None, "capture": {"path": str(video)}, "diagnostic": None} for index in range(3)]} @unittest.skipUnless(importlib.util.find_spec("PIL"), "Pillow is an optional rendering dependency") class DemoLayoutTests(unittest.TestCase): def test_cards_fit_and_preview_watermark_is_permanent(self): from PIL import Image with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) plan = preview_fixture_plan(root / "unused.mp4") assets = demo.render_assets(plan, root / "assets") self.assertEqual([asset["seconds"] for asset in assets], [8, 6, 14, 14, 14, 4]) self.assertEqual(sum(asset["seconds"] for asset in assets), 60) for asset in assets: image = Image.open(asset.get("overlay", asset["path"])) self.assertEqual(image.size, (1280, 720)) red, green, blue = image.convert("RGB").getpixel((5, 710)) self.assertGreater(red, green * 2) @unittest.skipUnless(os.environ.get("VIBEGAME_VIDEO_INTEGRATION") == "1" and shutil.which("ffmpeg") and shutil.which("ffprobe"), "Set VIBEGAME_VIDEO_INTEGRATION=1 for the CPU-only 60-second encoding test") def test_actual_encoding_is_60_seconds_at_720p_and_30fps(self): with tempfile.TemporaryDirectory(prefix="vibegame-video-test-") as temporary: root = Path(temporary) fixture = root / "test-color-source.mp4" subprocess.run([shutil.which("ffmpeg"), "-nostdin", "-v", "error", "-f", "lavfi", "-i", "color=c=blue:s=960x540:r=30:d=15", "-c:v", "libx264", "-pix_fmt", "yuv420p", str(fixture)], check=True) result = demo.encode_video(preview_fixture_plan(fixture), root / "test-output.mp4") self.assertAlmostEqual(result["seconds"], 60, places=2) evidence = demo.read_json(Path(result["evidence"])) self.assertEqual(evidence["probe"]["width"], 1280) self.assertEqual(evidence["probe"]["height"], 720) self.assertEqual(evidence["probe"]["r_frame_rate"], "30/1") self.assertEqual(evidence["watermark"], demo.PREVIEW_WATERMARK) if __name__ == "__main__": unittest.main()