| """Evidence and layout checks. Integration fixtures live only in temporary dirs.""" |
| import hashlib |
| import importlib.util |
| import json |
| import os |
| from pathlib import Path |
| import shutil |
| import subprocess |
| import tempfile |
| import unittest |
| from unittest.mock import patch |
|
|
| ROOT = Path(__file__).resolve().parents[1] |
| spec = importlib.util.spec_from_file_location("build_demo", ROOT / "scripts/build_demo.py") |
| demo = importlib.util.module_from_spec(spec) |
| spec.loader.exec_module(demo) |
|
|
| FILES = {"project.godot": 'config_version=5\n[application]\nrun/main_scene="res://main.tscn"\n', |
| "main.tscn": '[gd_scene load_steps=2 format=3]\n[ext_resource type="Script" path="res://main.gd" id="1"]\n[node name="Main" type="Node2D"]\nscript = ExtResource("1")\n', |
| "main.gd": "extends Node2D\n\nfunc _ready():\n queue_redraw()\n\nfunc _draw():\n draw_circle(Vector2(200, 180), 40, Color.CYAN)\n"} |
|
|
|
|
| def write_json(path, value): |
| path.parent.mkdir(parents=True, exist_ok=True) |
| path.write_text(json.dumps(value), encoding="utf-8") |
|
|
|
|
| def fixture_evaluation(root): |
| manifest = {"expected_count": 500, "run_seconds": 60, "smoke_test": False, "sample_indices": [0, 200, 400], |
| "generation": {"backend": "unit-test-only"}} |
| write_json(root / "manifest.json", manifest) |
| for index in range(500): |
| item = root / "items" / f"{index:04d}" |
| attempt = {"index": index, "prompt_id": f"test-{index}", "instruction": f"Distinct test fixture instruction number {index}."} |
| write_json(item / "attempt.json", attempt) |
| result = {"index": index, "prompt_id": attempt["prompt_id"], "compile_success": index in (0, 200), |
| "clean_run_60s": index in (0, 200), "run_seconds": 60, "project_sha256": demo.project_sha256(FILES), |
| "prompt_to_launch_seconds": 123.5 if index in (0, 200) else None} |
| write_json(item / "result.json", result) |
| if index in (0, 200, 400): |
| output = json.dumps({"files": FILES}) |
| generation = {"output": output, "output_sha256": hashlib.sha256(output.encode()).hexdigest(), "generation_seconds": 119.0} |
| write_json(item / "generation.json", generation) |
| video = root / "recordings" / "sample-000.mp4" |
| video.parent.mkdir() |
| video.write_bytes(b"unit test placeholder; ffprobe is mocked") |
| write_json(video.with_suffix(".recording.json"), {"capture_success": True, "duration_seconds": 15, |
| "fixed_timestep": True, "player_input": False, "sample_index": 0, "prompt_id": "test-0", |
| "evaluation_manifest_sha256": demo.file_hash(root / "manifest.json"), |
| "generation_sha256": demo.file_hash(root / "items/0000/generation.json"), |
| "project_sha256": demo.project_sha256(FILES), "output_sha256": demo.file_hash(video)}) |
| return video |
|
|
|
|
| class DemoEvidenceTests(unittest.TestCase): |
| def setUp(self): |
| self.temp = tempfile.TemporaryDirectory() |
| self.root = Path(self.temp.name) |
| self.video = fixture_evaluation(self.root) |
|
|
| def tearDown(self): |
| self.temp.cleanup() |
|
|
| @patch.object(demo, "probe_video", return_value={"duration_seconds": 15, "width": 960, "height": 540}) |
| def test_all_500_are_counted_and_fixed_failure_slots_retained(self, probe): |
| plan = demo.evaluation_plan(self.root) |
| self.assertEqual(plan["metrics"]["denominator"], 500) |
| self.assertEqual([sample["index"] for sample in plan["samples"]], [0, 200, 400]) |
| self.assertEqual(plan["metrics"]["compile_success_count"], 2) |
| self.assertEqual(plan["metrics"]["clean_run_60s_count"], 2) |
| self.assertEqual(plan["metrics"]["compile_success_rate"], .004) |
| self.assertEqual(plan["metrics"]["prompt_to_launch_median_seconds"], 123.5) |
| self.assertEqual(plan["metrics"]["under_60s_launch_count"], 0) |
| self.assertIsNotNone(plan["samples"][0]["capture"]) |
| self.assertIn("recording is unavailable", plan["samples"][1]["diagnostic"]) |
| self.assertIn("did not import", plan["samples"][2]["diagnostic"]) |
| self.assertTrue(plan["includes_failed_or_unrecorded_samples"]) |
| self.assertIn("119.0s", demo.latency_text(plan["samples"][0])) |
| self.assertIn("123.5s", demo.latency_text(plan["samples"][0])) |
|
|
| def test_partial_evaluation_is_rejected(self): |
| (self.root / "items/0499/result.json").unlink() |
| with self.assertRaisesRegex(ValueError, "exactly results"): |
| demo.evaluation_plan(self.root) |
|
|
| def test_no_captures_cannot_become_a_gameplay_demo(self): |
| self.video.unlink() |
| with self.assertRaisesRegex(ValueError, "At least one real recording"): |
| demo.evaluation_plan(self.root) |
|
|
| def test_wrong_generation_hash_is_rejected(self): |
| sidecar = self.video.with_suffix(".recording.json") |
| metadata = demo.read_json(sidecar) |
| metadata["generation_sha256"] = "different generation" |
| write_json(sidecar, metadata) |
| with self.assertRaisesRegex(ValueError, "generation_sha256"): |
| demo.evaluation_plan(self.root) |
|
|
| def test_video_content_hash_is_verified(self): |
| self.video.write_bytes(b"modified video") |
| with self.assertRaisesRegex(ValueError, "content hash"): |
| demo.evaluation_plan(self.root) |
|
|
| def test_infrastructure_failure_blocks_final_rates(self): |
| path = self.root / "items/0123/result.json" |
| value = demo.read_json(path); value["infrastructure_failure"] = True |
| write_json(path, value) |
| with self.assertRaisesRegex(ValueError, "Infrastructure failures"): |
| demo.evaluation_plan(self.root) |
|
|
| def test_summary_cannot_inflate_successes(self): |
| write_json(self.root / "summary.json", {"status": "complete", "recorded_prompts": 500, |
| "compile_success_count": 500, "clean_run_60s_count": 500}) |
| with self.assertRaisesRegex(ValueError, "summary disagrees"): |
| demo.evaluation_plan(self.root) |
|
|
| def test_changed_generation_cannot_use_original_capture(self): |
| path = self.root / "items/0000/generation.json" |
| generation = demo.read_json(path); generation["output"] += " " |
| write_json(path, generation) |
| with self.assertRaisesRegex(ValueError, "output hash mismatch"): |
| demo.evaluation_plan(self.root) |
|
|
|
|
| def preview_fixture_plan(video: Path) -> dict: |
| return {"kind": "unit-test-fixture-not-a-project-demo", "scaffold_preview": True, "metrics": None, |
| "includes_failed_or_unrecorded_samples": False, |
| "samples": [{"index": index, "prompt_id": f"test-{index}", "instruction": "This is a synthetic unit-test title card, not model performance evidence.", |
| "files": FILES, "generation_seconds": None, "prompt_to_launch_seconds": None, |
| "capture": {"path": str(video)}, "diagnostic": None} for index in range(3)]} |
|
|
|
|
| @unittest.skipUnless(importlib.util.find_spec("PIL"), "Pillow is an optional rendering dependency") |
| class DemoLayoutTests(unittest.TestCase): |
| def test_cards_fit_and_preview_watermark_is_permanent(self): |
| from PIL import Image |
| with tempfile.TemporaryDirectory() as temporary: |
| root = Path(temporary) |
| plan = preview_fixture_plan(root / "unused.mp4") |
| assets = demo.render_assets(plan, root / "assets") |
| self.assertEqual([asset["seconds"] for asset in assets], [8, 6, 14, 14, 14, 4]) |
| self.assertEqual(sum(asset["seconds"] for asset in assets), 60) |
| for asset in assets: |
| image = Image.open(asset.get("overlay", asset["path"])) |
| self.assertEqual(image.size, (1280, 720)) |
| red, green, blue = image.convert("RGB").getpixel((5, 710)) |
| self.assertGreater(red, green * 2) |
|
|
| @unittest.skipUnless(os.environ.get("VIBEGAME_VIDEO_INTEGRATION") == "1" and shutil.which("ffmpeg") and shutil.which("ffprobe"), |
| "Set VIBEGAME_VIDEO_INTEGRATION=1 for the CPU-only 60-second encoding test") |
| def test_actual_encoding_is_60_seconds_at_720p_and_30fps(self): |
| with tempfile.TemporaryDirectory(prefix="vibegame-video-test-") as temporary: |
| root = Path(temporary) |
| fixture = root / "test-color-source.mp4" |
| subprocess.run([shutil.which("ffmpeg"), "-nostdin", "-v", "error", "-f", "lavfi", "-i", "color=c=blue:s=960x540:r=30:d=15", |
| "-c:v", "libx264", "-pix_fmt", "yuv420p", str(fixture)], check=True) |
| result = demo.encode_video(preview_fixture_plan(fixture), root / "test-output.mp4") |
| self.assertAlmostEqual(result["seconds"], 60, places=2) |
| evidence = demo.read_json(Path(result["evidence"])) |
| self.assertEqual(evidence["probe"]["width"], 1280) |
| self.assertEqual(evidence["probe"]["height"], 720) |
| self.assertEqual(evidence["probe"]["r_frame_rate"], "30/1") |
| self.assertEqual(evidence["watermark"], demo.PREVIEW_WATERMARK) |
|
|
|
|
| if __name__ == "__main__": |
| unittest.main() |
|
|
