import copy import importlib.util import json import os import platform from pathlib import Path import tempfile import unittest from unittest.mock import patch from vibegame.project import ProjectError, parse_project, validate_files, write_project from vibegame.validate import ERROR_PATTERN, sandbox_command, validate_project def minimal(): return { "project.godot": 'config_version=5\n[application]\nconfig/name="Test"\nrun/main_scene="res://main.tscn"\n[rendering]\nrenderer/rendering_method="gl_compatibility"\n', "main.tscn": '[gd_scene load_steps=2 format=3]\n[ext_resource type="Script" path="res://main.gd" id="1"]\n[node name="Main" type="Node2D"]\nscript = ExtResource("1")\n', "main.gd": 'extends Node2D\nfunc _ready():\n queue_redraw()\nfunc _draw():\n draw_circle(Vector2(40, 40), 20, Color.CORAL)\n', } class ProjectContractTest(unittest.TestCase): def test_valid_roundtrip(self): files = minimal() self.assertEqual(parse_project(json.dumps({"files": files})), files) def test_no_fences_or_extra_fields(self): for value in ['```json\n{}\n```', '{"files":{},"explanation":"hi"}', '[]', 'null']: with self.subTest(value=value), self.assertRaises(ProjectError): parse_project(value) def test_duplicate_keys(self): with self.assertRaises(ProjectError): parse_project('{"files":{},"files":{}}') def test_traversal_asset_paths(self): for name in ["../evil.gd", "/evil.gd", "sub/../../evil.gd", "sub//evil.gd", "sub/./evil.gd", "sub\\evil.gd", "photo.png", "addon.gdextension", "project.godot/evil.gd"]: files = minimal() files[name] = "extends Node" if name == "project.godot/evil.gd": # A source-file path may not also be used as a directory. with tempfile.TemporaryDirectory() as folder: with self.assertRaises((ProjectError, OSError)): write_project(files, Path(folder) / "out") else: with self.subTest(name=name), self.assertRaises(ProjectError): validate_files(files) def test_forbidden_capabilities(self): snippets = ['OS.execute("sh", [])', 'FileAccess.open("data", 1)', 'HTTPRequest.new()', 'ClassDB.instantiate("OS")', 'Expression.new()', '@tool', 'load("res://main.gd")', 'obj.call("unsafe")', 'get_tree().change_scene_to_file("res://main.tscn")', 'JavaScriptBridge.eval("x")', 'ResourceLoader.load("res://main.gd")'] for code in snippets: files = minimal() files["main.gd"] += code with self.subTest(code=code), self.assertRaises(ProjectError): validate_files(files) def test_remote_absolute_missing_resources(self): for resource in ['https://example.com/data', 'user://secret', '/etc/passwd', 'C:\\secret', 'res://../secret.gd', 'res://missing.gd', 'uid://123']: files = minimal() files["main.gd"] += f'\nvar x = "{resource}"\n' with self.subTest(resource=resource), self.assertRaises(ProjectError): validate_files(files) def test_external_scene_resources(self): files = minimal() files["main.tscn"] += '[ext_resource type="PackedScene" path="../outside.tscn" id="2"]' with self.assertRaises(ProjectError): validate_files(files) def test_autoload_rejected(self): files = minimal() files["project.godot"] += '\n[autoload]\nBackdoor="res://main.gd"\n' with self.assertRaises(ProjectError): validate_files(files) def test_wrong_main_scene(self): files = minimal() files["project.godot"] = files["project.godot"].replace("res://main.tscn", "res://other.tscn") with self.assertRaises(ProjectError): validate_files(files) def test_size_limits(self): for content in ["", "x" * 128001, "\x00"]: files = minimal() files["main.gd"] = content with self.assertRaises(ProjectError): validate_files(files) def test_missing_required_file(self): files = minimal() del files["main.gd"] with self.assertRaises(ProjectError): validate_files(files) def test_symlink_destination(self): with tempfile.TemporaryDirectory() as folder: root = Path(folder).resolve() (root / "real").mkdir() (root / "link").symlink_to(root / "real", target_is_directory=True) with self.assertRaises(ProjectError): write_project(minimal(), root / "link" / "out") self.assertEqual(list((root / "real").iterdir()), []) def test_existing_directory_not_modified(self): with tempfile.TemporaryDirectory() as folder: root = Path(folder).resolve() (root / "sentinel").write_text("preserve") with self.assertRaises(ProjectError): write_project(minimal(), root) self.assertEqual((root / "sentinel").read_text(), "preserve") class HarnessTest(unittest.TestCase): def test_error_markers(self): for line in ["SCRIPT ERROR: Parse Error: bad", "ERROR: failed to load", "Parse Error: bad", "Compile Error: bad", "Segmentation fault", "Failed to load script"]: self.assertIsNotNone(ERROR_PATTERN.search(line)) self.assertIsNone(ERROR_PATTERN.search("Godot Engine v4.5.1.stable")) def test_import_failure_prevents_run(self): with tempfile.TemporaryDirectory() as folder: root = Path(folder).resolve() binary = root / "godot" binary.write_bytes(b"fake runtime") with patch("vibegame.validate.run_sandboxed", return_value={"success": False, "errors": ["Parse Error"]}) as launch: result = validate_project(minimal(), root / "result", str(binary)) self.assertFalse(result["compile_success"]) self.assertFalse(result["clean_run"]) self.assertEqual(launch.call_count, 1) self.assertTrue((root / "result/validation.json").exists()) def test_short_smoke_never_becomes_60s_metric(self): with tempfile.TemporaryDirectory() as folder: root = Path(folder).resolve() binary = root / "godot" binary.write_bytes(b"fake runtime") with patch("vibegame.validate.run_sandboxed", return_value={"success": True}): result = validate_project(minimal(), root / "result", str(binary), 0.1) self.assertTrue(result["compile_success"]) self.assertTrue(result["clean_run"]) self.assertFalse(result["clean_run_60s"]) def test_evidence_cannot_be_overwritten(self): with tempfile.TemporaryDirectory() as folder: root = Path(folder).resolve() (root / "validation.json").write_text("existing") with self.assertRaises(FileExistsError): validate_project(minimal(), root, "/missing") self.assertEqual((root / "validation.json").read_text(), "existing") def test_macos_fails_before_execution(self): with tempfile.TemporaryDirectory() as folder, patch("platform.system", return_value="Darwin"): with self.assertRaisesRegex(RuntimeError, "macOS validation is disabled"): sandbox_command(["/bin/true"], Path(folder)) def test_no_bwrap_fails_closed(self): with tempfile.TemporaryDirectory() as folder, patch("platform.system", return_value="Linux"), patch("shutil.which", return_value=None): with self.assertRaisesRegex(RuntimeError, "fails closed"): sandbox_command(["/bin/true"], Path(folder)) class EvaluationEvidenceTest(unittest.TestCase): @classmethod def setUpClass(cls): import sys scripts = Path(__file__).resolve().parents[1] / "scripts" sys.path.insert(0, str(scripts)) import evaluate cls.evaluate = evaluate def test_zero_denominator_is_null(self): with tempfile.TemporaryDirectory() as folder: root = Path(folder).resolve() (root / "items").mkdir() result = self.evaluate.summarize(root, 0, 60) self.assertIsNone(result["compile_success_rate"]) self.assertIsNone(result["clean_run_60s_rate"]) def test_incomplete_cannot_report_final_rate(self): with tempfile.TemporaryDirectory() as folder: root = Path(folder).resolve() item = root / "items/0000" item.mkdir(parents=True) (item / "result.json").write_text(json.dumps({"compile_success": True, "clean_run_60s": True})) result = self.evaluate.summarize(root, 500, 60) self.assertEqual(result["compile_success_count"], 1) self.assertIsNone(result["compile_success_rate"]) self.assertEqual(result["pending_prompts"], 499) self.assertEqual(result["sample_indices"], [0, 200, 400]) def test_failure_remains_in_denominator(self): with tempfile.TemporaryDirectory() as folder: root = Path(folder).resolve() for index in range(2): item = root / "items" / f"{index:04d}" item.mkdir(parents=True) (item / "result.json").write_text(json.dumps({"compile_success": index == 0, "clean_run_60s": False})) result = self.evaluate.summarize(root, 2, 60) self.assertEqual(result["compile_success_rate"], 0.5) self.assertEqual(result["clean_run_60s_rate"], 0) def test_completed_failure_never_revalidated(self): with tempfile.TemporaryDirectory() as folder: item = Path(folder).resolve() (item / "check").mkdir() (item / "check/validation.json").write_text(json.dumps({"compile_success": False, "clean_run_60s": False})) with patch.object(self.evaluate, "validate_project") as validate: result = self.evaluate.validate_saved_project(minimal(), item, "/bin/false", 60) self.assertFalse(result["compile_success"]) validate.assert_not_called() def test_interrupted_validation_preserves_evidence(self): with tempfile.TemporaryDirectory() as folder: item = Path(folder).resolve() (item / "check").mkdir() (item / "check/import.log").write_text("existing evidence") with patch.object(self.evaluate, "validate_project", return_value={"compile_success": True, "clean_run_60s": True}) as validate: result = self.evaluate.validate_saved_project(minimal(), item, "/bin/true", 60) self.assertEqual(validate.call_args.args[1], item / "check-resume-001") self.assertEqual((item / "check/import.log").read_text(), "existing evidence") self.assertTrue((item / "check/interruption.json").exists()) self.assertEqual(result["interrupted_validation_attempts"], 1) @unittest.skipUnless(platform.system() == "Linux" and os.environ.get("VIBEGAME_GODOT"), "Opt-in Linux engine integration: set VIBEGAME_GODOT") class RealGodotIntegrationTest(unittest.TestCase): def check(self, files): with tempfile.TemporaryDirectory() as folder: return validate_project(files, Path(folder).resolve() / "evidence", os.environ["VIBEGAME_GODOT"], 1) def test_actual_clean_window_is_not_publication_metric(self): result = self.check(minimal()) self.assertTrue(result["compile_success"], result) self.assertTrue(result["clean_run"], result) self.assertFalse(result["clean_run_60s"]) self.assertGreaterEqual(result["runtime"]["wall_seconds"], 1) def test_actual_parse_error_is_rejected(self): files = minimal() files["main.gd"] += "\nfunc broken(\n" result = self.check(files) self.assertFalse(result["compile_success"], result) def test_actual_runtime_error_is_rejected(self): files = minimal() files["main.gd"] = "extends Node2D\nfunc _ready():\n var target: Node = null\n target.get_child(0)\n" result = self.check(files) self.assertTrue(result["compile_success"], result) self.assertFalse(result["clean_run"], result) self.assertTrue(result["runtime"]["errors"], result) def test_actual_early_exit_is_rejected(self): files = minimal() files["main.gd"] = "extends Node2D\nfunc _ready():\n get_tree().quit()\n" result = self.check(files) self.assertTrue(result["compile_success"], result) self.assertFalse(result["clean_run"], result) self.assertFalse(result["runtime"]["terminated_by_harness"], result) @unittest.skipUnless(os.environ.get("VIBEGAME_CAPTURE_TEST") == "1", "Opt-in Xvfb/ffmpeg recording: set VIBEGAME_CAPTURE_TEST=1") def test_actual_capture_has_verified_mp4(self): import sys sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) from record_game import record with tempfile.TemporaryDirectory() as folder: output = Path(folder).resolve() / "sample.mp4" result = record(minimal(), Path(os.environ["VIBEGAME_GODOT"]).resolve(), output, seconds=2, fps=10) self.assertTrue(result["capture_success"], result) self.assertGreater(output.stat().st_size, 1000) self.assertTrue(output.with_suffix(".recording.json").is_file()) if __name__ == "__main__": unittest.main()