| import copy |
| import importlib.util |
| import json |
| import os |
| import platform |
| from pathlib import Path |
| import tempfile |
| import unittest |
| from unittest.mock import patch |
|
|
| from vibegame.project import ProjectError, parse_project, validate_files, write_project |
| from vibegame.validate import ERROR_PATTERN, sandbox_command, validate_project |
|
|
|
|
| def minimal(): |
| return { |
| "project.godot": 'config_version=5\n[application]\nconfig/name="Test"\nrun/main_scene="res://main.tscn"\n[rendering]\nrenderer/rendering_method="gl_compatibility"\n', |
| "main.tscn": '[gd_scene load_steps=2 format=3]\n[ext_resource type="Script" path="res://main.gd" id="1"]\n[node name="Main" type="Node2D"]\nscript = ExtResource("1")\n', |
| "main.gd": 'extends Node2D\nfunc _ready():\n queue_redraw()\nfunc _draw():\n draw_circle(Vector2(40, 40), 20, Color.CORAL)\n', |
| } |
|
|
|
|
| class ProjectContractTest(unittest.TestCase): |
| def test_valid_roundtrip(self): |
| files = minimal() |
| self.assertEqual(parse_project(json.dumps({"files": files})), files) |
|
|
| def test_no_fences_or_extra_fields(self): |
| for value in ['```json\n{}\n```', '{"files":{},"explanation":"hi"}', '[]', 'null']: |
| with self.subTest(value=value), self.assertRaises(ProjectError): |
| parse_project(value) |
|
|
| def test_duplicate_keys(self): |
| with self.assertRaises(ProjectError): |
| parse_project('{"files":{},"files":{}}') |
|
|
| def test_traversal_asset_paths(self): |
| for name in ["../evil.gd", "/evil.gd", "sub/../../evil.gd", "sub//evil.gd", "sub/./evil.gd", "sub\\evil.gd", "photo.png", "addon.gdextension", "project.godot/evil.gd"]: |
| files = minimal() |
| files[name] = "extends Node" |
| if name == "project.godot/evil.gd": |
| |
| with tempfile.TemporaryDirectory() as folder: |
| with self.assertRaises((ProjectError, OSError)): |
| write_project(files, Path(folder) / "out") |
| else: |
| with self.subTest(name=name), self.assertRaises(ProjectError): |
| validate_files(files) |
|
|
| def test_forbidden_capabilities(self): |
| snippets = ['OS.execute("sh", [])', 'FileAccess.open("data", 1)', 'HTTPRequest.new()', 'ClassDB.instantiate("OS")', 'Expression.new()', '@tool', 'load("res://main.gd")', 'obj.call("unsafe")', 'get_tree().change_scene_to_file("res://main.tscn")', 'JavaScriptBridge.eval("x")', 'ResourceLoader.load("res://main.gd")'] |
| for code in snippets: |
| files = minimal() |
| files["main.gd"] += code |
| with self.subTest(code=code), self.assertRaises(ProjectError): |
| validate_files(files) |
|
|
| def test_remote_absolute_missing_resources(self): |
| for resource in ['https://example.com/data', 'user://secret', '/etc/passwd', 'C:\\secret', 'res://../secret.gd', 'res://missing.gd', 'uid://123']: |
| files = minimal() |
| files["main.gd"] += f'\nvar x = "{resource}"\n' |
| with self.subTest(resource=resource), self.assertRaises(ProjectError): |
| validate_files(files) |
|
|
| def test_external_scene_resources(self): |
| files = minimal() |
| files["main.tscn"] += '[ext_resource type="PackedScene" path="../outside.tscn" id="2"]' |
| with self.assertRaises(ProjectError): |
| validate_files(files) |
|
|
| def test_autoload_rejected(self): |
| files = minimal() |
| files["project.godot"] += '\n[autoload]\nBackdoor="res://main.gd"\n' |
| with self.assertRaises(ProjectError): |
| validate_files(files) |
|
|
| def test_wrong_main_scene(self): |
| files = minimal() |
| files["project.godot"] = files["project.godot"].replace("res://main.tscn", "res://other.tscn") |
| with self.assertRaises(ProjectError): |
| validate_files(files) |
|
|
| def test_size_limits(self): |
| for content in ["", "x" * 128001, "\x00"]: |
| files = minimal() |
| files["main.gd"] = content |
| with self.assertRaises(ProjectError): |
| validate_files(files) |
|
|
| def test_missing_required_file(self): |
| files = minimal() |
| del files["main.gd"] |
| with self.assertRaises(ProjectError): |
| validate_files(files) |
|
|
| def test_symlink_destination(self): |
| with tempfile.TemporaryDirectory() as folder: |
| root = Path(folder).resolve() |
| (root / "real").mkdir() |
| (root / "link").symlink_to(root / "real", target_is_directory=True) |
| with self.assertRaises(ProjectError): |
| write_project(minimal(), root / "link" / "out") |
| self.assertEqual(list((root / "real").iterdir()), []) |
|
|
| def test_existing_directory_not_modified(self): |
| with tempfile.TemporaryDirectory() as folder: |
| root = Path(folder).resolve() |
| (root / "sentinel").write_text("preserve") |
| with self.assertRaises(ProjectError): |
| write_project(minimal(), root) |
| self.assertEqual((root / "sentinel").read_text(), "preserve") |
|
|
|
|
| class HarnessTest(unittest.TestCase): |
| def test_error_markers(self): |
| for line in ["SCRIPT ERROR: Parse Error: bad", "ERROR: failed to load", "Parse Error: bad", "Compile Error: bad", "Segmentation fault", "Failed to load script"]: |
| self.assertIsNotNone(ERROR_PATTERN.search(line)) |
| self.assertIsNone(ERROR_PATTERN.search("Godot Engine v4.5.1.stable")) |
|
|
| def test_import_failure_prevents_run(self): |
| with tempfile.TemporaryDirectory() as folder: |
| root = Path(folder).resolve() |
| binary = root / "godot" |
| binary.write_bytes(b"fake runtime") |
| with patch("vibegame.validate.run_sandboxed", return_value={"success": False, "errors": ["Parse Error"]}) as launch: |
| result = validate_project(minimal(), root / "result", str(binary)) |
| self.assertFalse(result["compile_success"]) |
| self.assertFalse(result["clean_run"]) |
| self.assertEqual(launch.call_count, 1) |
| self.assertTrue((root / "result/validation.json").exists()) |
|
|
| def test_short_smoke_never_becomes_60s_metric(self): |
| with tempfile.TemporaryDirectory() as folder: |
| root = Path(folder).resolve() |
| binary = root / "godot" |
| binary.write_bytes(b"fake runtime") |
| with patch("vibegame.validate.run_sandboxed", return_value={"success": True}): |
| result = validate_project(minimal(), root / "result", str(binary), 0.1) |
| self.assertTrue(result["compile_success"]) |
| self.assertTrue(result["clean_run"]) |
| self.assertFalse(result["clean_run_60s"]) |
|
|
| def test_evidence_cannot_be_overwritten(self): |
| with tempfile.TemporaryDirectory() as folder: |
| root = Path(folder).resolve() |
| (root / "validation.json").write_text("existing") |
| with self.assertRaises(FileExistsError): |
| validate_project(minimal(), root, "/missing") |
| self.assertEqual((root / "validation.json").read_text(), "existing") |
|
|
| def test_macos_fails_before_execution(self): |
| with tempfile.TemporaryDirectory() as folder, patch("platform.system", return_value="Darwin"): |
| with self.assertRaisesRegex(RuntimeError, "macOS validation is disabled"): |
| sandbox_command(["/bin/true"], Path(folder)) |
|
|
| def test_no_bwrap_fails_closed(self): |
| with tempfile.TemporaryDirectory() as folder, patch("platform.system", return_value="Linux"), patch("shutil.which", return_value=None): |
| with self.assertRaisesRegex(RuntimeError, "fails closed"): |
| sandbox_command(["/bin/true"], Path(folder)) |
|
|
|
|
| class EvaluationEvidenceTest(unittest.TestCase): |
| @classmethod |
| def setUpClass(cls): |
| import sys |
| scripts = Path(__file__).resolve().parents[1] / "scripts" |
| sys.path.insert(0, str(scripts)) |
| import evaluate |
| cls.evaluate = evaluate |
|
|
| def test_zero_denominator_is_null(self): |
| with tempfile.TemporaryDirectory() as folder: |
| root = Path(folder).resolve() |
| (root / "items").mkdir() |
| result = self.evaluate.summarize(root, 0, 60) |
| self.assertIsNone(result["compile_success_rate"]) |
| self.assertIsNone(result["clean_run_60s_rate"]) |
|
|
| def test_incomplete_cannot_report_final_rate(self): |
| with tempfile.TemporaryDirectory() as folder: |
| root = Path(folder).resolve() |
| item = root / "items/0000" |
| item.mkdir(parents=True) |
| (item / "result.json").write_text(json.dumps({"compile_success": True, "clean_run_60s": True})) |
| result = self.evaluate.summarize(root, 500, 60) |
| self.assertEqual(result["compile_success_count"], 1) |
| self.assertIsNone(result["compile_success_rate"]) |
| self.assertEqual(result["pending_prompts"], 499) |
| self.assertEqual(result["sample_indices"], [0, 200, 400]) |
|
|
| def test_failure_remains_in_denominator(self): |
| with tempfile.TemporaryDirectory() as folder: |
| root = Path(folder).resolve() |
| for index in range(2): |
| item = root / "items" / f"{index:04d}" |
| item.mkdir(parents=True) |
| (item / "result.json").write_text(json.dumps({"compile_success": index == 0, "clean_run_60s": False})) |
| result = self.evaluate.summarize(root, 2, 60) |
| self.assertEqual(result["compile_success_rate"], 0.5) |
| self.assertEqual(result["clean_run_60s_rate"], 0) |
|
|
| def test_completed_failure_never_revalidated(self): |
| with tempfile.TemporaryDirectory() as folder: |
| item = Path(folder).resolve() |
| (item / "check").mkdir() |
| (item / "check/validation.json").write_text(json.dumps({"compile_success": False, "clean_run_60s": False})) |
| with patch.object(self.evaluate, "validate_project") as validate: |
| result = self.evaluate.validate_saved_project(minimal(), item, "/bin/false", 60) |
| self.assertFalse(result["compile_success"]) |
| validate.assert_not_called() |
|
|
| def test_interrupted_validation_preserves_evidence(self): |
| with tempfile.TemporaryDirectory() as folder: |
| item = Path(folder).resolve() |
| (item / "check").mkdir() |
| (item / "check/import.log").write_text("existing evidence") |
| with patch.object(self.evaluate, "validate_project", return_value={"compile_success": True, "clean_run_60s": True}) as validate: |
| result = self.evaluate.validate_saved_project(minimal(), item, "/bin/true", 60) |
| self.assertEqual(validate.call_args.args[1], item / "check-resume-001") |
| self.assertEqual((item / "check/import.log").read_text(), "existing evidence") |
| self.assertTrue((item / "check/interruption.json").exists()) |
| self.assertEqual(result["interrupted_validation_attempts"], 1) |
|
|
|
|
| @unittest.skipUnless(platform.system() == "Linux" and os.environ.get("VIBEGAME_GODOT"), "Opt-in Linux engine integration: set VIBEGAME_GODOT") |
| class RealGodotIntegrationTest(unittest.TestCase): |
| def check(self, files): |
| with tempfile.TemporaryDirectory() as folder: |
| return validate_project(files, Path(folder).resolve() / "evidence", os.environ["VIBEGAME_GODOT"], 1) |
|
|
| def test_actual_clean_window_is_not_publication_metric(self): |
| result = self.check(minimal()) |
| self.assertTrue(result["compile_success"], result) |
| self.assertTrue(result["clean_run"], result) |
| self.assertFalse(result["clean_run_60s"]) |
| self.assertGreaterEqual(result["runtime"]["wall_seconds"], 1) |
|
|
| def test_actual_parse_error_is_rejected(self): |
| files = minimal() |
| files["main.gd"] += "\nfunc broken(\n" |
| result = self.check(files) |
| self.assertFalse(result["compile_success"], result) |
|
|
| def test_actual_runtime_error_is_rejected(self): |
| files = minimal() |
| files["main.gd"] = "extends Node2D\nfunc _ready():\n var target: Node = null\n target.get_child(0)\n" |
| result = self.check(files) |
| self.assertTrue(result["compile_success"], result) |
| self.assertFalse(result["clean_run"], result) |
| self.assertTrue(result["runtime"]["errors"], result) |
|
|
| def test_actual_early_exit_is_rejected(self): |
| files = minimal() |
| files["main.gd"] = "extends Node2D\nfunc _ready():\n get_tree().quit()\n" |
| result = self.check(files) |
| self.assertTrue(result["compile_success"], result) |
| self.assertFalse(result["clean_run"], result) |
| self.assertFalse(result["runtime"]["terminated_by_harness"], result) |
|
|
| @unittest.skipUnless(os.environ.get("VIBEGAME_CAPTURE_TEST") == "1", "Opt-in Xvfb/ffmpeg recording: set VIBEGAME_CAPTURE_TEST=1") |
| def test_actual_capture_has_verified_mp4(self): |
| import sys |
| sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) |
| from record_game import record |
| with tempfile.TemporaryDirectory() as folder: |
| output = Path(folder).resolve() / "sample.mp4" |
| result = record(minimal(), Path(os.environ["VIBEGAME_GODOT"]).resolve(), output, seconds=2, fps=10) |
| self.assertTrue(result["capture_success"], result) |
| self.assertGreater(output.stat().st_size, 1000) |
| self.assertTrue(output.with_suffix(".recording.json").is_file()) |
|
|
|
|
| if __name__ == "__main__": |
| unittest.main() |
|
|
