Buckets:
| import json | |
| import math | |
| import torch | |
| from PIL import Image | |
| from zebra_sft.bsq_codes import pack_visual_codes | |
| from zebra_sft.visual_quality import ( | |
| blank_image_metrics, | |
| geometry_metrics, | |
| native_code_metrics, | |
| summarize_samples, | |
| target_code_images, | |
| ) | |
| from zebra_sft.summarize_visual_quality import summarize | |
| def test_target_code_images_preserve_generated_state_order() -> None: | |
| codes = torch.arange(6 * 8).reshape(6, 1, 1, 8).remainder(2).to(torch.uint8) | |
| packed, shape = pack_visual_codes(codes) | |
| item = { | |
| "sample_id": "fixture", | |
| "visual_codes_packed": packed, | |
| "visual_codes_shape": shape, | |
| "generated_flat_mask": torch.tensor([False, True, True, False, True, True]), | |
| "generated_image_token_counts": torch.tensor([2, 2]), | |
| } | |
| images = target_code_images(item) | |
| assert len(images) == 2 | |
| assert torch.equal(images[0], codes[1:3]) | |
| assert torch.equal(images[1], codes[4:6]) | |
| def test_native_metrics_require_exact_code_topology() -> None: | |
| target = torch.zeros(2, 1, 1, 8, dtype=torch.uint8) | |
| generated = target.clone().unsqueeze(0) | |
| generated[0, 0, 0, 0, :2] = 1 | |
| metrics = native_code_metrics(generated, target) | |
| assert metrics["bits"] == 16 | |
| assert metrics["hamming_bits"] == 2 | |
| assert metrics["bit_accuracy"] == 14 / 16 | |
| assert metrics["token_agreement"] == 0.5 | |
| assert native_code_metrics(generated[:, :1], target) is None | |
| def test_geometry_gate_rejects_large_aspect_warp() -> None: | |
| near = geometry_metrics((512, 480), (512, 512), aspect_tolerance=0.10) | |
| far = geometry_metrics((512, 96), (320, 512), aspect_tolerance=0.10) | |
| assert near["geometry_comparable"] | |
| assert not near["exact_resolution"] | |
| assert not far["geometry_comparable"] | |
| assert far["aspect_ratio_relative_error"] > 1 | |
| def test_blank_metric_uses_luminance_variation() -> None: | |
| blank = Image.new("RGB", (32, 32), "white") | |
| varied = blank.copy() | |
| varied.putpixel((0, 0), (0, 0, 0)) | |
| assert blank_image_metrics(blank)["blank"] | |
| assert not blank_image_metrics(varied, std_threshold=0.005)["blank"] | |
| def test_summary_is_trajectory_balanced_before_image_aggregation() -> None: | |
| def pair(value): | |
| return { | |
| "geometry": { | |
| "exact_resolution": True, | |
| "geometry_comparable": True, | |
| }, | |
| "native": {"bit_accuracy": value}, | |
| "pixel_perceptual": {"ssim": value}, | |
| "semantic": {"dinov2_cosine": value}, | |
| } | |
| samples = [ | |
| { | |
| "sample_id": "one", | |
| "expected_images": 1, | |
| "generated_images": 1, | |
| "count_match": True, | |
| "generated_quality": [{"blank": False, "luma_std": 0.2}], | |
| "paired_images": [pair(0.0)], | |
| "adjacent_generated": [], | |
| }, | |
| { | |
| "sample_id": "four", | |
| "expected_images": 4, | |
| "generated_images": 4, | |
| "count_match": True, | |
| "generated_quality": [ | |
| {"blank": False, "luma_std": 0.2} for _ in range(4) | |
| ], | |
| "paired_images": [pair(1.0) for _ in range(4)], | |
| "adjacent_generated": [ | |
| {"near_duplicate": False, "dinov2_cosine": 0.5} | |
| for _ in range(3) | |
| ], | |
| }, | |
| ] | |
| summary = summarize_samples(samples, eval_run="fixture") | |
| metric = summary["paired_metrics"]["native.bit_accuracy"] | |
| assert math.isclose(metric["trajectory_balanced"]["mean"], 0.5) | |
| assert math.isclose(metric["image_weighted"]["mean"], 0.8) | |
| assert summary["paired_images"] == 5 | |
| assert summary["blank_fraction"] == 0 | |
| def test_campaign_summary_flattens_completed_children(tmp_path) -> None: | |
| child = tmp_path / "eval-a" | |
| child.mkdir() | |
| summary = { | |
| "eval_run": "/eval/a", | |
| "samples": 1, | |
| "count_matched_samples": 1, | |
| "generated_images": 1, | |
| "paired_images": 1, | |
| "exact_resolution_pairs": 1, | |
| "geometry_comparable_pairs": 1, | |
| "blank_fraction": 0.0, | |
| "near_duplicate_fraction": None, | |
| "paired_metrics": { | |
| "native.bit_accuracy": { | |
| "trajectory_balanced": {"mean": 0.75}, | |
| "image_weighted": {"mean": 0.75}, | |
| } | |
| }, | |
| } | |
| (child / "summary.json").write_text(json.dumps(summary), encoding="utf-8") | |
| (child / "COMPLETE").write_text("complete\n", encoding="utf-8") | |
| result = summarize(tmp_path) | |
| assert result["completed_runs"] == 1 | |
| assert result["rows"][0]["native.bit_accuracy"] == 0.75 | |
| assert (tmp_path / "comparison.csv").exists() | |
Xet Storage Details
- Size:
- 4.64 kB
- Xet hash:
- 779fa47b6733affe6768fef3b26c66081a57c7516cee5499957d932349b9b51f
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.