| """Public prompts and deterministic answer-slot seeds for UI-TestJev inference.""" | |
| import hashlib | |
| import json | |
| from pathlib import Path | |
| MODEL_ID = "google/diffusiongemma-26B-A4B-it" | |
| MODEL_REVISION = "f7f5b7f5fa82ffc52addd066915886d497f5517b" | |
| SEED = 3407 | |
| ISSUES = { | |
| "text_clipping": "Required text glyphs are clipped.", | |
| "contrast": "Text violates the stated minimum contrast requirement.", | |
| "required_content": "Required visible content is missing.", | |
| "occlusion": "The specified target is obstructed.", | |
| "image_aspect": "An image's aspect ratio differs beyond the allowed tolerance.", | |
| "target_size": "A control violates the stated minimum rendered dimensions.", | |
| } | |
| SCRY_LABELS = ("Color/Background", "Extra Element", "Icon/Nav", "Image/Asset", | |
| "Missing Content", "Selected/Disabled State", "Separator", "Shape/Size", | |
| "Spacing/Layout", "Typography", "badimage", "other") | |
| FIELDS = { | |
| "requirement_check": ("requirement_type", "requirement", "reference_target", "viewport"), | |
| "regression_search": ("viewport",), | |
| "keyboard_access": ("target", "observed_keyboard_trace"), | |
| "known_ui_differences": (), | |
| } | |
| INSTRUCTIONS = { | |
| "requirement_check": "Check ONLY the specified target and the single stated requirement. A visible styling change is not automatically a failure. Pass when that requirement remains satisfied; ignore other elements and other requirement categories. If evidence cannot establish the requirement, select unknown. For a failure select only the violated requirement category.", | |
| "regression_search": "Find the newly introduced regression across the full screen relative to the approved reference. Required visible content must remain present; glyphs must not be clipped; image aspect ratios must stay within 3% of the reference; controls previously at least 24 by 24 CSS pixels must remain above that minimum; text meeting 4.5:1 contrast must keep meeting it. Minor opacity changes still meeting contrast, outlines, one-pixel shifts, and aspect changes within tolerance are allowed. Ignore unchanged pre-existing defects. Select pass if these contracts remain satisfied, unknown if evidence is insufficient, or the violated category. This benchmark contains at most one introduced violation category. Do not infer a location from any supplied answer options.", | |
| "keyboard_access": "Can keyboard users reach the specified control using Tab in one complete focus cycle? Use the observed trace if supplied. A screenshot alone does not establish reachability: select unknown when the trace is absent. Select pass if reachable, fail if unreachable in a complete cycle.", | |
| } | |
| def digest(path): | |
| h = hashlib.sha256() | |
| with open(path, "rb") as f: | |
| for b in iter(lambda: f.read(4 * 1024 * 1024), b""): | |
| h.update(b) | |
| return h.hexdigest() | |
| def stable_seed(*values): | |
| return int(hashlib.sha256(json.dumps(values, sort_keys=True).encode()).hexdigest()[:15], 16) | |
| def options(task, public=None): | |
| if task == "keyboard_access": | |
| return [("pass", "Reachable"), ("fail", "Unreachable"), ("unknown", "Insufficient evidence")] | |
| if task == "requirement_check" and public and "requirement_type" in public: | |
| category = public["requirement_type"] | |
| if category not in ISSUES: raise ValueError("Unsupported public requirement type") | |
| return [("pass", "Stated requirement remains satisfied"), ("unknown", "Insufficient evidence"), (category, ISSUES[category])] | |
| issues = [k for k in ISSUES if task != "regression_search" or k != "occlusion"] | |
| return [("pass", "No violation of the checked contracts"), ("unknown", "Insufficient evidence")] + [(k, ISSUES[k]) for k in issues] | |
| def prompt_spec(public): | |
| """Only a public input is accepted. IDs/paths/provenance/targets never enter text.""" | |
| task = public["task"] | |
| state = {k: public[k] for k in FIELDS[task] if k in public} | |
| state["image_order"] = [dict(index=i+1, role=im["role"], **({"full_image_crop_xyxy": im["full_image_crop_xyxy"]} if "full_image_crop_xyxy" in im else {})) for i, im in enumerate(public["images"])] | |
| if task == "known_ui_differences": | |
| questions = [{"instruction": f"Is there a visible difference between the reference and implementation screens in category {tag!r}? Answer yes, no, or unknown. Images may have different dimensions.", "tag": tag, | |
| "options": [("yes", "Visible difference in this category"), ("no", "No visible difference in this category"), ("unknown", "Insufficient evidence")]} for tag in SCRY_LABELS] | |
| else: | |
| opts = options(task, public) | |
| questions = [{"instruction": INSTRUCTIONS[task], "options": opts}] | |
| system = "Answer a fixed set of questions about the supplied UI evidence. Each question lists its allowed answers. Reply with exactly one letter per question.\n" | |
| for i, q in enumerate(questions): | |
| system += f"\nQuestion q{i+1}: {q['instruction']}\n" | |
| for j, (name, description) in enumerate(q["options"]): | |
| system += f" {chr(65+j)}: {name} ({description})\n" | |
| system += "\nReply as lines in the form q1: A, q2: B, using only the allowed letters." | |
| return {"system": system, "state": json.dumps(state, ensure_ascii=False, sort_keys=True, separators=(",", ":")), "questions": questions} | |
| def public_identity(public, root): | |
| """Hash actual public prompt + ordered image bytes. No ID, label, path or oracle.""" | |
| spec = prompt_spec(public) | |
| content = [spec["system"], json.loads(spec["state"]), | |
| [digest(Path(root)/im["path"]) for im in public["images"]]] | |
| return hashlib.sha256(json.dumps(content, sort_keys=True, ensure_ascii=False).encode()).hexdigest() | |