Buckets:
| from argparse import Namespace | |
| from pathlib import Path | |
| import pytest | |
| from zebra_baselines.common import evaluation_prompt, fixed_records | |
| from zebra_baselines.anole import AnoleRunner | |
| from zebra_baselines.eval_zebra import PAPER_CONFIGS, _score, select_items | |
| ROOT = Path(__file__).resolve().parents[2] | |
| def test_fixed_cvbench_selection_is_exactly_twenty() -> None: | |
| records = fixed_records( | |
| ROOT / "data/CV-Bench", | |
| ROOT / "experiments/sites/zebra_rollout_viewer/scripts/cvbench_samples.json", | |
| ) | |
| assert len(records) == 20 | |
| assert len({record["idx"] for record in records}) == 20 | |
| assert all(evaluation_prompt(record).endswith("directly.") for record in records) | |
| def test_zebra_settings_are_reproducible_and_disjoint_in_meaning() -> None: | |
| args = Namespace( | |
| cache_root=ROOT / "data/Zebra-CoT-UniAR-960", | |
| ours18_root=ROOT / "experiments/output/uniar-sft/zebra-cot-full/unversioned/runs/eval/20260821-111254_zebra_native_context_test18", | |
| ) | |
| if not (args.cache_root / "manifest.jsonl").is_file(): | |
| pytest.skip("full Zebra packed cache is not materialized on this cluster") | |
| ours = select_items(Namespace(setting="ours_grouped_test18", **vars(args))) | |
| proxy = select_items(Namespace(setting="paper_domain_proxy6", **vars(args))) | |
| assert len(ours) == 18 | |
| assert len(proxy) == len(PAPER_CONFIGS) | |
| assert [item["config"] for item in proxy] == list(PAPER_CONFIGS) | |
| assert len({item["sample_id"] for item in proxy}) == len(proxy) | |
| def test_zebra_score_requires_explicit_final_answer() -> None: | |
| unfinished = _score("<think>option D is likely correct</think>", "D") | |
| finished = _score("<answer>Final Answer: D</answer>", "D") | |
| assert unfinished["raw_correct"] is True | |
| assert unfinished["has_final_answer"] is False | |
| assert unfinished["correct"] is False | |
| assert finished["correct"] is True | |
| def test_anole_prompt_has_one_explicit_slot_per_observation() -> None: | |
| assert AnoleRunner._model_prompt("Question: what?", 0) == "Question: what?" | |
| assert AnoleRunner._model_prompt("Question: what?", 2) == "Question: what?<image><image>" | |
Xet Storage Details
- Size:
- 2.14 kB
- Xet hash:
- cd0d9b54d0b05ae93e274a4ef3f5c47e951bf9227e450331c099cc8965e2c4c8
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.