Spaces:
Running
Running
Download tests/test_annotator.py from Melikshah/geoql-env: direct link, hf CLI and curl.
- Browser
- Download file 9.27 kB
-
https://huggingface.co/spaces/Melikshah/geoql-env/resolve/main/tests/test_annotator.py
- Command line
-
hf download hf://spaces/Melikshah/geoql-env/tests/test_annotator.py
-
curl -L -o test_annotator.py https://huggingface.co/spaces/Melikshah/geoql-env/resolve/main/tests/test_annotator.py
9.27 kB
| import dataclasses | |
| import json | |
| import zipfile | |
| from pathlib import Path | |
| import pytest | |
| from geoql_env import annotator, cli, prompt, tasks | |
| from geoql_env.tasks import Interpretation, Pair, Task | |
| CAFES = tasks.load(str(Path(__file__).parent / "fixtures" / "tasks" / "fx-cafes.yaml")) | |
| TASK = Task("t", "Which cafes are in Monaco?", "mc-monaco", "set", | |
| (Interpretation("A", '{{geocodeArea:"Monaco"}}->.a; nwr["amenity"="cafe"](area.a); out;', "cafes"),)) | |
| def test_the_blind_solve_shows_the_single_shot_prompt_and_nothing_of_the_gold(): | |
| chosen = tasks.load_all() | |
| parts = annotator.solve_parts(chosen) | |
| text = "\n".join(parts) | |
| assert all(part.count(prompt.CONVENTIONS) == 1 for part in parts) | |
| assert all(f"Task {t.id}\n{prompt.user(t)}\n" in text for t in chosen) | |
| gold = [x.strip() for t in chosen for x in (t.notes, *(i.program for i in t.interpretations), | |
| *(i.rationale for i in t.interpretations), | |
| *(p for _, p in t.validation.rewrites + t.validation.mutants))] | |
| assert not [x for x in gold if x and x in text] | |
| assert not [w for t in chosen for w in t.witnesses if w in text] | |
| def test_parts_hold_at_most_ten_tasks_and_keep_linked_tasks_apart(): | |
| many = [dataclasses.replace(TASK, id=f"t{i:02}", request=f"Request {i}?") for i in range(23)] | |
| many[1] = dataclasses.replace(many[1], pair=Pair("t00", "changed")) | |
| many[3] = dataclasses.replace(many[3], intent="t02") | |
| parts = annotator.solve_parts(many) | |
| assert [p.count("\nTask t") for p in parts] == [10, 10, 3] | |
| assert not any(f"Task {a}\n" in p and f"Task {b}\n" in p for p in parts for a, b in (("t00", "t01"), | |
| ("t02", "t03"))) | |
| with_pairs = annotator.solve_parts(tasks.load_all()) | |
| assert not any(f"Task {t.id}\n" in p and f"Task {t.pair.task}\n" in p | |
| for p in with_pairs for t in tasks.load_all() if t.pair) | |
| def test_replies_parse_from_json_or_a_copied_reply(tmp_path): | |
| a, b, c = (dataclasses.replace(TASK, id=i) for i in "abc") | |
| (tmp_path / "solve-reply-1.json").write_text('[{"task": "a", "query": "node(1); out;"}]') | |
| (tmp_path / "solve-reply-2.json").write_text('Here they are.\n\n```json\n[{"task": "b", "query": "way(2);\\nout;", ' | |
| '"note": "kept"}]\n```\n') | |
| files = sorted(tmp_path.glob("solve-reply-*.json")) | |
| assert annotator.replies(files, [a, b], annotator._query) == {"a": "node(1); out;", "b": "way(2);\nout;"} | |
| with pytest.raises(ValueError, match="unknown tasks: b; no reply for: c"): | |
| annotator.replies(files, [a, c], annotator._query) | |
| (tmp_path / "solve-reply-3.json").write_text('[{"task": "a", "query": "node(3); out;"}]') | |
| with pytest.raises(ValueError, match="a is answered twice"): | |
| annotator.replies(sorted(tmp_path.glob("solve-reply-*.json")), [a, b], annotator._query) | |
| (tmp_path / "bad.json").write_text('```json\n[{"task": "a", "query": 1}]\n```') | |
| with pytest.raises(ValueError, match="bad.json: a: the query must be a string"): | |
| annotator.replies([tmp_path / "bad.json"], [a], annotator._query) | |
| (tmp_path / "bad.json").write_text('{"task": "a", "query": "x"}') | |
| with pytest.raises(ValueError, match="no JSON list"): | |
| annotator.replies([tmp_path / "bad.json"], [a], annotator._query) | |
| def test_verdicts_outside_the_rubric_are_rejected(entry, error): | |
| with pytest.raises(ValueError, match=error): | |
| annotator._verdict({"task": "a", **entry}, "a") | |
| def test_verdicts_are_kept_with_their_reasons_and_proposal(): | |
| assert annotator._verdict({"task": "a", "verdict": "add_reading", "drop_reason": "", "reasons": ["One.", "Two."], | |
| "proposal": " way(1); out; "}, "a") == { | |
| "verdict": "add_reading", "drop_reason": None, "reasons": "One. Two.", "proposal": "way(1); out;"} | |
| assert annotator._verdict({"task": "a", "verdict": "drop", "drop_reason": "duplicate", "reasons": "Same as b.", | |
| "proposal": None}, "a")["drop_reason"] == "duplicate" | |
| def test_review_lines_come_from_the_batch_folder(tmp_path): | |
| worlds = {"w": {"equal": True, "got": 3, "expected": 3}, | |
| "t-w1": {"equal": False, "got": 4, "expected": 3, "extra": {"objects": 1, "groups": []}, | |
| "missing": {"objects": 0, "groups": []}}} | |
| graded = {"query": "nwr[amenity=cafe];\nout;", "category": "wrong_witness", "reading": "A", "message": "", | |
| "worlds": worlds} | |
| (tmp_path / annotator.GRADED).write_text(json.dumps({"model": "M (max)", "tasks": {"t": graded}})) | |
| found = annotator.load(tmp_path, [TASK]) | |
| assert found["t"]["verdict"] is None and annotator.sheet(TASK, found["t"]) == [ | |
| "- Blind solve (M (max)): wrong_witness, closest reading A; differs on t-w1 4 vs 3 (1 extra, 0 missing): " | |
| "`nwr[amenity=cafe]; out;`"] | |
| (tmp_path / "critique-reply-1.json").write_text(json.dumps([{"task": "t", "verdict": "reword", "drop_reason": None, | |
| "reasons": "Unclear.", "proposal": "Which cafés?"}])) | |
| assert annotator.sheet(TASK, annotator.load(tmp_path, [TASK])["t"])[1:] == [ | |
| "- Verdict (M (max)): reword: Unclear.", '- Proposal (M (max)): "Which cafés?"'] | |
| with pytest.raises(ValueError, match="no blind solve of u"): | |
| annotator.load(tmp_path, [TASK, dataclasses.replace(TASK, id="u")]) | |
| def test_the_critique_names_minimal_pair_partners(): | |
| a, b = dataclasses.replace(TASK, id="a"), dataclasses.replace(TASK, id="b", pair=Pair("a", "same")) | |
| solved = {"query": "nwr(1); out;", "category": "pass", "reading": "A", "message": "", "worlds": {}} | |
| text = annotator.critique_parts([a, b, dataclasses.replace(TASK, id="c")], {i: solved for i in "abc"})[0] | |
| assert text.count("form a minimal pair by design") == 2 and "should be the same" in text | |
| assert "Task a and task b form" in text and "Task b and task a form" in text and "Task c and" not in text | |
| def test_solve_command_writes_a_reproducible_zip(tmp_path, monkeypatch, capsys): | |
| monkeypatch.setattr(tasks, "load_all", lambda: [TASK]) | |
| folder = tmp_path / "batch" | |
| assert cli.main(["annotator", "solve", "-o", str(folder)]) == 0 | |
| assert capsys.readouterr().out == f"wrote {folder / 'solve.zip'}: 1 task in 1 part\n" | |
| with zipfile.ZipFile(folder / "solve.zip") as z: | |
| assert z.namelist() == ["HOW-TO.txt", "part-1.txt"] | |
| assert f"geoql-env annotator critique -o {folder}" in z.read("HOW-TO.txt").decode() | |
| assert prompt.user(TASK) in z.read("part-1.txt").decode() | |
| first = (folder / "solve.zip").read_bytes() | |
| cli.main(["annotator", "solve", "-o", str(folder)]) | |
| assert (folder / "solve.zip").read_bytes() == first | |
| assert cli.main(["annotator", "critique", "-o", str(folder)]) == 1 | |
| assert "model.txt" in capsys.readouterr().err | |
| def test_a_blind_query_is_graded_on_every_world_with_its_differences(cafes_world): | |
| nodes = '{{geocodeArea:"Testville"}}->.a; node["amenity"="cafe"](area.a); out;' | |
| graded = annotator.grade([CAFES], {"fx-cafes": f"```overpassql\n{nodes}\n```"}, cafes_world)["fx-cafes"] | |
| assert graded["query"] == nodes and graded["category"] == "wrong_real" and graded["reading"] == "A" | |
| real = graded["worlds"][CAFES.world] | |
| assert real["equal"] is False and real["extra"] == {"objects": 0, "groups": []} | |
| assert real["missing"] == {"objects": 1, "groups": [ | |
| {"objects": 1, "example": "way 103", "tags": {"amenity": "cafe", "building": "yes", "name": "Way Cafe"}, | |
| "types": {"way": 1}}]} | |
| assert set(graded["worlds"]) == set(CAFES.worlds) | |
| text = annotator.critique_parts([CAFES], {"fx-cafes": graded})[0] | |
| assert "It fails: on the real world its answer equals no reading's. Closest is reading A:" in text | |
| assert ("On the real world: your query prints 2 objects, reading A 3.\nOnly yours: 0 objects.\n" | |
| "Only reading A's: 1 object:\n- 1 object (1 way) tagged " | |
| '["amenity"="cafe"]["building"="yes"]["name"="Way Cafe"], e.g. way 103\n') in text | |
| assert "On witness world w1:" in text and CAFES.interpretations[0].program.strip() in text | |
| passed = annotator.grade([CAFES], {"fx-cafes": CAFES.interpretations[0].program}, cafes_world)["fx-cafes"] | |
| assert passed["category"] == "pass" and all(s["equal"] for s in passed["worlds"].values()) | |