File size: 2,480 Bytes
f8dbde1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 | import json
from deg.extractbench import normalize_value, prepare, score_record
SCHEMA = {"name": "x", "description": "y", "schema_definition": {
"type": "object", "properties": {
"title": {"type": "string", "evaluation_config": "string_exact"},
"year": {"anyOf": [{"type": "string"}, {"type": "integer"}, {"type": "null"}]},
"kind": {"type": "string", "enum": ["a", "b"]},
"authors": {"type": "array", "items": {"type": "string"}},
"extra": {"type": "object", "additionalProperties": {"type": "string"}}}}}
def test_prepare_projects_gold_and_skips_image_only(tmp_path):
d = tmp_path / "academic" / "research"
(d / "pdf+gold").mkdir(parents=True)
(d / "research-schema.json").write_text(json.dumps(SCHEMA))
gold = {"title": "Deep Nets", "year": 1989, "kind": "a", "authors": ["Le Cun"],
"extra": {"k": "v"}}
words = " ".join(["word"] * 30)
for name, text in (("doc1", f"Deep Nets (1989) by Le Cun. {words}"), ("scan", "")):
(d / "pdf+gold" / f"{name}.pdf").write_bytes(b"%PDF")
(d / "pdf+gold" / f"{name}.txt").write_text(text)
(d / "pdf+gold" / f"{name}.gold.json").write_text(json.dumps(gold))
out = tmp_path / "records.jsonl"
report = prepare(tmp_path, out)
assert report["documents"] == 1 and report["skipped"] == {"image_only": 1}
info = report["schemas"]["research"]
assert info["compiles_as_is"] is False and info["normalized"] == "ok"
assert info["dropped"] == ["/extra"]
row = json.loads(out.read_text())
assert row["split"] == "eval" and row["uid"] == "extractbench:academic/research/doc1"
assert "extra" not in row["value"] and row["value"]["year"] == 1989
def test_score_record_exact_and_normalized():
schema = {"type": "object", "properties": {
"title": {"type": "string"}, "n": {"type": "number"},
"kind": {"enum": ["a", "b"]}, "tags": {"type": "array", "items": {"type": "string"}},
"gone": {"type": ["string", "null"]}}}
gold = {"title": "Deep Nets.", "n": 3, "kind": "a", "tags": ["x", "y"], "gone": None}
pred = {"title": "deep nets", "n": 3.0, "kind": "b", "tags": ["x"], "gone": None}
c = score_record(schema, gold, pred)
# title (normalized only), n (both), tags/0 (both), tags/1 (missing); kind is decide
assert c == {"extract_n": 4, "exact": 2, "normalized": 3, "count_n": 1, "count_ok": 0}
assert normalize_value("1,989") == normalize_value(1989) == 1989.0
|