| import json |
|
|
| from deg.extractbench import normalize_value, prepare, score_record |
|
|
| SCHEMA = {"name": "x", "description": "y", "schema_definition": { |
| "type": "object", "properties": { |
| "title": {"type": "string", "evaluation_config": "string_exact"}, |
| "year": {"anyOf": [{"type": "string"}, {"type": "integer"}, {"type": "null"}]}, |
| "kind": {"type": "string", "enum": ["a", "b"]}, |
| "authors": {"type": "array", "items": {"type": "string"}}, |
| "extra": {"type": "object", "additionalProperties": {"type": "string"}}}}} |
|
|
|
|
| def test_prepare_projects_gold_and_skips_image_only(tmp_path): |
| d = tmp_path / "academic" / "research" |
| (d / "pdf+gold").mkdir(parents=True) |
| (d / "research-schema.json").write_text(json.dumps(SCHEMA)) |
| gold = {"title": "Deep Nets", "year": 1989, "kind": "a", "authors": ["Le Cun"], |
| "extra": {"k": "v"}} |
| words = " ".join(["word"] * 30) |
| for name, text in (("doc1", f"Deep Nets (1989) by Le Cun. {words}"), ("scan", "")): |
| (d / "pdf+gold" / f"{name}.pdf").write_bytes(b"%PDF") |
| (d / "pdf+gold" / f"{name}.txt").write_text(text) |
| (d / "pdf+gold" / f"{name}.gold.json").write_text(json.dumps(gold)) |
| out = tmp_path / "records.jsonl" |
| report = prepare(tmp_path, out) |
| assert report["documents"] == 1 and report["skipped"] == {"image_only": 1} |
| info = report["schemas"]["research"] |
| assert info["compiles_as_is"] is False and info["normalized"] == "ok" |
| assert info["dropped"] == ["/extra"] |
| row = json.loads(out.read_text()) |
| assert row["split"] == "eval" and row["uid"] == "extractbench:academic/research/doc1" |
| assert "extra" not in row["value"] and row["value"]["year"] == 1989 |
|
|
|
|
| def test_score_record_exact_and_normalized(): |
| schema = {"type": "object", "properties": { |
| "title": {"type": "string"}, "n": {"type": "number"}, |
| "kind": {"enum": ["a", "b"]}, "tags": {"type": "array", "items": {"type": "string"}}, |
| "gone": {"type": ["string", "null"]}}} |
| gold = {"title": "Deep Nets.", "n": 3, "kind": "a", "tags": ["x", "y"], "gone": None} |
| pred = {"title": "deep nets", "n": 3.0, "kind": "b", "tags": ["x"], "gone": None} |
| c = score_record(schema, gold, pred) |
| |
| assert c == {"extract_n": 4, "exact": 2, "normalized": 3, "count_n": 1, "count_ok": 0} |
| assert normalize_value("1,989") == normalize_value(1989) == 1989.0 |
|
|
