Download evaluation/core.py from SciKnowOrg/Scientific-Table-Reconstruction: direct link, hf CLI and curl.
- Browser
- Download file 5.77 kB
-
https://huggingface.co/spaces/SciKnowOrg/Scientific-Table-Reconstruction/resolve/main/evaluation/core.py
- Command line
-
hf download hf://spaces/SciKnowOrg/Scientific-Table-Reconstruction/evaluation/core.py
-
curl -L -o core.py https://huggingface.co/spaces/SciKnowOrg/Scientific-Table-Reconstruction/resolve/main/evaluation/core.py
5.77 kB
| """Versioned scoring of complete Step 2 prediction sets.""" | |
| from collections import defaultdict | |
| import hashlib | |
| import json | |
| from pathlib import Path | |
| from metric.grid_scoring import aggregate_grid_results, prepare_grid, score_table_grids | |
| PROTOCOL = "tabulusbench-step2-v1" | |
| METRICS = ("rms", "exact-mapping") | |
| EVALUATION_MODE = "physical-table" | |
| INPUT_MODE = "image-only" | |
| def canonical(value): | |
| return json.dumps(value, sort_keys=True, ensure_ascii=False, allow_nan=False, separators=(",", ":")).encode("utf-8") | |
| def digest(value): | |
| return hashlib.sha256(canonical(value)).hexdigest() | |
| def key(row): | |
| return row["paper_id"], row["table_id"] | |
| def check_grid(value): | |
| if not isinstance(value, list) or any( | |
| not isinstance(row, list) or any(not isinstance(cell, str) for cell in row) for row in value | |
| ): | |
| raise ValueError("rows must be a list of lists of strings") | |
| if len(value) > 10000 or sum(len(row) for row in value) > 1_000_000: | |
| raise ValueError("Prediction exceeds the documented table-size limit") | |
| return value | |
| def validate_submission(records, expected_keys): | |
| predictions = {} | |
| for position, item in enumerate(records, 1): | |
| if not isinstance(item, dict) or set(item) != {"paper_id", "table_id", "status", "rows"}: | |
| raise ValueError(f"Prediction {position} needs exactly paper_id, table_id, status and rows") | |
| if not all(isinstance(item[field], str) and item[field] for field in ("paper_id", "table_id")): | |
| raise ValueError(f"Prediction {position} has an invalid identity") | |
| identity = key(item) | |
| if identity not in expected_keys: | |
| raise ValueError(f"Unknown prediction identity {identity}") | |
| if identity in predictions: | |
| raise ValueError(f"Duplicate prediction identity {identity}") | |
| if item["status"] not in {"ok", "empty", "error"}: | |
| raise ValueError(f"Invalid status for {identity}") | |
| grid = check_grid(item["rows"]) | |
| if item["status"] != "ok" and grid: | |
| raise ValueError(f"Failed prediction {identity} must have rows=[]") | |
| predictions[identity] = item | |
| missing = expected_keys - predictions.keys() | |
| if missing: | |
| raise ValueError(f"{len(missing)} missing predictions; first {sorted(missing)[0]}. Use explicit empty/error records.") | |
| return predictions | |
| def validate_metadata(metadata, manifest): | |
| required = ("submission_id", "method_name", "model_revision", "code_url", "code_revision", "prompt", | |
| "parser", "generation_settings", "training_disclosure", "dataset_fingerprint", | |
| "evaluation_mode", "input_mode") | |
| if not isinstance(metadata, dict) or any( | |
| not isinstance(metadata.get(field), str) or not metadata[field].strip() for field in required | |
| ): | |
| raise ValueError("Missing submission metadata. See metadata.example.json") | |
| if metadata["dataset_fingerprint"] != manifest["dataset_fingerprint"]: | |
| raise ValueError("Submission targets a different dataset fingerprint") | |
| if metadata["evaluation_mode"] != EVALUATION_MODE or metadata["input_mode"] != INPUT_MODE: | |
| raise ValueError("v1 supports physical-table evaluation with image-only model inputs") | |
| if not metadata["submission_id"].isascii() or any( | |
| char not in "abcdefghijklmnopqrstuvwxyz0123456789-_" for char in metadata["submission_id"] | |
| ): | |
| raise ValueError("submission_id must contain lowercase ASCII letters, digits, hyphens or underscores") | |
| def evaluate(gold, records, metadata, manifest, progress=None): | |
| gold = sorted(gold, key=key) | |
| expected_keys = {key(row) for row in gold} | |
| if len(expected_keys) != len(gold) or not gold: | |
| raise ValueError("Gold identities must be unique and nonempty") | |
| validate_metadata(metadata, manifest) | |
| predictions = validate_submission(records, expected_keys) | |
| results = [] | |
| by_domain = defaultdict(list) | |
| for index, target in enumerate(gold, 1): | |
| prediction = predictions[key(target)] | |
| observed = prepare_grid(prediction["rows"]) | |
| status = "scored" | |
| if prediction["status"] == "error": | |
| status = "unusable" | |
| elif prediction["status"] == "empty" or not observed.nonempty_cells: | |
| status = "empty" | |
| scores = score_table_grids( | |
| target["gold_rows"], observed, metrics=METRICS, | |
| text_threshold=0.5, number_threshold=0.1, status=status, | |
| ) | |
| result = { | |
| "paper_id": target["paper_id"], "table_id": target["table_id"], "domain": target["domain"], | |
| "submitted_status": prediction["status"], **scores, | |
| } | |
| results.append(result) | |
| by_domain[target["domain"]].append(result) | |
| if progress: | |
| progress(index, len(gold)) | |
| summary = { | |
| "protocol_version": PROTOCOL, "dataset_version": manifest["version"], | |
| "dataset_fingerprint": manifest["dataset_fingerprint"], | |
| "evaluation_mode": EVALUATION_MODE, "input_mode": INPUT_MODE, | |
| "score_scale": "[0,100]", "ranking": "macro.rms.f1", | |
| "submission": metadata, "prediction_sha256": digest(sorted(records, key=key)), | |
| **aggregate_grid_results(results), | |
| "by_domain": {domain: aggregate_grid_results(rows) for domain, rows in sorted(by_domain.items())}, | |
| } | |
| return summary, results | |
| def read_jsonl(path): | |
| records = [] | |
| with Path(path).open(encoding="utf-8-sig") as handle: | |
| for number, line in enumerate(handle, 1): | |
| if line.strip(): | |
| try: | |
| records.append(json.loads(line)) | |
| except json.JSONDecodeError as error: | |
| raise ValueError(f"Malformed JSON on line {number}") from error | |
| return records | |