Download scripts/data/merge_swa_source_plans.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 2.07 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/scripts/data/merge_swa_source_plans.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/scripts/data/merge_swa_source_plans.py
-
curl -L -o merge_swa_source_plans.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/scripts/data/merge_swa_source_plans.py
2.07 kB
| #!/usr/bin/env python3 | |
| """Merge independently admitted SWA plans without manufacturing repeats.""" | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| from collections import Counter | |
| from pathlib import Path | |
| def normalize(row: dict) -> dict: | |
| item = dict(row) | |
| question = item.get("question") | |
| nested_flag = ( | |
| question.get("swa_has_question") if isinstance(question, dict) else None | |
| ) | |
| item["swa_has_question"] = bool( | |
| item.get("swa_has_question", True if nested_flag is None else nested_flag) | |
| ) | |
| return item | |
| def merge(plans: list[list[dict]]) -> list[dict]: | |
| output = [] | |
| seen = set() | |
| for plan in plans: | |
| for row in plan: | |
| item = normalize(row) | |
| item_id = str(item["id"]) | |
| if item_id in seen: | |
| raise ValueError(f"Duplicate SWA source id: {item_id}") | |
| seen.add(item_id) | |
| output.append(item) | |
| return output | |
| def main() -> None: | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument("--input", type=Path, action="append", required=True) | |
| parser.add_argument("--output", type=Path, required=True) | |
| args = parser.parse_args() | |
| plans = [json.loads(path.read_text(encoding="utf-8")) for path in args.input] | |
| output = merge(plans) | |
| args.output.parent.mkdir(parents=True, exist_ok=True) | |
| args.output.write_text( | |
| json.dumps(output, ensure_ascii=False, indent=2), encoding="utf-8" | |
| ) | |
| report = { | |
| "sources": len(output), | |
| "split_sources": dict(Counter(str(row["split"]) for row in output)), | |
| "qa_sources": sum(row["swa_has_question"] for row in output), | |
| "ordinary_sources": sum(not row["swa_has_question"] for row in output), | |
| "repeat_factor": 1, | |
| "input_plans": [str(path.resolve()) for path in args.input], | |
| } | |
| args.output.with_suffix(".audit.json").write_text( | |
| json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8" | |
| ) | |
| print(json.dumps(report, ensure_ascii=False, indent=2)) | |
| if __name__ == "__main__": | |
| main() | |