code-HFD / scripts /data /merge_swa_source_plans.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
9fc9873 verified
Raw History Blame Contribute Delete
2.07 kB
#!/usr/bin/env python3
"""Merge independently admitted SWA plans without manufacturing repeats."""
from __future__ import annotations
import argparse
import json
from collections import Counter
from pathlib import Path
def normalize(row: dict) -> dict:
item = dict(row)
question = item.get("question")
nested_flag = (
question.get("swa_has_question") if isinstance(question, dict) else None
)
item["swa_has_question"] = bool(
item.get("swa_has_question", True if nested_flag is None else nested_flag)
)
return item
def merge(plans: list[list[dict]]) -> list[dict]:
output = []
seen = set()
for plan in plans:
for row in plan:
item = normalize(row)
item_id = str(item["id"])
if item_id in seen:
raise ValueError(f"Duplicate SWA source id: {item_id}")
seen.add(item_id)
output.append(item)
return output
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--input", type=Path, action="append", required=True)
parser.add_argument("--output", type=Path, required=True)
args = parser.parse_args()
plans = [json.loads(path.read_text(encoding="utf-8")) for path in args.input]
output = merge(plans)
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(
json.dumps(output, ensure_ascii=False, indent=2), encoding="utf-8"
)
report = {
"sources": len(output),
"split_sources": dict(Counter(str(row["split"]) for row in output)),
"qa_sources": sum(row["swa_has_question"] for row in output),
"ordinary_sources": sum(not row["swa_has_question"] for row in output),
"repeat_factor": 1,
"input_plans": [str(path.resolve()) for path in args.input],
}
args.output.with_suffix(".audit.json").write_text(
json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8"
)
print(json.dumps(report, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()