File size: 3,081 Bytes
41c8683 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 | """
检查 val.json 中每条记录的 image 和 mask 文件是否存在,
缺任一文件则删除该条,保存到新文件 (或原文件备份后覆盖)。
用法:
python clean_val_json.py --json /path/to/val.json
python clean_val_json.py --json /path/to/val.json --out /path/to/val_clean.json
python clean_val_json.py --json /path/to/val.json --inplace # 原地覆盖,自动备份为 .bak
"""
import argparse
import json
import os
from pathlib import Path
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--json", required=True, help="val.json 路径 (jsonl 格式)")
ap.add_argument("--root", default=None,
help="image/mask 路径的根目录;默认 = val.json 所在目录")
ap.add_argument("--out", default=None, help="输出路径,默认在原文件旁加 _clean")
ap.add_argument("--inplace", action="store_true",
help="原地覆盖原文件 (会自动备份为 .bak)")
args = ap.parse_args()
json_path = Path(args.json)
root = Path(args.root) if args.root else json_path.parent
# 读 jsonl
lines = [json.loads(l) for l in json_path.read_text().splitlines() if l.strip()]
print(f"读取 {len(lines)} 条记录,根目录: {root}")
kept, dropped = [], []
for item in lines:
img_rel = item["file_name"]
# attn_list 是 [[label, mask_path], ...] 列表,所有 mask 都要存在
mask_rels = [m[1] for m in item.get("attn_list", [])]
img_ok = (root / img_rel).is_file()
mask_ok = all((root / m).is_file() for m in mask_rels)
if img_ok and mask_ok:
kept.append(item)
else:
missing = []
if not img_ok:
missing.append(f"img:{img_rel}")
for m in mask_rels:
if not (root / m).is_file():
missing.append(f"mask:{m}")
dropped.append((item.get("file_name"), missing))
print(f"保留 {len(kept)} 条, 删除 {len(dropped)} 条")
if dropped:
print("\n删除示例 (最多前 5 条):")
for fn, miss in dropped[:5]:
print(f" - {fn} missing: {miss}")
# 决定输出路径
if args.inplace:
backup = json_path.with_suffix(json_path.suffix + ".bak")
if not backup.exists(): # 已有备份不覆盖,防止丢失原始版本
os.rename(json_path, backup)
print(f"\n原文件已备份: {backup}")
else:
print(f"\n[warn] 备份已存在,不再覆盖: {backup}")
out_path = json_path
else:
out_path = Path(args.out) if args.out else \
json_path.with_name(json_path.stem + "_clean" + json_path.suffix)
with open(out_path, "w") as f:
for item in kept:
f.write(json.dumps(item) + "\n")
print(f"已写出: {out_path}")
if __name__ == "__main__":
main()
"""
python clean_val_json.py --json /home/jovyan/AURAD/detection/ex_validation/siim-ptx-test_prompt.json --root /home/jovyan/AURAD/detection/ex_validation/siim-ptx_test
""" |