File size: 2,572 Bytes
e379257 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 | """Concatenate gallery JSONLs into one training/eval file in a new directory, rewriting image paths so they
resolve relative to the output file (train.py / evaluate.py take a single --data file).
Each input is `path` (all records) or `path@N` (N records sampled with --seed, input order kept).
Every output record gets meta.mix_source = the input path (relative to the output dir).
usage: python scripts/make_mix.py out.jsonl in1.jsonl in2.jsonl@300 [--seed 0]
"""
import json, os, random, sys
from collections import Counter
def main(argv):
seed = 0
if "--seed" in argv:
i = argv.index("--seed"); seed = int(argv[i + 1]); del argv[i:i + 2]
dst, inputs = argv[0], argv[1:]
odir = os.path.dirname(os.path.abspath(dst)); os.makedirs(odir, exist_ok=True)
rng = random.Random(seed); n_out = 0; per = Counter(); seen = set()
with open(dst, "w", encoding="utf-8", newline="\n") as out:
for spec in inputs:
path, n = (spec.rsplit("@", 1) + [None])[:2] if "@" in spec else (spec, None)
with open(path, encoding="utf-8") as f:
rows = [json.loads(l) for l in f if l.strip()]
if n is not None:
keep = set(rng.sample(range(len(rows)), min(int(n), len(rows))))
rows = [r for i, r in enumerate(rows) if i in keep]
idir = os.path.dirname(os.path.abspath(path))
src_rel = os.path.relpath(os.path.abspath(path), odir).replace("\\", "/")
for r in rows:
if r["id"] in seen:
sys.exit(f"duplicate id {r['id']} ({spec})")
seen.add(r["id"])
r["images"] = [im if os.path.isabs(im) else os.path.relpath(os.path.join(idir, im), odir).replace("\\", "/")
for im in r["images"]]
missing = [im for im in r["images"] if not os.path.exists(os.path.join(odir, im))]
if missing:
sys.exit(f"missing image {missing} ({spec})")
r.setdefault("meta", {})["mix_source"] = src_rel
out.write(json.dumps(r, ensure_ascii=False) + "\n"); n_out += 1; per[src_rel] += 1
n_box = 0
with open(dst, encoding="utf-8") as f:
for l in f:
r = json.loads(l); n_box += any(q.get("type") == "box" for q in r["questions"].values())
print(json.dumps({"out": dst, "records": n_out, "records_with_box_questions": n_box, "per_input": per, "seed": seed}))
if __name__ == "__main__":
main(sys.argv[1:])
|