"""Concatenate gallery JSONLs into one training/eval file in a new directory, rewriting image paths so they resolve relative to the output file (train.py / evaluate.py take a single --data file). Each input is `path` (all records) or `path@N` (N records sampled with --seed, input order kept). Every output record gets meta.mix_source = the input path (relative to the output dir). usage: python scripts/make_mix.py out.jsonl in1.jsonl in2.jsonl@300 [--seed 0] """ import json, os, random, sys from collections import Counter def main(argv): seed = 0 if "--seed" in argv: i = argv.index("--seed"); seed = int(argv[i + 1]); del argv[i:i + 2] dst, inputs = argv[0], argv[1:] odir = os.path.dirname(os.path.abspath(dst)); os.makedirs(odir, exist_ok=True) rng = random.Random(seed); n_out = 0; per = Counter(); seen = set() with open(dst, "w", encoding="utf-8", newline="\n") as out: for spec in inputs: path, n = (spec.rsplit("@", 1) + [None])[:2] if "@" in spec else (spec, None) with open(path, encoding="utf-8") as f: rows = [json.loads(l) for l in f if l.strip()] if n is not None: keep = set(rng.sample(range(len(rows)), min(int(n), len(rows)))) rows = [r for i, r in enumerate(rows) if i in keep] idir = os.path.dirname(os.path.abspath(path)) src_rel = os.path.relpath(os.path.abspath(path), odir).replace("\\", "/") for r in rows: if r["id"] in seen: sys.exit(f"duplicate id {r['id']} ({spec})") seen.add(r["id"]) r["images"] = [im if os.path.isabs(im) else os.path.relpath(os.path.join(idir, im), odir).replace("\\", "/") for im in r["images"]] missing = [im for im in r["images"] if not os.path.exists(os.path.join(odir, im))] if missing: sys.exit(f"missing image {missing} ({spec})") r.setdefault("meta", {})["mix_source"] = src_rel out.write(json.dumps(r, ensure_ascii=False) + "\n"); n_out += 1; per[src_rel] += 1 n_box = 0 with open(dst, encoding="utf-8") as f: for l in f: r = json.loads(l); n_box += any(q.get("type") == "box" for q in r["questions"].values()) print(json.dumps({"out": dst, "records": n_out, "records_with_box_questions": n_box, "per_input": per, "seed": seed})) if __name__ == "__main__": main(sys.argv[1:])