"""Parity check: wikalid (installed package; formerly ourlid) vs the sidecar that labels Himay (built by the research-repo c7 scripts). usage: parity_himay.py [n=3000] [workers=3] Samples n docs (seed 20261115; every non-tgl label is over-sampled), runs `ourlid sidecar`, compares every column.""" import json, os, subprocess, sys import numpy as np, pyarrow as pa, pyarrow.compute as pc, pyarrow.parquet as pq docs, shipped, wd = sys.argv[1:4]; n = int(sys.argv[4]) if len(sys.argv) > 4 else 3000; workers = sys.argv[5] if len(sys.argv) > 5 else "3" os.makedirs(wd, exist_ok=True) S = pq.read_table(shipped); lab = S.column("ourlid_label_final").to_numpy(zero_copy_only=False); ids = S.column("document_id").to_numpy(zero_copy_only=False) rng = np.random.default_rng(20261115) pick = set(rng.choice(ids[lab == "tgl"], size=min(n // 2, int((lab == "tgl").sum())), replace=False)) rest = ids[lab != "tgl"]; pick |= set(rng.choice(rest, size=min(n - len(pick), len(rest)), replace=False)) f = pq.ParquetFile(docs); parts = [] for i in range(f.num_row_groups): t = f.read_row_group(i, columns=["document_id", "source_name", "text"]); t = t.filter(pc.is_in(t.column("document_id"), pa.array(sorted(pick)))) if t.num_rows: parts.append(t) sub = pa.concat_tables(parts); src = os.path.join(wd, "parity_docs.parquet"); pq.write_table(sub, src, row_group_size=250) out = os.path.join(wd, "parity_sidecar.parquet") if os.path.exists(out): os.remove(out) exe = os.path.join(os.path.dirname(sys.executable), "wikalid") subprocess.run([exe, "sidecar", src, out, "--workers", workers], check=True, stdout=subprocess.DEVNULL) N = pq.read_table(out).to_pandas().set_index("document_id"); O = S.filter(pc.is_in(S.column("document_id"), pa.array(sorted(pick)))).to_pandas().set_index("document_id").loc[N.index] rep = {"n_docs": len(N), "labels_sampled": O.ourlid_label_final.value_counts().to_dict(), "columns": {}} for c in O.columns: a, b = O[c], N[c] if a.dtype.kind == "f": eq = ((a == b) | (a.isna() & b.isna())); rep["columns"][c] = {"equal": int(eq.sum()), "max_abs_diff": float(np.nanmax(np.abs(a.values - b.values)) if len(a) else 0)} else: eq = ((a == b) | (a.isna() & b.isna())); rep["columns"][c] = {"equal": int(eq.sum())} rep["all_columns_identical"] = all(v["equal"] == len(N) for v in rep["columns"].values()) print(json.dumps(rep, indent=1)) json.dump(rep, open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "PARITY_HIMAY_v1.0.0.json"), "w"), indent=1)