File size: 2,541 Bytes
8f78c5a 863c520 8f78c5a 863c520 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 | """Parity check: wikalid (installed package; formerly ourlid) vs the sidecar that labels Himay (built by the research-repo c7 scripts).
usage: parity_himay.py <documents.parquet> <shipped sidecar.parquet> <workdir> [n=3000] [workers=3]
Samples n docs (seed 20261115; every non-tgl label is over-sampled), runs `ourlid sidecar`, compares every column."""
import json, os, subprocess, sys
import numpy as np, pyarrow as pa, pyarrow.compute as pc, pyarrow.parquet as pq
docs, shipped, wd = sys.argv[1:4]; n = int(sys.argv[4]) if len(sys.argv) > 4 else 3000; workers = sys.argv[5] if len(sys.argv) > 5 else "3"
os.makedirs(wd, exist_ok=True)
S = pq.read_table(shipped); lab = S.column("ourlid_label_final").to_numpy(zero_copy_only=False); ids = S.column("document_id").to_numpy(zero_copy_only=False)
rng = np.random.default_rng(20261115)
pick = set(rng.choice(ids[lab == "tgl"], size=min(n // 2, int((lab == "tgl").sum())), replace=False))
rest = ids[lab != "tgl"]; pick |= set(rng.choice(rest, size=min(n - len(pick), len(rest)), replace=False))
f = pq.ParquetFile(docs); parts = []
for i in range(f.num_row_groups):
t = f.read_row_group(i, columns=["document_id", "source_name", "text"]); t = t.filter(pc.is_in(t.column("document_id"), pa.array(sorted(pick))))
if t.num_rows: parts.append(t)
sub = pa.concat_tables(parts); src = os.path.join(wd, "parity_docs.parquet"); pq.write_table(sub, src, row_group_size=250)
out = os.path.join(wd, "parity_sidecar.parquet")
if os.path.exists(out): os.remove(out)
exe = os.path.join(os.path.dirname(sys.executable), "wikalid")
subprocess.run([exe, "sidecar", src, out, "--workers", workers], check=True, stdout=subprocess.DEVNULL)
N = pq.read_table(out).to_pandas().set_index("document_id"); O = S.filter(pc.is_in(S.column("document_id"), pa.array(sorted(pick)))).to_pandas().set_index("document_id").loc[N.index]
rep = {"n_docs": len(N), "labels_sampled": O.ourlid_label_final.value_counts().to_dict(), "columns": {}}
for c in O.columns:
a, b = O[c], N[c]
if a.dtype.kind == "f": eq = ((a == b) | (a.isna() & b.isna())); rep["columns"][c] = {"equal": int(eq.sum()), "max_abs_diff": float(np.nanmax(np.abs(a.values - b.values)) if len(a) else 0)}
else: eq = ((a == b) | (a.isna() & b.isna())); rep["columns"][c] = {"equal": int(eq.sum())}
rep["all_columns_identical"] = all(v["equal"] == len(N) for v in rep["columns"].values())
print(json.dumps(rep, indent=1))
json.dump(rep, open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "PARITY_HIMAY_v1.0.0.json"), "w"), indent=1)
|