Download scripts/regression_after.py from offCanada/reference-db-system: direct link, hf CLI and curl.
- Browser
- Download file 2.99 kB
-
https://huggingface.co/offCanada/reference-db-system/resolve/main/scripts/regression_after.py
- Command line
-
hf download hf://offCanada/reference-db-system/scripts/regression_after.py
-
curl -L -o regression_after.py https://huggingface.co/offCanada/reference-db-system/resolve/main/scripts/regression_after.py
2.99 kB
| #!/usr/bin/env python3 | |
| """Classify all phase_2 products with the NEW rules, compare to prev PASS set. | |
| Writes results incrementally to /tmp so we can poll. Deterministic, no LLM. | |
| """ | |
| import pickle | |
| import sys | |
| import time | |
| from multiprocessing import Pool, cpu_count | |
| import pandas as pd | |
| sys.path.insert(0, "/home/sara/reference-db-system/src") | |
| t0 = time.time() | |
| P2 = "/home/sara/reference-db-system/data/phase_2/standardized_products.parquet" | |
| PREV_PASS = "/tmp/phase3_backup/prev_pass.pkl" | |
| OUT = "/tmp/phase3_backup/after_full_pass_ids.pkl" | |
| STREAM = "/tmp/phase3_backup/after_classify_stream.parquet" | |
| from reference_db.classification.classifier import classify_product | |
| def classify_one(name: str) -> dict: | |
| try: | |
| r = classify_product(name) | |
| return { | |
| "taxonomy": r.taxonomy, | |
| "taxonomy_status": r.taxonomy_status, | |
| "taxonomy_rule": r.taxonomy_rule, | |
| "taxonomy_confidence": r.taxonomy_confidence, | |
| "domain": r.domain, | |
| "domain_status": r.domain_status, | |
| "domain_rule": r.domain_rule, | |
| "domain_confidence": r.domain_confidence, | |
| } | |
| except Exception as e: # noqa: BLE001 | |
| return { | |
| "taxonomy": None, | |
| "taxonomy_status": f"ERROR:{type(e).__name__}:{e}", | |
| "taxonomy_rule": None, | |
| "taxonomy_confidence": None, | |
| "domain": None, | |
| "domain_status": "ERROR", | |
| "domain_rule": None, | |
| "domain_confidence": None, | |
| } | |
| def main() -> None: | |
| print("loading phase_2 ...", flush=True) | |
| df = pd.read_parquet(P2, columns=["external_id", "product_name"]) | |
| names = df["product_name"].tolist() | |
| print(f"phase_2: {len(names)} rows", flush=True) | |
| with Pool(cpu_count()) as pool: | |
| rows = list( | |
| pool.imap(classify_one, names, chunksize=64) | |
| ) | |
| out = pd.DataFrame(rows) | |
| out["external_id"] = df["external_id"].astype(str).values | |
| out.to_parquet(STREAM, index=False) | |
| with open(PREV_PASS, "rb") as f: | |
| prev = pickle.load(f) | |
| new_pass = set(out.loc[out["taxonomy_status"] == "PASS", "external_id"]) | |
| print(f"prev PASS: {len(prev)} new PASS: {len(new_pass)}", flush=True) | |
| print(f"prev still PASS (all): {prev.issubset(new_pass)}", flush=True) | |
| dropped = prev - new_pass | |
| print(f"dropped: {len(dropped)}", flush=True) | |
| for eid in sorted(dropped): | |
| row = df[df["external_id"].astype(str) == eid].iloc[0] | |
| got = out[out["external_id"] == eid].iloc[0] | |
| print(f" DROPPED {row['product_name'][:60]!r} -> {got['taxonomy_status']} {got['taxonomy_rule']}", flush=True) | |
| print("taxonomy_status counts:", out["taxonomy_status"].value_counts().to_dict(), flush=True) | |
| print("domain counts:", out["domain"].value_counts().to_dict(), flush=True) | |
| with open(OUT, "wb") as f: | |
| pickle.dump(new_pass, f) | |
| print(f"done in {time.time()-t0:.1f}s", flush=True) | |
| if __name__ == "__main__": | |
| main() | |