Download scripts/regression_check.py from offCanada/reference-db-system: direct link, hf CLI and curl.
- Browser
- Download file 2.32 kB
-
https://huggingface.co/offCanada/reference-db-system/resolve/main/scripts/regression_check.py
- Command line
-
hf download hf://offCanada/reference-db-system/scripts/regression_check.py
-
curl -L -o regression_check.py https://huggingface.co/offCanada/reference-db-system/resolve/main/scripts/regression_check.py
2.32 kB
| #!/usr/bin/env python3 | |
| """Full regression: classify all phase_2 products, compare to prev PASS set.""" | |
| import pickle | |
| import sys | |
| from multiprocessing import Pool, cpu_count | |
| import pandas as pd | |
| sys.path.insert(0, "/home/sara/reference-db-system/src") | |
| from reference_db.classification.classifier import classify_product | |
| P2 = "/home/sara/reference-db-system/data/phase_2/standardized_products.parquet" | |
| OUT = "/tmp/phase3_backup/after_all_classified.pkl" | |
| def classify_one(name): | |
| try: | |
| r = classify_product(name) | |
| return { | |
| "taxonomy": r.taxonomy, | |
| "tax_status": r.taxonomy_status, | |
| "tax_check": r.taxonomy_resolution if r.taxonomy_resolution else r.taxonomy_rule, | |
| "domain": r.domain, | |
| "domain_conf": r.domain_confidence, | |
| "domain_rule": r.domain_rule, | |
| "tax_rule": r.taxonomy_rule, | |
| } | |
| except Exception as e: # noqa: BLE001 — keep one product failure from aborting the full regression | |
| return {"taxonomy": None, "tax_status": f"ERROR:{e}", "tax_check": None, | |
| "domain": None, "domain_conf": 0, "domain_rule": None, "tax_rule": None} | |
| def main(): | |
| df = pd.read_parquet(P2) | |
| print("phase2:", df.shape, flush=True) | |
| names = df["product_name"].tolist() | |
| with Pool(cpu_count()) as pool: | |
| results = pool.map(classify_one, names) | |
| out = pd.DataFrame(results) | |
| out["external_id"] = df["external_id"].astype(str) | |
| print("saving", out.shape, flush=True) | |
| out.to_pickle(OUT) | |
| with open("/tmp/phase3_backup/prev_pass.pkl", "rb") as f: | |
| prev = pickle.load(f) | |
| new_pass = set(out.loc[out["tax_status"] == "PASS", "external_id"]) | |
| dropped = prev - new_pass | |
| print("new PASS:", len(new_pass), "prev:", len(prev), "dropped:", len(dropped), flush=True) | |
| if dropped: | |
| sub = df[df["external_id"].astype(str).isin(dropped)] | |
| for _, r in sub.iterrows(): | |
| row = out[out["external_id"] == str(r["external_id"])].iloc[0] | |
| print(" DROPPED:", r["product_name"], "->", row["tax_status"], row["tax_check"], flush=True) | |
| print("status counts:", out["tax_status"].value_counts().to_dict(), flush=True) | |
| print("domain counts:", out["domain"].value_counts().to_dict(), flush=True) | |
| if __name__ == "__main__": | |
| main() | |