File size: 2,315 Bytes
1cd137c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
#!/usr/bin/env python3
"""Full regression: classify all phase_2 products, compare to prev PASS set."""
import pickle
import sys
from multiprocessing import Pool, cpu_count

import pandas as pd

sys.path.insert(0, "/home/sara/reference-db-system/src")
from reference_db.classification.classifier import classify_product

P2 = "/home/sara/reference-db-system/data/phase_2/standardized_products.parquet"
OUT = "/tmp/phase3_backup/after_all_classified.pkl"

def classify_one(name):
    try:
        r = classify_product(name)
        return {
            "taxonomy": r.taxonomy,
            "tax_status": r.taxonomy_status,
            "tax_check": r.taxonomy_resolution if r.taxonomy_resolution else r.taxonomy_rule,
            "domain": r.domain,
            "domain_conf": r.domain_confidence,
            "domain_rule": r.domain_rule,
            "tax_rule": r.taxonomy_rule,
        }
    except Exception as e:  # noqa: BLE001 — keep one product failure from aborting the full regression
        return {"taxonomy": None, "tax_status": f"ERROR:{e}", "tax_check": None,
                "domain": None, "domain_conf": 0, "domain_rule": None, "tax_rule": None}

def main():
    df = pd.read_parquet(P2)
    print("phase2:", df.shape, flush=True)
    names = df["product_name"].tolist()
    with Pool(cpu_count()) as pool:
        results = pool.map(classify_one, names)

    out = pd.DataFrame(results)
    out["external_id"] = df["external_id"].astype(str)
    print("saving", out.shape, flush=True)
    out.to_pickle(OUT)

    with open("/tmp/phase3_backup/prev_pass.pkl", "rb") as f:
        prev = pickle.load(f)
    new_pass = set(out.loc[out["tax_status"] == "PASS", "external_id"])
    dropped = prev - new_pass
    print("new PASS:", len(new_pass), "prev:", len(prev), "dropped:", len(dropped), flush=True)
    if dropped:
        sub = df[df["external_id"].astype(str).isin(dropped)]
        for _, r in sub.iterrows():
            row = out[out["external_id"] == str(r["external_id"])].iloc[0]
            print("  DROPPED:", r["product_name"], "->", row["tax_status"], row["tax_check"], flush=True)
    print("status counts:", out["tax_status"].value_counts().to_dict(), flush=True)
    print("domain counts:", out["domain"].value_counts().to_dict(), flush=True)

if __name__ == "__main__":
    main()