File size: 2,989 Bytes
1cd137c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
#!/usr/bin/env python3
"""Classify all phase_2 products with the NEW rules, compare to prev PASS set.

Writes results incrementally to /tmp so we can poll. Deterministic, no LLM.
"""
import pickle
import sys
import time
from multiprocessing import Pool, cpu_count

import pandas as pd

sys.path.insert(0, "/home/sara/reference-db-system/src")

t0 = time.time()
P2 = "/home/sara/reference-db-system/data/phase_2/standardized_products.parquet"
PREV_PASS = "/tmp/phase3_backup/prev_pass.pkl"
OUT = "/tmp/phase3_backup/after_full_pass_ids.pkl"
STREAM = "/tmp/phase3_backup/after_classify_stream.parquet"

from reference_db.classification.classifier import classify_product


def classify_one(name: str) -> dict:
    try:
        r = classify_product(name)
        return {
            "taxonomy": r.taxonomy,
            "taxonomy_status": r.taxonomy_status,
            "taxonomy_rule": r.taxonomy_rule,
            "taxonomy_confidence": r.taxonomy_confidence,
            "domain": r.domain,
            "domain_status": r.domain_status,
            "domain_rule": r.domain_rule,
            "domain_confidence": r.domain_confidence,
        }
    except Exception as e:  # noqa: BLE001
        return {
            "taxonomy": None,
            "taxonomy_status": f"ERROR:{type(e).__name__}:{e}",
            "taxonomy_rule": None,
            "taxonomy_confidence": None,
            "domain": None,
            "domain_status": "ERROR",
            "domain_rule": None,
            "domain_confidence": None,
        }


def main() -> None:
    print("loading phase_2 ...", flush=True)
    df = pd.read_parquet(P2, columns=["external_id", "product_name"])
    names = df["product_name"].tolist()
    print(f"phase_2: {len(names)} rows", flush=True)

    with Pool(cpu_count()) as pool:
        rows = list(
            pool.imap(classify_one, names, chunksize=64)
        )
    out = pd.DataFrame(rows)
    out["external_id"] = df["external_id"].astype(str).values
    out.to_parquet(STREAM, index=False)

    with open(PREV_PASS, "rb") as f:
        prev = pickle.load(f)
    new_pass = set(out.loc[out["taxonomy_status"] == "PASS", "external_id"])
    print(f"prev PASS: {len(prev)}  new PASS: {len(new_pass)}", flush=True)
    print(f"prev still PASS (all): {prev.issubset(new_pass)}", flush=True)
    dropped = prev - new_pass
    print(f"dropped: {len(dropped)}", flush=True)
    for eid in sorted(dropped):
        row = df[df["external_id"].astype(str) == eid].iloc[0]
        got = out[out["external_id"] == eid].iloc[0]
        print(f"  DROPPED {row['product_name'][:60]!r} -> {got['taxonomy_status']} {got['taxonomy_rule']}", flush=True)
    print("taxonomy_status counts:", out["taxonomy_status"].value_counts().to_dict(), flush=True)
    print("domain counts:", out["domain"].value_counts().to_dict(), flush=True)
    with open(OUT, "wb") as f:
        pickle.dump(new_pass, f)
    print(f"done in {time.time()-t0:.1f}s", flush=True)


if __name__ == "__main__":
    main()