File size: 2,989 Bytes
1cd137c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 | #!/usr/bin/env python3
"""Classify all phase_2 products with the NEW rules, compare to prev PASS set.
Writes results incrementally to /tmp so we can poll. Deterministic, no LLM.
"""
import pickle
import sys
import time
from multiprocessing import Pool, cpu_count
import pandas as pd
sys.path.insert(0, "/home/sara/reference-db-system/src")
t0 = time.time()
P2 = "/home/sara/reference-db-system/data/phase_2/standardized_products.parquet"
PREV_PASS = "/tmp/phase3_backup/prev_pass.pkl"
OUT = "/tmp/phase3_backup/after_full_pass_ids.pkl"
STREAM = "/tmp/phase3_backup/after_classify_stream.parquet"
from reference_db.classification.classifier import classify_product
def classify_one(name: str) -> dict:
try:
r = classify_product(name)
return {
"taxonomy": r.taxonomy,
"taxonomy_status": r.taxonomy_status,
"taxonomy_rule": r.taxonomy_rule,
"taxonomy_confidence": r.taxonomy_confidence,
"domain": r.domain,
"domain_status": r.domain_status,
"domain_rule": r.domain_rule,
"domain_confidence": r.domain_confidence,
}
except Exception as e: # noqa: BLE001
return {
"taxonomy": None,
"taxonomy_status": f"ERROR:{type(e).__name__}:{e}",
"taxonomy_rule": None,
"taxonomy_confidence": None,
"domain": None,
"domain_status": "ERROR",
"domain_rule": None,
"domain_confidence": None,
}
def main() -> None:
print("loading phase_2 ...", flush=True)
df = pd.read_parquet(P2, columns=["external_id", "product_name"])
names = df["product_name"].tolist()
print(f"phase_2: {len(names)} rows", flush=True)
with Pool(cpu_count()) as pool:
rows = list(
pool.imap(classify_one, names, chunksize=64)
)
out = pd.DataFrame(rows)
out["external_id"] = df["external_id"].astype(str).values
out.to_parquet(STREAM, index=False)
with open(PREV_PASS, "rb") as f:
prev = pickle.load(f)
new_pass = set(out.loc[out["taxonomy_status"] == "PASS", "external_id"])
print(f"prev PASS: {len(prev)} new PASS: {len(new_pass)}", flush=True)
print(f"prev still PASS (all): {prev.issubset(new_pass)}", flush=True)
dropped = prev - new_pass
print(f"dropped: {len(dropped)}", flush=True)
for eid in sorted(dropped):
row = df[df["external_id"].astype(str) == eid].iloc[0]
got = out[out["external_id"] == eid].iloc[0]
print(f" DROPPED {row['product_name'][:60]!r} -> {got['taxonomy_status']} {got['taxonomy_rule']}", flush=True)
print("taxonomy_status counts:", out["taxonomy_status"].value_counts().to_dict(), flush=True)
print("domain counts:", out["domain"].value_counts().to_dict(), flush=True)
with open(OUT, "wb") as f:
pickle.dump(new_pass, f)
print(f"done in {time.time()-t0:.1f}s", flush=True)
if __name__ == "__main__":
main()
|