File size: 1,873 Bytes
1cd137c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 | from __future__ import annotations
from pathlib import Path
from typing import Any
import pandas as pd
OUTPUT_COLUMNS = {
"product_classification": [
"external_id",
"product_name",
"domain",
"domain_confidence",
"domain_rule",
"domain_status",
"taxonomy",
"taxonomy_confidence",
"taxonomy_rule",
"taxonomy_status",
"taxonomy_resolution",
"taxonomy_candidates",
],
"product_taxonomy": [
"external_id",
"taxonomy",
"taxonomy_confidence",
"taxonomy_rule",
"taxonomy_status",
"taxonomy_resolution",
"taxonomy_candidates",
],
"taxonomy_review_queue": [
"external_id",
"product_name",
"taxonomy",
"taxonomy_confidence",
"taxonomy_rule",
"taxonomy_status",
"taxonomy_resolution",
"taxonomy_candidates",
],
"product_groups": [
"external_id",
"group_id",
],
"grouping_review_queue": [
"external_id_a",
"external_id_b",
"decision",
"confidence",
"reason",
],
"grouping_contradictions": [
"external_id_a",
"external_id_b",
"decision",
"confidence",
"reason",
],
}
def write_phase3_outputs(
results: dict[str, list[dict[str, Any]]],
output_dir: str | Path = "phase_3",
) -> dict[str, Path]:
output_path = Path(output_dir)
output_path.mkdir(parents=True, exist_ok=True)
paths: dict[str, Path] = {}
for dataset_name, records in results.items():
columns = OUTPUT_COLUMNS.get(dataset_name)
df = pd.DataFrame(records, columns=columns)
path = output_path / f"{dataset_name}.parquet"
df.to_parquet(path, index=False)
paths[dataset_name] = path
return paths |