File size: 1,873 Bytes
1cd137c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
from __future__ import annotations

from pathlib import Path
from typing import Any

import pandas as pd

OUTPUT_COLUMNS = {
    "product_classification": [
        "external_id",
        "product_name",
        "domain",
        "domain_confidence",
        "domain_rule",
        "domain_status",
        "taxonomy",
        "taxonomy_confidence",
        "taxonomy_rule",
        "taxonomy_status",
        "taxonomy_resolution",
        "taxonomy_candidates",
    ],
    "product_taxonomy": [
        "external_id",
        "taxonomy",
        "taxonomy_confidence",
        "taxonomy_rule",
        "taxonomy_status",
        "taxonomy_resolution",
        "taxonomy_candidates",
    ],
    "taxonomy_review_queue": [
        "external_id",
        "product_name",
        "taxonomy",
        "taxonomy_confidence",
        "taxonomy_rule",
        "taxonomy_status",
        "taxonomy_resolution",
        "taxonomy_candidates",
    ],
    "product_groups": [
        "external_id",
        "group_id",
    ],
    "grouping_review_queue": [
        "external_id_a",
        "external_id_b",
        "decision",
        "confidence",
        "reason",
    ],
    "grouping_contradictions": [
        "external_id_a",
        "external_id_b",
        "decision",
        "confidence",
        "reason",
    ],
}


def write_phase3_outputs(
    results: dict[str, list[dict[str, Any]]],
    output_dir: str | Path = "phase_3",
) -> dict[str, Path]:
    output_path = Path(output_dir)
    output_path.mkdir(parents=True, exist_ok=True)

    paths: dict[str, Path] = {}

    for dataset_name, records in results.items():
        columns = OUTPUT_COLUMNS.get(dataset_name)
        df = pd.DataFrame(records, columns=columns)
        path = output_path / f"{dataset_name}.parquet"
        df.to_parquet(path, index=False)
        paths[dataset_name] = path

    return paths