saraNour's picture
Upload folder using huggingface_hub
1cd137c verified
Raw History Blame Contribute Delete
1.87 kB
from __future__ import annotations
from pathlib import Path
from typing import Any
import pandas as pd
OUTPUT_COLUMNS = {
"product_classification": [
"external_id",
"product_name",
"domain",
"domain_confidence",
"domain_rule",
"domain_status",
"taxonomy",
"taxonomy_confidence",
"taxonomy_rule",
"taxonomy_status",
"taxonomy_resolution",
"taxonomy_candidates",
],
"product_taxonomy": [
"external_id",
"taxonomy",
"taxonomy_confidence",
"taxonomy_rule",
"taxonomy_status",
"taxonomy_resolution",
"taxonomy_candidates",
],
"taxonomy_review_queue": [
"external_id",
"product_name",
"taxonomy",
"taxonomy_confidence",
"taxonomy_rule",
"taxonomy_status",
"taxonomy_resolution",
"taxonomy_candidates",
],
"product_groups": [
"external_id",
"group_id",
],
"grouping_review_queue": [
"external_id_a",
"external_id_b",
"decision",
"confidence",
"reason",
],
"grouping_contradictions": [
"external_id_a",
"external_id_b",
"decision",
"confidence",
"reason",
],
}
def write_phase3_outputs(
results: dict[str, list[dict[str, Any]]],
output_dir: str | Path = "phase_3",
) -> dict[str, Path]:
output_path = Path(output_dir)
output_path.mkdir(parents=True, exist_ok=True)
paths: dict[str, Path] = {}
for dataset_name, records in results.items():
columns = OUTPUT_COLUMNS.get(dataset_name)
df = pd.DataFrame(records, columns=columns)
path = output_path / f"{dataset_name}.parquet"
df.to_parquet(path, index=False)
paths[dataset_name] = path
return paths