Download src/reference_db/phase_3/outputs.py from offCanada/reference-db-system: direct link, hf CLI and curl.
- Browser
- Download file 1.87 kB
-
https://huggingface.co/offCanada/reference-db-system/resolve/main/src/reference_db/phase_3/outputs.py
- Command line
-
hf download hf://offCanada/reference-db-system/src/reference_db/phase_3/outputs.py
-
curl -L -o outputs.py https://huggingface.co/offCanada/reference-db-system/resolve/main/src/reference_db/phase_3/outputs.py
1.87 kB
| from __future__ import annotations | |
| from pathlib import Path | |
| from typing import Any | |
| import pandas as pd | |
| OUTPUT_COLUMNS = { | |
| "product_classification": [ | |
| "external_id", | |
| "product_name", | |
| "domain", | |
| "domain_confidence", | |
| "domain_rule", | |
| "domain_status", | |
| "taxonomy", | |
| "taxonomy_confidence", | |
| "taxonomy_rule", | |
| "taxonomy_status", | |
| "taxonomy_resolution", | |
| "taxonomy_candidates", | |
| ], | |
| "product_taxonomy": [ | |
| "external_id", | |
| "taxonomy", | |
| "taxonomy_confidence", | |
| "taxonomy_rule", | |
| "taxonomy_status", | |
| "taxonomy_resolution", | |
| "taxonomy_candidates", | |
| ], | |
| "taxonomy_review_queue": [ | |
| "external_id", | |
| "product_name", | |
| "taxonomy", | |
| "taxonomy_confidence", | |
| "taxonomy_rule", | |
| "taxonomy_status", | |
| "taxonomy_resolution", | |
| "taxonomy_candidates", | |
| ], | |
| "product_groups": [ | |
| "external_id", | |
| "group_id", | |
| ], | |
| "grouping_review_queue": [ | |
| "external_id_a", | |
| "external_id_b", | |
| "decision", | |
| "confidence", | |
| "reason", | |
| ], | |
| "grouping_contradictions": [ | |
| "external_id_a", | |
| "external_id_b", | |
| "decision", | |
| "confidence", | |
| "reason", | |
| ], | |
| } | |
| def write_phase3_outputs( | |
| results: dict[str, list[dict[str, Any]]], | |
| output_dir: str | Path = "phase_3", | |
| ) -> dict[str, Path]: | |
| output_path = Path(output_dir) | |
| output_path.mkdir(parents=True, exist_ok=True) | |
| paths: dict[str, Path] = {} | |
| for dataset_name, records in results.items(): | |
| columns = OUTPUT_COLUMNS.get(dataset_name) | |
| df = pd.DataFrame(records, columns=columns) | |
| path = output_path / f"{dataset_name}.parquet" | |
| df.to_parquet(path, index=False) | |
| paths[dataset_name] = path | |
| return paths |