Download src/reference_db/phase_4/pipeline.py from offCanada/reference-db-system: direct link, hf CLI and curl.
- Browser
- Download file 2.94 kB
-
https://huggingface.co/offCanada/reference-db-system/resolve/main/src/reference_db/phase_4/pipeline.py
- Command line
-
hf download hf://offCanada/reference-db-system/src/reference_db/phase_4/pipeline.py
-
curl -L -o pipeline.py https://huggingface.co/offCanada/reference-db-system/resolve/main/src/reference_db/phase_4/pipeline.py
2.94 kB
| from __future__ import annotations | |
| import pandas as pd | |
| from reference_db.phase_4.outputs import write_phase4_outputs | |
| from reference_db.phase_4.validation import validate_phase4 | |
| from reference_db.phase_4.variants import ( | |
| assign_variants, | |
| build_variant_mapping, | |
| build_variant_summary, | |
| build_variant_table, | |
| ) | |
| def run_phase4( | |
| phase2_path: str = "data/phase_2/standardized_products.parquet", | |
| phase3_path: str = "data/phase_3/product_groups.parquet", | |
| output_dir: str = "data/phase_4", | |
| ) -> dict: | |
| phase2 = pd.read_parquet(phase2_path) | |
| phase3 = pd.read_parquet(phase3_path) | |
| required_phase2 = { | |
| "external_id", | |
| "variant_attributes", | |
| "core_title", | |
| "size", | |
| } | |
| required_phase3 = { | |
| "external_id", | |
| "group_id", | |
| } | |
| missing_phase2 = required_phase2 - set(phase2.columns) | |
| missing_phase3 = required_phase3 - set(phase3.columns) | |
| if missing_phase2: | |
| raise ValueError( | |
| f"Phase 2 missing required columns: {sorted(missing_phase2)}" | |
| ) | |
| if missing_phase3: | |
| raise ValueError( | |
| f"Phase 3 missing required columns: {sorted(missing_phase3)}" | |
| ) | |
| if phase2["external_id"].duplicated().any(): | |
| raise ValueError("Phase 2 contains duplicate external_id values") | |
| if phase3["external_id"].duplicated().any(): | |
| raise ValueError("Phase 3 contains duplicate external_id values") | |
| df = phase3.merge( | |
| phase2[ | |
| [ | |
| "external_id", | |
| "variant_attributes", | |
| "core_title", | |
| "size", | |
| ] | |
| ], | |
| on="external_id", | |
| how="left", | |
| validate="one_to_one", | |
| ) | |
| if len(df) != len(phase3): | |
| raise ValueError("Phase 4 merge changed the Phase 3 product count") | |
| if df["group_id"].isna().any(): | |
| raise ValueError("Phase 4 found products without group_id") | |
| df = assign_variants(df) | |
| variant_table = build_variant_table(df) | |
| variant_mapping = build_variant_mapping(df) | |
| variant_summary = build_variant_summary(df) | |
| validation = validate_phase4( | |
| df, | |
| variant_table, | |
| variant_mapping, | |
| variant_summary, | |
| ) | |
| write_phase4_outputs( | |
| variant_table, | |
| variant_mapping, | |
| variant_summary, | |
| validation, | |
| output_dir, | |
| ) | |
| return { | |
| "product_variants": variant_table, | |
| "product_variant_mapping": variant_mapping, | |
| "product_variant_summary": variant_summary, | |
| "validation": validation, | |
| } | |
| if __name__ == "__main__": | |
| results = run_phase4() | |
| print("\n=== PHASE 4 COMPLETE ===") | |
| print(f"Products: {len(results['product_variant_mapping'])}") | |
| print(f"Groups: {len(results['product_variant_summary'])}") | |
| print(f"Variants: {len(results['product_variants'])}") | |
| print(f"Validation: {results['validation']['overall']['result']}") | |