File size: 2,937 Bytes
1cd137c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 | from __future__ import annotations
import pandas as pd
from reference_db.phase_4.outputs import write_phase4_outputs
from reference_db.phase_4.validation import validate_phase4
from reference_db.phase_4.variants import (
assign_variants,
build_variant_mapping,
build_variant_summary,
build_variant_table,
)
def run_phase4(
phase2_path: str = "data/phase_2/standardized_products.parquet",
phase3_path: str = "data/phase_3/product_groups.parquet",
output_dir: str = "data/phase_4",
) -> dict:
phase2 = pd.read_parquet(phase2_path)
phase3 = pd.read_parquet(phase3_path)
required_phase2 = {
"external_id",
"variant_attributes",
"core_title",
"size",
}
required_phase3 = {
"external_id",
"group_id",
}
missing_phase2 = required_phase2 - set(phase2.columns)
missing_phase3 = required_phase3 - set(phase3.columns)
if missing_phase2:
raise ValueError(
f"Phase 2 missing required columns: {sorted(missing_phase2)}"
)
if missing_phase3:
raise ValueError(
f"Phase 3 missing required columns: {sorted(missing_phase3)}"
)
if phase2["external_id"].duplicated().any():
raise ValueError("Phase 2 contains duplicate external_id values")
if phase3["external_id"].duplicated().any():
raise ValueError("Phase 3 contains duplicate external_id values")
df = phase3.merge(
phase2[
[
"external_id",
"variant_attributes",
"core_title",
"size",
]
],
on="external_id",
how="left",
validate="one_to_one",
)
if len(df) != len(phase3):
raise ValueError("Phase 4 merge changed the Phase 3 product count")
if df["group_id"].isna().any():
raise ValueError("Phase 4 found products without group_id")
df = assign_variants(df)
variant_table = build_variant_table(df)
variant_mapping = build_variant_mapping(df)
variant_summary = build_variant_summary(df)
validation = validate_phase4(
df,
variant_table,
variant_mapping,
variant_summary,
)
write_phase4_outputs(
variant_table,
variant_mapping,
variant_summary,
validation,
output_dir,
)
return {
"product_variants": variant_table,
"product_variant_mapping": variant_mapping,
"product_variant_summary": variant_summary,
"validation": validation,
}
if __name__ == "__main__":
results = run_phase4()
print("\n=== PHASE 4 COMPLETE ===")
print(f"Products: {len(results['product_variant_mapping'])}")
print(f"Groups: {len(results['product_variant_summary'])}")
print(f"Variants: {len(results['product_variants'])}")
print(f"Validation: {results['validation']['overall']['result']}")
|