Download src/reference_db/production/validation.py from offCanada/reference-db-system: direct link, hf CLI and curl.
- Browser
- Download file 4.14 kB
-
https://huggingface.co/offCanada/reference-db-system/resolve/main/src/reference_db/production/validation.py
- Command line
-
hf download hf://offCanada/reference-db-system/src/reference_db/production/validation.py
-
curl -L -o validation.py https://huggingface.co/offCanada/reference-db-system/resolve/main/src/reference_db/production/validation.py
4.14 kB
| from pathlib import Path | |
| import pandas as pd | |
| BASE_DIR = Path(__file__).resolve().parents[3] | |
| PRODUCTION_DIR = BASE_DIR / "data" / "production" | |
| FILES = { | |
| "product_metadata": PRODUCTION_DIR / "product_metadata.parquet", | |
| "product_groups": PRODUCTION_DIR / "product_groups.parquet", | |
| "product_variants": PRODUCTION_DIR / "product_variants.parquet", | |
| "product_nutrition": PRODUCTION_DIR / "product_nutrition.parquet", | |
| "product_scores": PRODUCTION_DIR / "product_scores.parquet", | |
| } | |
| def load_production(): | |
| return { | |
| name: pd.read_parquet(path) | |
| for name, path in FILES.items() | |
| } | |
| def validate_production(): | |
| tables = load_production() | |
| metadata = tables["product_metadata"] | |
| groups = tables["product_groups"] | |
| variants = tables["product_variants"] | |
| nutrition = tables["product_nutrition"] | |
| scores = tables["product_scores"] | |
| errors = [] | |
| # 1. Product-level uniqueness | |
| for name, df in { | |
| "product_metadata": metadata, | |
| "product_nutrition": nutrition, | |
| "product_scores": scores, | |
| }.items(): | |
| if "external_id" not in df.columns: | |
| errors.append(f"{name}: missing external_id") | |
| elif df["external_id"].duplicated().any(): | |
| errors.append( | |
| f"{name}: duplicate external_id values" | |
| ) | |
| # 2. Group-level uniqueness | |
| if groups["group_id"].duplicated().any(): | |
| errors.append("product_groups: duplicate group_id values") | |
| # 3. Variant-level uniqueness | |
| if variants["variant_id"].duplicated().any(): | |
| errors.append("product_variants: duplicate variant_id values") | |
| # 4. Product coverage | |
| metadata_ids = set(metadata["external_id"]) | |
| nutrition_ids = set(nutrition["external_id"]) | |
| score_ids = set(scores["external_id"]) | |
| missing_nutrition = metadata_ids - nutrition_ids | |
| missing_scores = metadata_ids - score_ids | |
| if missing_nutrition: | |
| errors.append( | |
| f"Products missing from nutrition: {len(missing_nutrition)}" | |
| ) | |
| if missing_scores: | |
| errors.append( | |
| f"Products missing from scores: {len(missing_scores)}" | |
| ) | |
| # 5. Group coverage | |
| group_ids = set(groups["group_id"]) | |
| for name, df in { | |
| "product_metadata": metadata, | |
| "product_nutrition": nutrition, | |
| "product_scores": scores, | |
| }.items(): | |
| if "group_id" in df.columns: | |
| referenced = set(df["group_id"].dropna()) | |
| orphan_groups = referenced - group_ids | |
| if orphan_groups: | |
| errors.append( | |
| f"{name}: {len(orphan_groups)} orphan group_id values" | |
| ) | |
| # 6. Variant coverage | |
| variant_ids = set(variants["variant_id"]) | |
| for name, df in { | |
| "product_nutrition": nutrition, | |
| "product_scores": scores, | |
| }.items(): | |
| if "variant_id" in df.columns: | |
| referenced = set(df["variant_id"].dropna()) | |
| orphan_variants = referenced - variant_ids | |
| if orphan_variants: | |
| errors.append( | |
| f"{name}: {len(orphan_variants)} orphan variant_id values" | |
| ) | |
| # 7. Basic row-count expectations | |
| expected_counts = { | |
| "product_metadata": 4440, | |
| "product_groups": 3845, | |
| "product_variants": 4320, | |
| "product_nutrition": 4440, | |
| "product_scores": 4440, | |
| } | |
| for name, expected in expected_counts.items(): | |
| actual = len(tables[name]) | |
| if actual != expected: | |
| errors.append( | |
| f"{name}: expected {expected} rows, found {actual}" | |
| ) | |
| # Final result | |
| print("\n=== Production Validation ===") | |
| for name, df in tables.items(): | |
| print( | |
| f"{name}: {len(df):,} rows × {len(df.columns)} columns" | |
| ) | |
| if errors: | |
| print("\n❌ VALIDATION FAILED") | |
| for error in errors: | |
| print(f"- {error}") | |
| raise SystemExit(1) | |
| print("\n✅ VALIDATION PASSED") | |
| print("All production tables are structurally consistent.") | |
| if __name__ == "__main__": | |
| validate_production() | |