saraNour's picture
Upload folder using huggingface_hub
1cd137c verified
Raw History Blame Contribute Delete
4.14 kB
from pathlib import Path
import pandas as pd
BASE_DIR = Path(__file__).resolve().parents[3]
PRODUCTION_DIR = BASE_DIR / "data" / "production"
FILES = {
"product_metadata": PRODUCTION_DIR / "product_metadata.parquet",
"product_groups": PRODUCTION_DIR / "product_groups.parquet",
"product_variants": PRODUCTION_DIR / "product_variants.parquet",
"product_nutrition": PRODUCTION_DIR / "product_nutrition.parquet",
"product_scores": PRODUCTION_DIR / "product_scores.parquet",
}
def load_production():
return {
name: pd.read_parquet(path)
for name, path in FILES.items()
}
def validate_production():
tables = load_production()
metadata = tables["product_metadata"]
groups = tables["product_groups"]
variants = tables["product_variants"]
nutrition = tables["product_nutrition"]
scores = tables["product_scores"]
errors = []
# 1. Product-level uniqueness
for name, df in {
"product_metadata": metadata,
"product_nutrition": nutrition,
"product_scores": scores,
}.items():
if "external_id" not in df.columns:
errors.append(f"{name}: missing external_id")
elif df["external_id"].duplicated().any():
errors.append(
f"{name}: duplicate external_id values"
)
# 2. Group-level uniqueness
if groups["group_id"].duplicated().any():
errors.append("product_groups: duplicate group_id values")
# 3. Variant-level uniqueness
if variants["variant_id"].duplicated().any():
errors.append("product_variants: duplicate variant_id values")
# 4. Product coverage
metadata_ids = set(metadata["external_id"])
nutrition_ids = set(nutrition["external_id"])
score_ids = set(scores["external_id"])
missing_nutrition = metadata_ids - nutrition_ids
missing_scores = metadata_ids - score_ids
if missing_nutrition:
errors.append(
f"Products missing from nutrition: {len(missing_nutrition)}"
)
if missing_scores:
errors.append(
f"Products missing from scores: {len(missing_scores)}"
)
# 5. Group coverage
group_ids = set(groups["group_id"])
for name, df in {
"product_metadata": metadata,
"product_nutrition": nutrition,
"product_scores": scores,
}.items():
if "group_id" in df.columns:
referenced = set(df["group_id"].dropna())
orphan_groups = referenced - group_ids
if orphan_groups:
errors.append(
f"{name}: {len(orphan_groups)} orphan group_id values"
)
# 6. Variant coverage
variant_ids = set(variants["variant_id"])
for name, df in {
"product_nutrition": nutrition,
"product_scores": scores,
}.items():
if "variant_id" in df.columns:
referenced = set(df["variant_id"].dropna())
orphan_variants = referenced - variant_ids
if orphan_variants:
errors.append(
f"{name}: {len(orphan_variants)} orphan variant_id values"
)
# 7. Basic row-count expectations
expected_counts = {
"product_metadata": 4440,
"product_groups": 3845,
"product_variants": 4320,
"product_nutrition": 4440,
"product_scores": 4440,
}
for name, expected in expected_counts.items():
actual = len(tables[name])
if actual != expected:
errors.append(
f"{name}: expected {expected} rows, found {actual}"
)
# Final result
print("\n=== Production Validation ===")
for name, df in tables.items():
print(
f"{name}: {len(df):,} rows × {len(df.columns)} columns"
)
if errors:
print("\n❌ VALIDATION FAILED")
for error in errors:
print(f"- {error}")
raise SystemExit(1)
print("\n✅ VALIDATION PASSED")
print("All production tables are structurally consistent.")
if __name__ == "__main__":
validate_production()