Download src/reference_db/phase_6/outputs.py from offCanada/reference-db-system: direct link, hf CLI and curl.
- Browser
- Download file 4.92 kB
-
https://huggingface.co/offCanada/reference-db-system/resolve/main/src/reference_db/phase_6/outputs.py
- Command line
-
hf download hf://offCanada/reference-db-system/src/reference_db/phase_6/outputs.py
-
curl -L -o outputs.py https://huggingface.co/offCanada/reference-db-system/resolve/main/src/reference_db/phase_6/outputs.py
4.92 kB
| from __future__ import annotations | |
| from pathlib import Path | |
| import pandas as pd | |
| PRODUCT_SCORE_COLUMNS = [ | |
| "external_id", | |
| "group_id", | |
| "variant_id", | |
| "domain", | |
| "domain_confidence", | |
| "domain_status", | |
| "taxonomy", | |
| "taxonomy_confidence", | |
| "taxonomy_rule", | |
| "taxonomy_status", | |
| "nutrition_quality_status", | |
| "score_eligibility", | |
| "score_exclusion_reason", | |
| "calories_per_100g", | |
| "energy_kj_per_100g", | |
| "sugars_g_per_100g", | |
| "saturated_fat_g_per_100g", | |
| "sodium_mg_per_100g", | |
| "salt_g_per_100g", | |
| "fibre_g_per_100g", | |
| "protein_g_per_100g", | |
| "fvl_percent", | |
| "fvl_method", | |
| "fvl_confidence", | |
| "fvl_provenance", | |
| "energy_points", | |
| "sugar_points", | |
| "saturated_fat_points", | |
| "salt_points", | |
| "negative_points", | |
| "protein_points", | |
| "fibre_points", | |
| "fvl_points", | |
| "positive_points", | |
| "nutri_score_raw", | |
| "nutri_score_grade", | |
| "nutri_score_calculated", | |
| "nutri_score_algorithm", | |
| "nutri_score_status", | |
| ] | |
| AGRIBALYSE_COLUMNS = [ | |
| "external_id", | |
| "group_id", | |
| "domain", | |
| "taxonomy", | |
| "agribalyse_category", | |
| "mapping_method", | |
| "mapping_confidence", | |
| "mapping_status", | |
| "mapping_source", | |
| "mapping_version", | |
| "mapping_note", | |
| ] | |
| EXCLUSION_COLUMNS = [ | |
| "external_id", | |
| "group_id", | |
| "domain", | |
| "taxonomy", | |
| "nutrition_quality_status", | |
| "score_exclusion_reason", | |
| ] | |
| def _select_existing( | |
| df: pd.DataFrame, | |
| columns: list[str], | |
| ) -> pd.DataFrame: | |
| """Select columns while failing loudly if a required output column is absent.""" | |
| missing = [column for column in columns if column not in df.columns] | |
| if missing: | |
| raise ValueError( | |
| f"Missing required output columns: {missing}" | |
| ) | |
| return df[columns].copy() | |
| def build_product_scores(df: pd.DataFrame) -> pd.DataFrame: | |
| return _select_existing(df, PRODUCT_SCORE_COLUMNS) | |
| def build_agribalyse_mapping(df: pd.DataFrame) -> pd.DataFrame: | |
| return _select_existing(df, AGRIBALYSE_COLUMNS) | |
| def build_score_exclusions(df: pd.DataFrame) -> pd.DataFrame: | |
| exclusions = df[ | |
| ~df["score_eligibility"].fillna(False) | |
| ].copy() | |
| return _select_existing( | |
| exclusions, | |
| EXCLUSION_COLUMNS, | |
| ) | |
| def build_summary( | |
| df: pd.DataFrame, | |
| algorithm_version: str, | |
| mapping_version: str, | |
| ) -> pd.DataFrame: | |
| total = len(df) | |
| scored = df[ | |
| df["nutri_score_calculated"].astype("boolean").fillna(False) | |
| ] | |
| grades = scored["nutri_score_grade"] | |
| summary = { | |
| "total_products": total, | |
| "eligible_products": int( | |
| df["score_eligibility"].fillna(False).sum() | |
| ), | |
| "scored_products": len(scored), | |
| "not_eligible_products": int( | |
| (~df["score_eligibility"].fillna(False)).sum() | |
| ), | |
| "scored_A": int((grades == "A").sum()), | |
| "scored_B": int((grades == "B").sum()), | |
| "scored_C": int((grades == "C").sum()), | |
| "scored_D": int((grades == "D").sum()), | |
| "scored_E": int((grades == "E").sum()), | |
| "fvl_available": int( | |
| df["fvl_percent"].notna().sum() | |
| ), | |
| "fvl_not_available": int( | |
| df["fvl_percent"].isna().sum() | |
| ), | |
| "fvl_estimated": int( | |
| (df["fvl_provenance"] == "ESTIMATED").sum() | |
| ), | |
| "agribalyse_mapped": int( | |
| (df["mapping_status"] == "MAPPED").sum() | |
| ), | |
| "agribalyse_not_available": int( | |
| (df["mapping_status"] == "NOT_AVAILABLE").sum() | |
| ), | |
| "algorithm_version": algorithm_version, | |
| "mapping_version": mapping_version, | |
| } | |
| return pd.DataFrame([summary]) | |
| def write_phase6_outputs( | |
| df: pd.DataFrame, | |
| output_dir: str | Path = "data/phase_6", | |
| algorithm_version: str = "Nutri-Score-2023", | |
| mapping_version: str = "phase6-taxonomy-v1", | |
| ) -> dict[str, Path]: | |
| output_dir = Path(output_dir) | |
| output_dir.mkdir(parents=True, exist_ok=True) | |
| product_scores = build_product_scores(df) | |
| agribalyse_mapping = build_agribalyse_mapping(df) | |
| exclusions = build_score_exclusions(df) | |
| summary = build_summary( | |
| df, | |
| algorithm_version, | |
| mapping_version, | |
| ) | |
| paths = { | |
| "product_scores": output_dir / "product_scores.parquet", | |
| "agribalyse_mapping": output_dir / "agribalyse_mapping.parquet", | |
| "score_exclusions": output_dir / "score_exclusions.parquet", | |
| "phase6_summary": output_dir / "phase6_summary.parquet", | |
| } | |
| product_scores.to_parquet( | |
| paths["product_scores"], | |
| index=False, | |
| ) | |
| agribalyse_mapping.to_parquet( | |
| paths["agribalyse_mapping"], | |
| index=False, | |
| ) | |
| exclusions.to_parquet( | |
| paths["score_exclusions"], | |
| index=False, | |
| ) | |
| summary.to_parquet( | |
| paths["phase6_summary"], | |
| index=False, | |
| ) | |
| return paths | |