saraNour's picture
Upload folder using huggingface_hub
1cd137c verified
Raw History Blame Contribute Delete
23.6 kB
"""
Phase 6 — Nutri-Score + Environmental/Agribalyse Mapping
Algorithm: Nutri-Score 2023 (updated algorithm)
Source: Eurofins referencing Santé Publique France FAQ v21.Dec.2023
Authoritative source: https://www.eurofins.de/food-analysis/other-services/nutri-score/
This module implements:
1. Nutri-Score 2023 calculation (general food + beverage categories)
2. FVL estimation from taxonomy
3. Agribalyse category-level mapping
4. Output table creation with full provenance
DO NOT modify Phase 1–5 production code or outputs.
"""
import os
from datetime import UTC, datetime
import pandas as pd
# CONSTANTS
ALGORITHM_VERSION = "nutri_score_2023"
ALGORITHM_SOURCE = "Eurofins referencing Santé Publique France FAQ v21.Dec.2023"
ALGORITHM_SOURCE_URL = "https://www.eurofins.de/food-analysis/other-services/nutri-score/"
MAPPING_SOURCE = "agribalyse_v3.2"
MAPPING_VERSION = "3.2"
MAPPING_NOTE = "Category-level proxy. Not a product-specific LCA."
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
PROJECT_DIR = os.path.dirname(BASE_DIR)
REQUIRED_FIELDS = [
'calories_per_100g', 'sugars_g_per_100g', 'saturated_fat_g_per_100g',
'sodium_mg_per_100g', 'fibre_g_per_100g', 'protein_g_per_100g'
]
# NUTRI-SCORE 2023 POINT TABLES (General Food)
# Negative points: energy_kj, saturated_fat_g, sugar_g, salt_g
# Each table: list of (upper_bound, points) — score is points where value <= upper_bound
# Using > threshold semantics: score = highest points where value > threshold
GENERAL_FOOD_NEGATIVE = {
'energy_kj': [
(335, 0), (670, 1), (1005, 2), (1340, 3), (1675, 4),
(2010, 5), (2345, 6), (2680, 7), (3015, 8), (3350, 9), (float('inf'), 10)
],
'saturated_fat_g': [
(1, 0), (2, 1), (3, 2), (4, 3), (5, 4),
(6, 5), (7, 6), (8, 7), (9, 8), (10, 9), (float('inf'), 10)
],
'sugar_g': [
(3.4, 0), (6.8, 1), (10, 2), (14, 3), (17, 4),
(20, 5), (24, 6), (27, 7), (31, 8), (34, 9),
(37, 10), (41, 11), (44, 12), (48, 13), (51, 14), (float('inf'), 15)
],
'salt_g': [
(0.2, 0), (0.4, 1), (0.6, 2), (0.8, 3), (1.0, 4),
(1.2, 5), (1.4, 6), (1.6, 7), (1.8, 8), (2.0, 9),
(2.2, 10), (2.4, 11), (2.6, 12), (2.8, 13), (3.0, 14),
(3.2, 15), (3.4, 16), (3.6, 17), (3.8, 18), (4.0, 19), (float('inf'), 20)
]
}
# Positive points: protein_g, fibre_g, fvl_g_per_100g
GENERAL_FOOD_POSITIVE = {
'protein_g': [
(2.4, 0), (4.8, 1), (7.2, 2), (9.6, 3), (12, 4),
(14, 5), (17, 6), (float('inf'), 7)
],
'fibre_g': [
(3.0, 0), (4.1, 1), (5.2, 2), (6.3, 3), (7.4, 4), (float('inf'), 5)
],
'fvl_g_per_100g': [
(40, 0), (60, 1), (80, 2), (float('inf'), 5)
]
}
# Beverage point tables
BEVERAGE_NEGATIVE = {
'energy_kj': [
(30, 0), (90, 1), (150, 2), (210, 3), (240, 4),
(270, 5), (300, 6), (330, 7), (360, 8), (390, 9), (float('inf'), 10)
],
'saturated_fat_g': [
(1, 0), (2, 1), (3, 2), (4, 3), (5, 4),
(6, 5), (7, 6), (8, 7), (9, 8), (10, 9), (float('inf'), 10)
],
'sugar_g': [
(0.5, 0), (2, 1), (3.5, 2), (5, 3), (6, 4),
(7, 5), (8, 6), (9, 7), (10, 8), (11, 9), (float('inf'), 10)
],
'salt_g': [
(0.2, 0), (0.4, 1), (0.6, 2), (0.8, 3), (1.0, 4),
(1.2, 5), (1.4, 6), (1.6, 7), (1.8, 8), (2.0, 9),
(2.2, 10), (2.4, 11), (2.6, 12), (2.8, 13), (3.0, 14),
(3.2, 15), (3.4, 16), (3.6, 17), (3.8, 18), (4.0, 19), (float('inf'), 20)
]
}
BEVERAGE_POSITIVE = {
'protein_g': [
(1.2, 0), (1.5, 1), (1.8, 2), (2.1, 3), (2.4, 4),
(2.7, 5), (3.0, 6), (float('inf'), 7)
],
'fibre_g': [
(3.0, 0), (4.1, 1), (5.2, 2), (6.3, 3), (7.4, 4), (float('inf'), 5)
],
'fvl_percent': [
(40, 0), (60, 2), (80, 4), (float('inf'), 10)
]
}
# Grade boundaries
GRADE_BOUNDARIES = {
'general_food': [(-float('inf'), 0, 'A'), (0, 2, 'B'), (2, 10, 'C'), (10, 18, 'D'), (18, float('inf'), 'E')],
'beverages': [(-float('inf'), 0, 'A'), (0, 2, 'B'), (2, 6, 'C'), (6, 9, 'D'), (9, float('inf'), 'E')]
}
# FVL estimation
FVL_ESTIMATE = {
'PRODUCE': 90,
'default': 0
}
# Agribalyse category mapping
AGRIBALYSE_MAP = {
'DAIRY': {'category': 'Produits laitiers et fromages', 'ciqual': '19xxx', 'confidence': 'HIGH'},
'MEAT_SEAFOOD': {'category': 'Viandes, poissons et oeufs', 'ciqual': '25xxx/31xxx', 'confidence': 'HIGH'},
'PRODUCE': {'category': 'Fruits et légumes', 'ciqual': '13xxx', 'confidence': 'HIGH'},
'BEVERAGES': {'category': 'Boissons', 'ciqual': '14xxx', 'confidence': 'HIGH'},
'CONFECTIONERY': {'category': 'Confiseries et chocolat', 'ciqual': '22xxx', 'confidence': 'HIGH'},
'BAKERY': {'category': 'Boulangerie et pâtisserie', 'ciqual': '07xxx', 'confidence': 'MEDIUM'},
'BREAKFAST': {'category': 'Petit-déjeuner et céréales', 'ciqual': '08xxx', 'confidence': 'MEDIUM'},
'SNACKS': {'category': 'Snacks salés', 'ciqual': '23xxx', 'confidence': 'MEDIUM'},
'CONDIMENTS_SAUCES': {'category': 'Condiments et sauces', 'ciqual': '11xxx', 'confidence': 'MEDIUM'},
'FROZEN': {'category': 'Produits surgelés', 'ciqual': 'Multiple', 'confidence': 'MEDIUM'},
'PASTA_RICE': {'category': 'Féculents et légumineuses', 'ciqual': '09xxx', 'confidence': 'MEDIUM'},
'GENERAL_GROCERY': {'category': 'Multiple categories', 'ciqual': 'Various', 'confidence': 'LOW'},
'HEALTH_REMEDIES': {'category': 'NONE', 'ciqual': 'NONE', 'confidence': 'NONE'},
'BABY_CARE': {'category': 'NONE', 'ciqual': 'NONE', 'confidence': 'NONE'},
'HOUSEHOLD_CLEANING': {'category': 'NONE', 'ciqual': 'NONE', 'confidence': 'NONE'},
'PERSONAL_CARE': {'category': 'NONE', 'ciqual': 'NONE', 'confidence': 'NONE'},
'PET_FOOD': {'category': 'NONE', 'ciqual': 'NONE', 'confidence': 'NONE'},
'HOUSEHOLD_SUPPLIES': {'category': 'NONE', 'ciqual': 'NONE', 'confidence': 'NONE'},
}
# SCORING FUNCTIONS
def get_points(value, table):
"""Get points from a threshold table. Returns points where value <= threshold."""
if pd.isna(value):
return None
for threshold, points in table:
if value <= threshold:
return points
return table[-1][1] # Last entry's points (max)
def calculate_nutri_score_general(energy_kj, sat_fat_g, sugar_g, salt_g,
fibre_g, protein_g, fvl_g):
"""
Calculate Nutri-Score 2023 for general food category.
Returns: (negative_points, positive_points, raw_score, grade)
"""
# Negative points
n_energy = get_points(energy_kj, GENERAL_FOOD_NEGATIVE['energy_kj'])
n_sat_fat = get_points(sat_fat_g, GENERAL_FOOD_NEGATIVE['saturated_fat_g'])
n_sugar = get_points(sugar_g, GENERAL_FOOD_NEGATIVE['sugar_g'])
n_salt = get_points(salt_g, GENERAL_FOOD_NEGATIVE['salt_g'])
if None in [n_energy, n_sat_fat, n_sugar, n_salt]:
return None, None, None, None
negative = n_energy + n_sat_fat + n_sugar + n_salt
# Positive points
p_protein = get_points(protein_g, GENERAL_FOOD_POSITIVE['protein_g'])
p_fibre = get_points(fibre_g, GENERAL_FOOD_POSITIVE['fibre_g'])
p_fvl = get_points(fvl_g, GENERAL_FOOD_POSITIVE['fvl_g_per_100g'])
if None in [p_protein, p_fibre, p_fvl]:
return None, None, None, None
positive = p_protein + p_fibre + p_fvl
# Final score
raw_score = negative - positive
# Grade
grade = None
for lower, upper, g in GRADE_BOUNDARIES['general_food']:
if lower < raw_score <= upper:
grade = g
break
if grade is None:
grade = 'E' # fallback
return negative, positive, raw_score, grade
def calculate_nutri_score_beverage(energy_kj, sat_fat_g, sugar_g, salt_g,
fibre_g, protein_g, fvl_percent):
"""
Calculate Nutri-Score 2023 for beverage category.
Returns: (negative_points, positive_points, raw_score, grade)
"""
# Negative points
n_energy = get_points(energy_kj, BEVERAGE_NEGATIVE['energy_kj'])
n_sat_fat = get_points(sat_fat_g, BEVERAGE_NEGATIVE['saturated_fat_g'])
n_sugar = get_points(sugar_g, BEVERAGE_NEGATIVE['sugar_g'])
n_salt = get_points(salt_g, BEVERAGE_NEGATIVE['salt_g'])
if None in [n_energy, n_sat_fat, n_sugar, n_salt]:
return None, None, None, None
negative = n_energy + n_sat_fat + n_sugar + n_salt
# Positive points
p_protein = get_points(protein_g, BEVERAGE_POSITIVE['protein_g'])
p_fibre = get_points(fibre_g, BEVERAGE_POSITIVE['fibre_g'])
p_fvl = get_points(fvl_percent, BEVERAGE_POSITIVE['fvl_percent'])
if None in [p_protein, p_fibre, p_fvl]:
return None, None, None, None
positive = p_protein + p_fibre + p_fvl
# Final score
raw_score = negative - positive
# Grade
grade = None
for lower, upper, g in GRADE_BOUNDARIES['beverages']:
if lower < raw_score <= upper:
grade = g
break
if grade is None:
grade = 'E' # fallback
return negative, positive, raw_score, grade
# FVL ESTIMATION
def estimate_fvl(taxonomy):
"""Estimate FVL percentage from taxonomy."""
if taxonomy == 'PRODUCE':
return 90, 'taxonomy_estimate', 'LOW', 'not_in_data'
return 0, 'taxonomy_estimate', 'LOW', 'not_in_data'
# MAIN PIPELINE
def run_phase6():
"""Execute the complete Phase 6 pipeline."""
print("=" * 60)
print("PHASE 6 — Nutri-Score + Environmental/Agribalyse Mapping")
print("=" * 60)
print(f"Algorithm: {ALGORITHM_VERSION}")
print(f"Source: {ALGORITHM_SOURCE}")
print(f"Date: {datetime.now(UTC).strftime('%Y-%m-%d %H:%M')}")
print()
# 1. LOAD INPUTS
print("[1/8] Loading inputs...")
phase5_dir = os.path.join(PROJECT_DIR, 'phase5', 'outputs')
phase3_dir = os.path.join(PROJECT_DIR, 'phase3', 'outputs')
phase4_dir = os.path.join(PROJECT_DIR, 'phase4', 'outputs')
np100 = pd.read_parquet(os.path.join(phase5_dir, 'nutrition_per_100g.parquet'))
nc = pd.read_parquet(os.path.join(phase5_dir, 'nutrition_cleaned.parquet'))
pgm = pd.read_csv(os.path.join(phase3_dir, 'product_group_mapping.csv'), dtype={"upc": "string"})
pvm = pd.read_parquet(os.path.join(phase4_dir, 'product_variant_mapping.parquet'))
print(f" nutrition_per_100g: {np100.shape}")
print(f" nutrition_cleaned: {nc.shape}")
print(f" product_group_mapping: {pgm.shape}")
print(f" product_variant_mapping: {pvm.shape}")
# 2. MERGE DATA
print("\n[2/8] Merging data...")
domain_map = pgm[['external_id', 'product_domain', 'reference_db_taxonomy', 'group_id', 'core_title']].drop_duplicates('external_id')
nc_status = nc[['external_id', 'nutrition_quality_status', 'nutrition_quality_flags']].drop_duplicates('external_id')
variant_map = pvm[['external_id', 'variant_id']].drop_duplicates('external_id')
df = np100.merge(domain_map, on='external_id', how='left')
df = df.merge(nc_status, on='external_id', how='left')
df = df.merge(variant_map, on='external_id', how='left')
print(f" Merged: {df.shape}")
# 3. ELIGIBILITY
print("\n[3/8] Computing eligibility...")
df['has_all_required'] = df[REQUIRED_FIELDS].notna().all(axis=1)
def get_eligibility(row):
if row['product_domain'] == 'non_food':
return 'NOT_ELIGIBLE', 'NON_FOOD'
if row['product_domain'] == 'unknown':
return 'NOT_ELIGIBLE', 'UNKNOWN_DOMAIN'
if row['nutrition_quality_status'] == 'MISSING':
return 'NOT_ELIGIBLE', 'MISSING_NUTRITION'
if row['nutrition_quality_status'] == 'SUSPICIOUS':
return 'NOT_ELIGIBLE', 'SUSPICIOUS_NUTRITION'
if not row['has_all_required']:
missing = [f.replace('_per_100g', '') for f in REQUIRED_FIELDS if pd.isna(row[f])]
return 'NOT_ELIGIBLE', f"MISSING_REQUIRED_FIELDS: {', '.join(missing)}"
return 'ELIGIBLE', 'NONE'
df['score_eligibility'], df['score_exclusion_reason'] = zip(*df.apply(get_eligibility, axis=1))
elig_counts = df['score_eligibility'].value_counts()
print(f" ELIGIBLE: {elig_counts.get('ELIGIBLE', 0)}")
print(f" NOT_ELIGIBLE: {elig_counts.get('NOT_ELIGIBLE', 0)}")
# 4. FVL ESTIMATION
print("\n[4/8] Estimating FVL...")
df['fvl_percent'], df['fvl_method'], df['fvl_confidence'], df['fvl_source'] = zip(
*df['reference_db_taxonomy'].apply(estimate_fvl)
)
fvl_dist = df['fvl_percent'].value_counts().sort_index()
print(f" FVL distribution: {dict(fvl_dist)}")
# 5. NUTRI-SCORE CALCULATION
print("\n[5/8] Calculating Nutri-Score...")
# Convert units
df['energy_kj_100g'] = df['calories_per_100g'] * 4.184
df['salt_g_100g'] = df['sodium_mg_per_100g'] * 2.5 / 1000
# Initialize columns
df['negative_points'] = None
df['positive_points'] = None
df['nutri_score_raw'] = None
df['nutri_score_grade'] = None
df['nutri_score_algorithm'] = ALGORITHM_VERSION
df['nutri_score_version'] = '2023'
df['nutri_score_calculated'] = False
# Calculate for eligible products
for idx, row in df.iterrows():
if row['score_eligibility'] != 'ELIGIBLE':
continue
is_beverage = row['reference_db_taxonomy'] == 'BEVERAGES'
if is_beverage:
neg, pos, raw, grade = calculate_nutri_score_beverage(
row['energy_kj_100g'], row['saturated_fat_g_per_100g'],
row['sugars_g_per_100g'], row['salt_g_100g'],
row['fibre_g_per_100g'], row['protein_g_per_100g'],
row['fvl_percent']
)
else:
fvl_g = row['fvl_percent'] / 100 * 100 # Convert % to g/100g (same as %)
neg, pos, raw, grade = calculate_nutri_score_general(
row['energy_kj_100g'], row['saturated_fat_g_per_100g'],
row['sugars_g_per_100g'], row['salt_g_100g'],
row['fibre_g_per_100g'], row['protein_g_per_100g'],
fvl_g
)
if neg is not None:
df.at[idx, 'negative_points'] = neg
df.at[idx, 'positive_points'] = pos
df.at[idx, 'nutri_score_raw'] = raw
df.at[idx, 'nutri_score_grade'] = grade
df.at[idx, 'nutri_score_calculated'] = True
scored = df[df['nutri_score_calculated']].shape[0]
print(f" Scored: {scored}")
# Grade distribution
grade_dist = df[df['nutri_score_calculated']]['nutri_score_grade'].value_counts().sort_index()
print(" Grade distribution:")
for g, c in grade_dist.items():
print(f" {g}: {c}")
# 6. AGRIBALYSE MAPPING
print("\n[6/8] Mapping Agribalyse categories...")
def map_agribalyse(taxonomy):
if taxonomy in AGRIBALYSE_MAP:
m = AGRIBALYSE_MAP[taxonomy]
return (m['category'], m['ciqual'], 'deterministic_taxonomy',
m['confidence'], MAPPING_SOURCE, MAPPING_VERSION, MAPPING_NOTE)
return ('NONE', 'NONE', 'deterministic_taxonomy', 'NONE',
MAPPING_SOURCE, MAPPING_VERSION, MAPPING_NOTE)
df['agribalyse_category'], df['agribalyse_ciqual'], df['mapping_method'], \
df['mapping_confidence'], df['mapping_source'], df['mapping_version'], \
df['mapping_note'] = zip(*df['reference_db_taxonomy'].apply(map_agribalyse))
conf_dist = df['mapping_confidence'].value_counts()
print(" Mapping confidence distribution:")
for c, n in conf_dist.items():
print(f" {c}: {n}")
# 7. CREATE OUTPUT TABLES
print("\n[7/8] Creating output tables...")
# Table 1: phase6_product_scores.parquet (product-level, one row per external_id)
product_scores = df[[
'external_id', 'group_id', 'variant_id', 'upc', 'core_title', 'product_domain', 'reference_db_taxonomy',
'nutrition_quality_status', 'nutrition_quality_flags',
'score_eligibility', 'score_exclusion_reason',
'calories_per_100g', 'energy_kj_100g', 'sugars_g_per_100g',
'saturated_fat_g_per_100g', 'sodium_mg_per_100g', 'salt_g_100g',
'fibre_g_per_100g', 'protein_g_per_100g',
'fvl_percent', 'fvl_method', 'fvl_confidence', 'fvl_source',
'negative_points', 'positive_points', 'nutri_score_raw', 'nutri_score_grade',
'nutri_score_algorithm', 'nutri_score_version', 'nutri_score_calculated',
'normalization_method'
]].copy()
# Table 2: phase6_agribalyse_mapping.parquet
agribalyse_mapping = df[[
'external_id', 'group_id', 'reference_db_taxonomy', 'product_domain',
'agribalyse_category', 'agribalyse_ciqual',
'mapping_method', 'mapping_confidence', 'mapping_source',
'mapping_version', 'mapping_note'
]].copy()
# Table 3: phase6_score_exclusions.parquet
exclusions = df[df['score_eligibility'] == 'NOT_ELIGIBLE'][[
'external_id', 'group_id', 'product_domain', 'reference_db_taxonomy',
'nutrition_quality_status', 'score_exclusion_reason'
]].copy()
# Table 4: phase6_review_queue.parquet (empty for now — no ambiguous cases)
review_queue = pd.DataFrame(columns=[
'external_id', 'group_id', 'reference_db_taxonomy', 'review_reason',
'review_status', 'review_notes'
])
# Table 5: phase6_summary.parquet
summary_data = {
'metric': [
'total_products', 'eligible', 'not_eligible',
'scored_a', 'scored_b', 'scored_c', 'scored_d', 'scored_e',
'agribalyse_high', 'agribalyse_medium', 'agribalyse_low', 'agribalyse_none',
'fvl_90_percent', 'fvl_0_percent',
'suspicious_excluded', 'missing_nutrition_excluded',
'non_food_excluded', 'unknown_domain_excluded',
'algorithm_version', 'mapping_source'
],
'value': [
df.shape[0],
int(elig_counts.get('ELIGIBLE', 0)),
int(elig_counts.get('NOT_ELIGIBLE', 0)),
int(grade_dist.get('A', 0)),
int(grade_dist.get('B', 0)),
int(grade_dist.get('C', 0)),
int(grade_dist.get('D', 0)),
int(grade_dist.get('E', 0)),
int(conf_dist.get('HIGH', 0)),
int(conf_dist.get('MEDIUM', 0)),
int(conf_dist.get('LOW', 0)),
int(conf_dist.get('NONE', 0)),
int(df[df['fvl_percent'] == 90].shape[0]),
int(df[df['fvl_percent'] == 0].shape[0]),
int(df[df['score_exclusion_reason'] == 'SUSPICIOUS_NUTRITION'].shape[0]),
int(df[df['score_exclusion_reason'] == 'MISSING_NUTRITION'].shape[0]),
int(df[df['score_exclusion_reason'] == 'NON_FOOD'].shape[0]),
int(df[df['score_exclusion_reason'] == 'UNKNOWN_DOMAIN'].shape[0]),
ALGORITHM_VERSION,
MAPPING_SOURCE
]
}
summary = pd.DataFrame(summary_data)
summary['value'] = summary['value'].astype(str)
# 8. SAVE OUTPUTS
print("\n[8/8] Saving outputs...")
output_dir = os.path.join(BASE_DIR, 'outputs')
os.makedirs(output_dir, exist_ok=True)
product_scores.to_parquet(os.path.join(output_dir, 'phase6_product_scores.parquet'), index=False)
agribalyse_mapping.to_parquet(os.path.join(output_dir, 'phase6_agribalyse_mapping.parquet'), index=False)
exclusions.to_parquet(os.path.join(output_dir, 'phase6_score_exclusions.parquet'), index=False)
review_queue.to_parquet(os.path.join(output_dir, 'phase6_review_queue.parquet'), index=False)
summary.to_parquet(os.path.join(output_dir, 'phase6_summary.parquet'), index=False)
# Also save as CSV for audit
product_scores.to_csv(os.path.join(output_dir, 'phase6_product_scores.csv'), index=False)
exclusions.to_csv(os.path.join(output_dir, 'phase6_score_exclusions.csv'), index=False)
summary.to_csv(os.path.join(output_dir, 'phase6_summary.csv'), index=False)
print(f" Saved to: {output_dir}/")
print(f" - phase6_product_scores.parquet ({product_scores.shape})")
print(f" - phase6_agribalyse_mapping.parquet ({agribalyse_mapping.shape})")
print(f" - phase6_score_exclusions.parquet ({exclusions.shape})")
print(f" - phase6_review_queue.parquet ({review_queue.shape})")
print(f" - phase6_summary.parquet ({summary.shape})")
# VALIDATION
print("\n" + "=" * 60)
print("VALIDATION")
print("=" * 60)
errors = []
# Identity checks
if product_scores.shape[0] != 4440:
errors.append(f"Row count: expected 4440, got {product_scores.shape[0]}")
if product_scores['external_id'].nunique() != 4440:
errors.append(f"Unique external_ids: expected 4440, got {product_scores['external_id'].nunique()}")
# No non_food scored
non_food_scored = product_scores[(product_scores['product_domain'] == 'non_food') &
(product_scores['nutri_score_calculated'] == True)]
if len(non_food_scored) > 0:
errors.append(f"Non-food products scored: {len(non_food_scored)}")
# No unknown scored
unknown_scored = product_scores[(product_scores['product_domain'] == 'unknown') &
(product_scores['nutri_score_calculated'] == True)]
if len(unknown_scored) > 0:
errors.append(f"Unknown domain products scored: {len(unknown_scored)}")
# No suspicious scored
suspicious_scored = product_scores[(product_scores['nutrition_quality_status'] == 'SUSPICIOUS') &
(product_scores['nutri_score_calculated'] == True)]
if len(suspicious_scored) > 0:
errors.append(f"Suspicious products scored: {len(suspicious_scored)}")
# Valid grades
valid_grades = {'A', 'B', 'C', 'D', 'E'}
scored_grades = set(product_scores[product_scores['nutri_score_calculated']]['nutri_score_grade'].unique())
if not scored_grades.issubset(valid_grades):
errors.append(f"Invalid grades: {scored_grades - valid_grades}")
if errors:
print("\nVALIDATION ERRORS:")
for e in errors:
print(f" ERROR: {e}")
else:
print("\nALL VALIDATION CHECKS PASSED")
# STATISTICS
print("\n" + "=" * 60)
print("STATISTICS")
print("=" * 60)
print("\nInput verification:")
print(f" Total products: {df.shape[0]} (expected 4440)")
print(f" Food: {df[df['product_domain']=='food'].shape[0]} (expected 2803)")
print(f" Unknown: {df[df['product_domain']=='unknown'].shape[0]} (expected 1243)")
print(f" Non-food: {df[df['product_domain']=='non_food'].shape[0]} (expected 394)")
print("\nNutri-Score eligibility:")
print(f" Eligible: {elig_counts.get('ELIGIBLE', 0)} (8.2% of total)")
print(f" Not eligible: {elig_counts.get('NOT_ELIGIBLE', 0)} (91.8% of total)")
print("\nNutri-Score results (among eligible):")
for g in ['A', 'B', 'C', 'D', 'E']:
c = grade_dist.get(g, 0)
total_eligible = elig_counts.get('ELIGIBLE', 0)
print(f" {g}: {c} ({c/total_eligible*100:.1f}%)")
print("\nFVL estimation:")
print(f" 90% (PRODUCE): {df[df['fvl_percent']==90].shape[0]}")
print(f" 0% (other): {df[df['fvl_percent']==0].shape[0]}")
print("\nAgribalyse mapping:")
for c in ['HIGH', 'MEDIUM', 'LOW', 'NONE']:
n = conf_dist.get(c, 0)
print(f" {c}: {n} ({n/df.shape[0]*100:.1f}%)")
print("\nLLM usage: 0 (deterministic algorithm only)")
return product_scores, agribalyse_mapping, exclusions, review_queue, summary
if __name__ == '__main__':
run_phase6()