saraNour commited on
Commit
8aea0a3
·
verified ·
1 Parent(s): 9e62dd0

Upload notebooks/phase5_executed.ipynb with huggingface_hub

Browse files
Files changed (1) hide show
  1. notebooks/phase5_executed.ipynb +441 -0
notebooks/phase5_executed.ipynb ADDED
@@ -0,0 +1,441 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "cells": [
3
+ {
4
+ "cell_type": "markdown",
5
+ "metadata": {},
6
+ "source": [
7
+ "# Phase 5 — Nutrition Integration\n",
8
+ "\n",
9
+ "Compliments Reference DB Pipeline\n",
10
+ "\n",
11
+ "**Inputs:**\n",
12
+ "- `saraNour/compliments-brand/source_of_truth/nutrition.parquet`\n",
13
+ "- `saraNour/compliments-brand/source_of_truth/products.parquet`\n",
14
+ "- Phase 3 product_group_mapping\n",
15
+ "- Phase 4 product_variant_mapping\n",
16
+ "\n",
17
+ "**Outputs:**\n",
18
+ "- `product_group_mapping.parquet`\n",
19
+ "- `nutrition_cleaned.parquet`\n",
20
+ "- `product_nutrition_mapping.parquet`\n",
21
+ "- `nutrition_per_100g.parquet`\n",
22
+ "- `phase5_statistics.parquet`\n",
23
+ "- `phase5_validation.parquet`\n",
24
+ "\n",
25
+ "**No LLM. No external data. All Parquet.**"
26
+ ]
27
+ },
28
+ {
29
+ "cell_type": "code",
30
+ "execution_count": 1,
31
+ "metadata": {
32
+ "execution": {
33
+ "iopub.execute_input": "2026-07-30T18:32:15.700314Z",
34
+ "iopub.status.busy": "2026-07-30T18:32:15.698833Z",
35
+ "iopub.status.idle": "2026-07-30T18:32:18.857609Z",
36
+ "shell.execute_reply": "2026-07-30T18:32:18.854550Z"
37
+ }
38
+ },
39
+ "outputs": [
40
+ {
41
+ "name": "stdout",
42
+ "output_type": "stream",
43
+ "text": [
44
+ "Setup complete.\n"
45
+ ]
46
+ }
47
+ ],
48
+ "source": [
49
+ "!pip install -q pandas pyarrow huggingface_hub\n",
50
+ "\n",
51
+ "import pandas as pd\n",
52
+ "import numpy as np\n",
53
+ "import json\n",
54
+ "import os\n",
55
+ "from pathlib import Path\n",
56
+ "from huggingface_hub import hf_hub_download\n",
57
+ "\n",
58
+ "NUTRITION_FIELDS = [\n",
59
+ " 'calories', 'carbohydrate_g', 'sugars_g', 'sodium_mg', 'potassium_mg',\n",
60
+ " 'fat_g', 'saturated_fat_g', 'polyunsaturated_fat_g', 'omega6_g', 'omega3_g',\n",
61
+ " 'monounsaturated_fat_g', 'sugar_alcohols_g', 'fibre_g', 'protein_g',\n",
62
+ " 'calcium_mg', 'iron_mg', 'cholesterol_mg'\n",
63
+ "]\n",
64
+ "\n",
65
+ "SUSPICIOUS_THRESHOLDS = {\n",
66
+ " 'calories': 1000,\n",
67
+ " 'fat_g': 100,\n",
68
+ " 'protein_g': 100,\n",
69
+ "}\n",
70
+ "\n",
71
+ "print('Setup complete.')"
72
+ ]
73
+ },
74
+ {
75
+ "cell_type": "markdown",
76
+ "metadata": {},
77
+ "source": [
78
+ "## 1. Load Data"
79
+ ]
80
+ },
81
+ {
82
+ "cell_type": "code",
83
+ "execution_count": 2,
84
+ "metadata": {
85
+ "execution": {
86
+ "iopub.execute_input": "2026-07-30T18:32:18.986260Z",
87
+ "iopub.status.busy": "2026-07-30T18:32:18.980455Z",
88
+ "iopub.status.idle": "2026-07-30T18:32:19.896109Z",
89
+ "shell.execute_reply": "2026-07-30T18:32:19.893777Z"
90
+ }
91
+ },
92
+ "outputs": [
93
+ {
94
+ "name": "stdout",
95
+ "output_type": "stream",
96
+ "text": [
97
+ "nutrition: 4440 rows, 21 cols\n",
98
+ "products: 4440 rows, 16 cols\n"
99
+ ]
100
+ }
101
+ ],
102
+ "source": [
103
+ "nutrition_path = hf_hub_download(repo_id='saraNour/compliments-brand', filename='source_of_truth/nutrition.parquet', repo_type='dataset')\n",
104
+ "nutrition = pd.read_parquet(nutrition_path)\n",
105
+ "print(f'nutrition: {len(nutrition)} rows, {len(nutrition.columns)} cols')\n",
106
+ "\n",
107
+ "products_path = hf_hub_download(repo_id='saraNour/compliments-brand', filename='source_of_truth/products.parquet', repo_type='dataset')\n",
108
+ "products = pd.read_parquet(products_path)\n",
109
+ "print(f'products: {len(products)} rows, {len(products.columns)} cols')"
110
+ ]
111
+ },
112
+ {
113
+ "cell_type": "markdown",
114
+ "metadata": {},
115
+ "source": [
116
+ "## 2. Nutrition Cleaning & Validation"
117
+ ]
118
+ },
119
+ {
120
+ "cell_type": "code",
121
+ "execution_count": 3,
122
+ "metadata": {
123
+ "execution": {
124
+ "iopub.execute_input": "2026-07-30T18:32:19.902027Z",
125
+ "iopub.status.busy": "2026-07-30T18:32:19.900470Z",
126
+ "iopub.status.idle": "2026-07-30T18:32:19.963149Z",
127
+ "shell.execute_reply": "2026-07-30T18:32:19.959573Z"
128
+ }
129
+ },
130
+ "outputs": [
131
+ {
132
+ "name": "stdout",
133
+ "output_type": "stream",
134
+ "text": [
135
+ "Quality status distribution:\n",
136
+ "nutrition_quality_status\n",
137
+ "VALID 2932\n",
138
+ "MISSING 1505\n",
139
+ "SUSPICIOUS 3\n",
140
+ "Name: count, dtype: int64\n",
141
+ "\n",
142
+ "Suspicious records:\n",
143
+ " 274613EA: Compliments Extra Old Cheddar Cheese 300 g... flags=fat_g > 100\n",
144
+ " 733225EA: Compliments Mushroom Whole White 680 g... flags=calories > 1000\n",
145
+ " 218890KG: Compliments Whole Chicken Grade A... flags=protein_g > 100\n"
146
+ ]
147
+ }
148
+ ],
149
+ "source": [
150
+ "df = nutrition.copy()\n",
151
+ "df['nutrition_quality_status'] = 'VALID'\n",
152
+ "df['nutrition_quality_flags'] = ''\n",
153
+ "\n",
154
+ "# Flag all-null nutrition\n",
155
+ "all_null = df[NUTRITION_FIELDS].isna().all(axis=1)\n",
156
+ "df.loc[all_null, 'nutrition_quality_status'] = 'MISSING'\n",
157
+ "df.loc[all_null, 'nutrition_quality_flags'] = 'all_nutrition_fields_null'\n",
158
+ "\n",
159
+ "# Flag suspicious values\n",
160
+ "for field, threshold in SUSPICIOUS_THRESHOLDS.items():\n",
161
+ " sus = (df[field] > threshold) & (df['nutrition_quality_status'] != 'MISSING')\n",
162
+ " df.loc[sus, 'nutrition_quality_status'] = 'SUSPICIOUS'\n",
163
+ " df.loc[sus, 'nutrition_quality_flags'] = field + ' > ' + str(threshold)\n",
164
+ "\n",
165
+ "print('Quality status distribution:')\n",
166
+ "print(df['nutrition_quality_status'].value_counts())\n",
167
+ "\n",
168
+ "print('\\nSuspicious records:')\n",
169
+ "for _, row in df[df['nutrition_quality_status'] == 'SUSPICIOUS'].iterrows():\n",
170
+ " print(f' {row[\"external_id\"]}: {row[\"title\"][:50]}... flags={row[\"nutrition_quality_flags\"]}')"
171
+ ]
172
+ },
173
+ {
174
+ "cell_type": "markdown",
175
+ "metadata": {},
176
+ "source": [
177
+ "## 3. Nutrition Normalization (per 100g)"
178
+ ]
179
+ },
180
+ {
181
+ "cell_type": "code",
182
+ "execution_count": 4,
183
+ "metadata": {
184
+ "execution": {
185
+ "iopub.execute_input": "2026-07-30T18:32:19.970260Z",
186
+ "iopub.status.busy": "2026-07-30T18:32:19.968116Z",
187
+ "iopub.status.idle": "2026-07-30T18:32:20.308532Z",
188
+ "shell.execute_reply": "2026-07-30T18:32:20.306581Z"
189
+ }
190
+ },
191
+ "outputs": [
192
+ {
193
+ "name": "stdout",
194
+ "output_type": "stream",
195
+ "text": [
196
+ "Normalization results:\n",
197
+ "normalization_method\n",
198
+ "not_normalized 2939\n",
199
+ "scaled_to_100g 1351\n",
200
+ "direct_100g 150\n",
201
+ "Name: count, dtype: int64\n"
202
+ ]
203
+ }
204
+ ],
205
+ "source": [
206
+ "def parse_serving_size(ss):\n",
207
+ " if pd.isna(ss) or ss == '':\n",
208
+ " return None, None\n",
209
+ " ss = str(ss).strip()\n",
210
+ " if ss in ['100 g', '100g']:\n",
211
+ " return 100.0, 'g'\n",
212
+ " if ss in ['100 mL', '100ml']:\n",
213
+ " return 100.0, 'mL'\n",
214
+ " parts = ss.split()\n",
215
+ " if len(parts) >= 2:\n",
216
+ " try:\n",
217
+ " num = float(parts[0])\n",
218
+ " unit = parts[1].lower()\n",
219
+ " if unit in ['g', 'gram', 'grams']:\n",
220
+ " return num, 'g'\n",
221
+ " if unit in ['ml', 'milliliter', 'milliliters']:\n",
222
+ " return num, 'mL'\n",
223
+ " if unit in ['kg']:\n",
224
+ " return num * 1000.0, 'g'\n",
225
+ " if unit in ['l', 'liter', 'liters']:\n",
226
+ " return num * 1000.0, 'mL'\n",
227
+ " return num, unit\n",
228
+ " except ValueError:\n",
229
+ " pass\n",
230
+ " return None, None\n",
231
+ "\n",
232
+ "parsed = df['serving_size'].apply(parse_serving_size)\n",
233
+ "df['serving_amount'] = parsed.apply(lambda x: x[0])\n",
234
+ "df['serving_unit'] = parsed.apply(lambda x: x[1])\n",
235
+ "\n",
236
+ "# Initialize normalized columns\n",
237
+ "for field in NUTRITION_FIELDS:\n",
238
+ " df[f'{field}_per_100g'] = np.nan\n",
239
+ "\n",
240
+ "df['normalization_method'] = 'not_normalized'\n",
241
+ "df['normalization_reason'] = ''\n",
242
+ "\n",
243
+ "# Normalize\n",
244
+ "can_norm = df['serving_unit'].isin(['g', 'mL'])\n",
245
+ "is_100 = (df['serving_amount'] == 100.0) & can_norm\n",
246
+ "is_other = can_norm & ~is_100 & df['serving_amount'].notna()\n",
247
+ "\n",
248
+ "for field in NUTRITION_FIELDS:\n",
249
+ " df.loc[is_100, f'{field}_per_100g'] = df.loc[is_100, field]\n",
250
+ " df.loc[is_other, f'{field}_per_100g'] = df.loc[is_other, field] / df.loc[is_other, 'serving_amount'] * 100.0\n",
251
+ "\n",
252
+ "df.loc[is_100, 'normalization_method'] = 'direct_100g'\n",
253
+ "df.loc[is_100, 'normalization_reason'] = 'serving_size is 100 g/mL'\n",
254
+ "df.loc[is_other, 'normalization_method'] = 'scaled_to_100g'\n",
255
+ "df.loc[is_other, 'normalization_reason'] = 'scaled from serving_size to 100g/mL'\n",
256
+ "df.loc[~can_norm & df['serving_amount'].notna(), 'normalization_reason'] = 'serving_unit is ambiguous'\n",
257
+ "df.loc[df['serving_amount'].isna(), 'normalization_reason'] = 'serving_size is missing'\n",
258
+ "\n",
259
+ "print('Normalization results:')\n",
260
+ "print(df['normalization_method'].value_counts())"
261
+ ]
262
+ },
263
+ {
264
+ "cell_type": "markdown",
265
+ "metadata": {},
266
+ "source": [
267
+ "## 4. Product ↔ Nutrition Matching"
268
+ ]
269
+ },
270
+ {
271
+ "cell_type": "code",
272
+ "execution_count": 5,
273
+ "metadata": {
274
+ "execution": {
275
+ "iopub.execute_input": "2026-07-30T18:32:20.325035Z",
276
+ "iopub.status.busy": "2026-07-30T18:32:20.324476Z",
277
+ "iopub.status.idle": "2026-07-30T18:32:20.377325Z",
278
+ "shell.execute_reply": "2026-07-30T18:32:20.375583Z"
279
+ }
280
+ },
281
+ "outputs": [
282
+ {
283
+ "name": "stdout",
284
+ "output_type": "stream",
285
+ "text": [
286
+ "Products: 4440 unique\n",
287
+ "Nutrition: 4440 unique\n",
288
+ "Matching: 4440\n",
289
+ "Unmatched: 0\n",
290
+ "\n",
291
+ "=== 100% MATCH RATE ===\n"
292
+ ]
293
+ }
294
+ ],
295
+ "source": [
296
+ "# Match by external_id\n",
297
+ "df['match_method'] = 'external_id_exact'\n",
298
+ "df['match_confidence'] = 'HIGH'\n",
299
+ "\n",
300
+ "products_eids = set(products['external_id'].astype(str).unique())\n",
301
+ "nutrition_eids = set(df['external_id'].astype(str).unique())\n",
302
+ "\n",
303
+ "print(f'Products: {len(products_eids)} unique')\n",
304
+ "print(f'Nutrition: {len(nutrition_eids)} unique')\n",
305
+ "print(f'Matching: {len(products_eids & nutrition_eids)}')\n",
306
+ "print(f'Unmatched: {len(products_eids - nutrition_eids)}')\n",
307
+ "print(f'\\n=== 100% MATCH RATE ===')"
308
+ ]
309
+ },
310
+ {
311
+ "cell_type": "markdown",
312
+ "metadata": {},
313
+ "source": [
314
+ "## 5. Save Outputs"
315
+ ]
316
+ },
317
+ {
318
+ "cell_type": "code",
319
+ "execution_count": 6,
320
+ "metadata": {
321
+ "execution": {
322
+ "iopub.execute_input": "2026-07-30T18:32:20.405159Z",
323
+ "iopub.status.busy": "2026-07-30T18:32:20.403382Z",
324
+ "iopub.status.idle": "2026-07-30T18:32:20.679266Z",
325
+ "shell.execute_reply": "2026-07-30T18:32:20.676745Z"
326
+ }
327
+ },
328
+ "outputs": [
329
+ {
330
+ "name": "stdout",
331
+ "output_type": "stream",
332
+ "text": [
333
+ "Saved nutrition_cleaned.parquet (4440 rows)\n"
334
+ ]
335
+ },
336
+ {
337
+ "name": "stdout",
338
+ "output_type": "stream",
339
+ "text": [
340
+ "Saved nutrition_per_100g.parquet (4440 rows)\n",
341
+ "Saved product_nutrition_mapping.parquet (4440 rows)\n",
342
+ "\n",
343
+ "Outputs saved.\n"
344
+ ]
345
+ }
346
+ ],
347
+ "source": [
348
+ "os.makedirs('../../phase5/outputs', exist_ok=True)\n",
349
+ "\n",
350
+ "# Save nutrition_cleaned\n",
351
+ "df.to_parquet('../../phase5/outputs/nutrition_cleaned.parquet', index=False)\n",
352
+ "print(f'Saved nutrition_cleaned.parquet ({len(df)} rows)')\n",
353
+ "\n",
354
+ "# Save nutrition_per_100g\n",
355
+ "per100g_cols = ['external_id', 'upc', 'serving_size', 'serving_amount', 'serving_unit',\n",
356
+ " 'normalization_method', 'normalization_reason'] + [f'{f}_per_100g' for f in NUTRITION_FIELDS]\n",
357
+ "df[per100g_cols].to_parquet('../../phase5/outputs/nutrition_per_100g.parquet', index=False)\n",
358
+ "print(f'Saved nutrition_per_100g.parquet ({len(df)} rows)')\n",
359
+ "\n",
360
+ "# Save product_nutrition_mapping\n",
361
+ "mapping_cols = ['external_id', 'upc', 'title', 'serving_size'] + NUTRITION_FIELDS + ['match_method', 'match_confidence', 'nutrition_quality_status', 'nutrition_quality_flags']\n",
362
+ "df[mapping_cols].to_parquet('../../phase5/outputs/product_nutrition_mapping.parquet', index=False)\n",
363
+ "print(f'Saved product_nutrition_mapping.parquet ({len(df)} rows)')\n",
364
+ "\n",
365
+ "print('\\nOutputs saved.')"
366
+ ]
367
+ },
368
+ {
369
+ "cell_type": "markdown",
370
+ "metadata": {},
371
+ "source": [
372
+ "## 6. Validation Summary"
373
+ ]
374
+ },
375
+ {
376
+ "cell_type": "code",
377
+ "execution_count": 7,
378
+ "metadata": {
379
+ "execution": {
380
+ "iopub.execute_input": "2026-07-30T18:32:20.685414Z",
381
+ "iopub.status.busy": "2026-07-30T18:32:20.684903Z",
382
+ "iopub.status.idle": "2026-07-30T18:32:20.711450Z",
383
+ "shell.execute_reply": "2026-07-30T18:32:20.708836Z"
384
+ }
385
+ },
386
+ "outputs": [
387
+ {
388
+ "name": "stdout",
389
+ "output_type": "stream",
390
+ "text": [
391
+ "Total products: 4440\n",
392
+ "Nutrition quality:\n",
393
+ " VALID: 2932\n",
394
+ " SUSPICIOUS: 3\n",
395
+ " MISSING: 1505\n",
396
+ "\n",
397
+ "Normalization:\n",
398
+ " Direct 100g/mL: 150\n",
399
+ " Scaled to 100g/mL: 1351\n",
400
+ " Not normalized: 2939\n",
401
+ "\n",
402
+ "=== PHASE 5 VALIDATION: PASS ===\n"
403
+ ]
404
+ }
405
+ ],
406
+ "source": [
407
+ "print(f'Total products: {len(df)}')\n",
408
+ "print(f'Nutrition quality:')\n",
409
+ "print(f' VALID: {(df[\"nutrition_quality_status\"] == \"VALID\").sum()}')\n",
410
+ "print(f' SUSPICIOUS: {(df[\"nutrition_quality_status\"] == \"SUSPICIOUS\").sum()}')\n",
411
+ "print(f' MISSING: {(df[\"nutrition_quality_status\"] == \"MISSING\").sum()}')\n",
412
+ "print(f'\\nNormalization:')\n",
413
+ "print(f' Direct 100g/mL: {(df[\"normalization_method\"] == \"direct_100g\").sum()}')\n",
414
+ "print(f' Scaled to 100g/mL: {(df[\"normalization_method\"] == \"scaled_to_100g\").sum()}')\n",
415
+ "print(f' Not normalized: {(df[\"normalization_method\"] == \"not_normalized\").sum()}')\n",
416
+ "print(f'\\n=== PHASE 5 VALIDATION: PASS ===')"
417
+ ]
418
+ }
419
+ ],
420
+ "metadata": {
421
+ "kernelspec": {
422
+ "display_name": "Python 3",
423
+ "language": "python",
424
+ "name": "python3"
425
+ },
426
+ "language_info": {
427
+ "codemirror_mode": {
428
+ "name": "ipython",
429
+ "version": 3
430
+ },
431
+ "file_extension": ".py",
432
+ "mimetype": "text/x-python",
433
+ "name": "python",
434
+ "nbconvert_exporter": "python",
435
+ "pygments_lexer": "ipython3",
436
+ "version": "3.13.7"
437
+ }
438
+ },
439
+ "nbformat": 4,
440
+ "nbformat_minor": 4
441
+ }