saraNour commited on
Commit
89636ca
·
verified ·
1 Parent(s): bf2fcf4

Upload src/phase4.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. src/phase4.py +492 -0
src/phase4.py ADDED
@@ -0,0 +1,492 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """
3
+ Phase 4 — Variant Assignment / Variant Modeling
4
+ Compliments Reference DB Pipeline
5
+
6
+ Input: Phase 3 outputs + Phase 2 output
7
+ Output: Variant tables and mappings
8
+
9
+ This phase transforms the Phase 3 product-group representation into a
10
+ structured product variant model, where products belonging to the same
11
+ reference product group can have different purchasable variants such as
12
+ size, package quantity, formulation, flavour, fat level, etc.
13
+
14
+ CRITICAL: This phase does NOT use nutrition data.
15
+ CRITICAL: This phase does NOT use any external taxonomy.
16
+ CRITICAL: This phase is 100% deterministic (no LLM).
17
+ """
18
+
19
+ import json
20
+ import hashlib
21
+ import sys
22
+ from datetime import datetime, timezone
23
+ from pathlib import Path
24
+
25
+ import pandas as pd
26
+ import numpy as np
27
+
28
+ # ---------------------------------------------------------------------------
29
+ # Configuration
30
+ # ---------------------------------------------------------------------------
31
+ BASE_DIR = Path(__file__).resolve().parent.parent
32
+ PHASE3_DIR = BASE_DIR.parent / "phase3"
33
+ PHASE2_DIR = BASE_DIR.parent / "phase2"
34
+ OUTPUT_DIR = BASE_DIR / "outputs"
35
+ VALIDATION_DIR = BASE_DIR / "validation"
36
+ STATISTICS_DIR = BASE_DIR / "statistics"
37
+ AUDIT_DIR = BASE_DIR / "audit"
38
+
39
+ VERSION = "1.0.0"
40
+ TIMESTAMP = datetime.now(timezone.utc).isoformat()
41
+
42
+
43
+ def log(msg: str) -> None:
44
+ print(f"[Phase4] {msg}")
45
+
46
+
47
+ # ---------------------------------------------------------------------------
48
+ # 1. VARIANT DEFINITION
49
+ # ---------------------------------------------------------------------------
50
+ # A variant is a purchasable representation of the same reference product
51
+ # that differs in attributes such as:
52
+ # - Size (500 g, 1 kg, 2 kg)
53
+ # - Package quantity (1 × 500 g, 2 × 500 g)
54
+ # - Pack configuration (single, multipack, case)
55
+ #
56
+ # Variants within the same group share the same identity attributes:
57
+ # - is_organic, is_gluten_free, is_naturally_simple, etc.
58
+ # - product_line
59
+ # - flavour
60
+ # - formulation
61
+ # - fat_level
62
+ #
63
+ # Variants differ only in size/package attributes.
64
+
65
+
66
+ # ---------------------------------------------------------------------------
67
+ # 2. VARIANT KEY GENERATION
68
+ # ---------------------------------------------------------------------------
69
+
70
+ def build_variant_key(row: dict) -> str:
71
+ """
72
+ Build a deterministic variant key from normalized variant attributes.
73
+
74
+ The variant key captures size/package information that distinguishes
75
+ variants within the same reference product group.
76
+
77
+ Structure: amount|unit|qty|count
78
+ """
79
+ va_str = row.get("variant_attributes", "{}")
80
+ if isinstance(va_str, str):
81
+ try:
82
+ va = json.loads(va_str)
83
+ except (json.JSONDecodeError, TypeError):
84
+ va = {}
85
+ elif isinstance(va_str, dict):
86
+ va = va_str
87
+ else:
88
+ va = {}
89
+
90
+ # Extract variant-distinguishing attributes
91
+ amount = va.get("amount")
92
+ unit = va.get("unit", "")
93
+ qty = va.get("qty", 1)
94
+ count = va.get("count")
95
+ multiplier = va.get("multiplier", 1)
96
+ raw = va.get("raw", "")
97
+
98
+ # Build key parts
99
+ parts = []
100
+
101
+ if count is not None:
102
+ # Count-based variant (e.g., "20 per pack")
103
+ parts.append(f"count{int(count)}")
104
+ elif amount is not None:
105
+ # Size-based variant
106
+ parts.append(f"amt{amount}")
107
+ if unit:
108
+ parts.append(f"unit{unit}")
109
+ if qty and qty > 1:
110
+ parts.append(f"qty{int(qty)}")
111
+ if multiplier and multiplier > 1:
112
+ parts.append(f"mult{int(multiplier)}")
113
+ elif raw:
114
+ # Fallback to raw string
115
+ parts.append(f"raw{raw.strip().lower()}")
116
+ else:
117
+ # No size information
118
+ parts.append("nosize")
119
+
120
+ return "|".join(parts)
121
+
122
+
123
+ def generate_variant_id(group_id: str, variant_key: str) -> str:
124
+ """
125
+ Generate deterministic variant_id from group_id + variant_key.
126
+
127
+ Uses UUID5 (namespace-based) for reproducibility.
128
+ """
129
+ import uuid
130
+ namespace = uuid.NAMESPACE_DNS
131
+ name = f"{group_id}||{variant_key}"
132
+ return str(uuid.uuid5(namespace, name))
133
+
134
+
135
+ # ---------------------------------------------------------------------------
136
+ # 3. VARIANT ASSIGNMENT
137
+ # ---------------------------------------------------------------------------
138
+
139
+ def assign_variants(df: pd.DataFrame) -> pd.DataFrame:
140
+ """
141
+ Assign variant_ids to products within each group.
142
+
143
+ Products in the same group with the same variant_key get the same variant_id.
144
+ Products in the same group with different variant_keys get different variant_ids.
145
+ """
146
+ log("Building variant keys ...")
147
+ df["variant_key"] = df.apply(lambda row: build_variant_key(row.to_dict()), axis=1)
148
+
149
+ log("Generating variant IDs ...")
150
+ df["variant_id"] = df.apply(
151
+ lambda row: generate_variant_id(row["group_id"], row["variant_key"]),
152
+ axis=1
153
+ )
154
+
155
+ n_variants = df["variant_id"].nunique()
156
+ log(f"Total unique variants: {n_variants}")
157
+
158
+ return df
159
+
160
+
161
+ # ---------------------------------------------------------------------------
162
+ # 4. OUTPUT GENERATION
163
+ # ---------------------------------------------------------------------------
164
+
165
+ def build_variant_table(df: pd.DataFrame) -> pd.DataFrame:
166
+ """
167
+ Build the main variant table: one row per unique variant.
168
+ """
169
+ log("Building variant table ...")
170
+
171
+ # Group by variant_id and aggregate
172
+ variant_rows = []
173
+ for vid, grp in df.groupby("variant_id"):
174
+ row = grp.iloc[0]
175
+
176
+ # Get all sizes in this variant
177
+ sizes = grp["size"].dropna().unique().tolist()
178
+
179
+ variant_rows.append({
180
+ "variant_id": vid,
181
+ "group_id": row["group_id"],
182
+ "group_name": row["group_name"],
183
+ "brand": row["brand"],
184
+ "product_line": row["product_line"],
185
+ "product_domain": row["product_domain"],
186
+ "taxonomy": row["taxonomy"],
187
+ "core_title": row["core_title"],
188
+ "variant_key": row["variant_key"],
189
+ "size": sizes[0] if len(sizes) == 1 else "; ".join(str(s) for s in sizes),
190
+ "size_amount": row.get("size_amount"),
191
+ "size_unit": row.get("size_unit"),
192
+ "size_qty": row.get("size_qty"),
193
+ "flavour": row.get("flavour"),
194
+ "formulation": row.get("formulation"),
195
+ "fat_level": row.get("fat_level"),
196
+ "fat_percentage": row.get("fat_percentage"),
197
+ "product_count": len(grp),
198
+ "unique_upcs": grp["upc"].nunique(),
199
+ "source": row["source"],
200
+ "source_url": row["source_url"],
201
+ })
202
+
203
+ variant_df = pd.DataFrame(variant_rows)
204
+ log(f" Variant table: {len(variant_df)} variants")
205
+ return variant_df
206
+
207
+
208
+ def build_variant_mapping(df: pd.DataFrame) -> pd.DataFrame:
209
+ """
210
+ Build the product → variant mapping table.
211
+ """
212
+ log("Building variant mapping ...")
213
+
214
+ mapping = df[[
215
+ "external_id", "upc", "group_id", "variant_id",
216
+ "core_title", "original_title", "brand", "size",
217
+ "variant_key", "source", "source_url"
218
+ ]].copy()
219
+
220
+ log(f" Variant mapping: {len(mapping)} products")
221
+ return mapping
222
+
223
+
224
+ def build_variant_summary(df: pd.DataFrame) -> pd.DataFrame:
225
+ """
226
+ Build the group-level variant summary.
227
+ """
228
+ log("Building variant summary ...")
229
+
230
+ summary_rows = []
231
+ for gid, grp in df.groupby("group_id"):
232
+ variant_ids = grp["variant_id"].unique().tolist()
233
+ row = grp.iloc[0]
234
+ summary_rows.append({
235
+ "group_id": gid,
236
+ "group_name": row["group_name"],
237
+ "brand": row["brand"],
238
+ "product_domain": row["product_domain"],
239
+ "taxonomy": row["taxonomy"],
240
+ "variant_count": len(variant_ids),
241
+ "variant_ids": ", ".join(variant_ids),
242
+ "product_count": len(grp),
243
+ })
244
+
245
+ summary_df = pd.DataFrame(summary_rows)
246
+ log(f" Variant summary: {len(summary_df)} groups")
247
+ return summary_df
248
+
249
+
250
+ # ---------------------------------------------------------------------------
251
+ # 5. VALIDATION
252
+ # ---------------------------------------------------------------------------
253
+
254
+ def validate_phase4(df: pd.DataFrame, variant_table: pd.DataFrame,
255
+ variant_mapping: pd.DataFrame, variant_summary: pd.DataFrame) -> dict:
256
+ """Comprehensive Phase 4 validation."""
257
+ checks = {}
258
+
259
+ # Rule 1: Every product has exactly one variant
260
+ checks["rule_1_product_has_one_variant"] = {
261
+ "description": "Every product has exactly one variant_id",
262
+ "products": len(df),
263
+ "unique_variant_ids_per_product": int(df.groupby("external_id")["variant_id"].nunique().max()),
264
+ "pass": int(df.groupby("external_id")["variant_id"].nunique().max()) == 1,
265
+ }
266
+
267
+ # Rule 2: Every variant belongs to exactly one group
268
+ checks["rule_2_variant_belongs_to_one_group"] = {
269
+ "description": "Every variant belongs to exactly one group_id",
270
+ "variants": len(variant_table),
271
+ "unique_groups_per_variant": int(variant_table.groupby("variant_id")["group_id"].nunique().max()),
272
+ "pass": int(variant_table.groupby("variant_id")["group_id"].nunique().max()) == 1,
273
+ }
274
+
275
+ # Rule 3: No product maps to multiple variants
276
+ checks["rule_3_no_product_multiple_variants"] = {
277
+ "description": "No product maps to multiple variant_ids",
278
+ "products_with_multiple_variants": int((df.groupby("external_id")["variant_id"].nunique() > 1).sum()),
279
+ "pass": int((df.groupby("external_id")["variant_id"].nunique() > 1).sum()) == 0,
280
+ }
281
+
282
+ # Rule 4: Variant count matches
283
+ checks["rule_4_variant_count"] = {
284
+ "description": "Variant count in mapping equals variant table",
285
+ "mapping_variants": int(variant_mapping["variant_id"].nunique()),
286
+ "table_variants": len(variant_table),
287
+ "pass": int(variant_mapping["variant_id"].nunique()) == len(variant_table),
288
+ }
289
+
290
+ # Rule 5: All products have group_id
291
+ checks["rule_5_all_products_have_group"] = {
292
+ "description": "All products have group_id",
293
+ "null_group_ids": int(df["group_id"].isna().sum()),
294
+ "pass": int(df["group_id"].isna().sum()) == 0,
295
+ }
296
+
297
+ # Rule 6: All products have variant_id
298
+ checks["rule_6_all_products_have_variant"] = {
299
+ "description": "All products have variant_id",
300
+ "null_variant_ids": int(df["variant_id"].isna().sum()),
301
+ "pass": int(df["variant_id"].isna().sum()) == 0,
302
+ }
303
+
304
+ # Overall
305
+ all_pass = all(c.get("pass", True) for c in checks.values())
306
+ checks["overall"] = {"result": "PASS" if all_pass else "FAIL"}
307
+
308
+ return checks
309
+
310
+
311
+ # ---------------------------------------------------------------------------
312
+ # 6. STATISTICS
313
+ # ---------------------------------------------------------------------------
314
+
315
+ def build_statistics(df: pd.DataFrame, variant_table: pd.DataFrame,
316
+ variant_mapping: pd.DataFrame, variant_summary: pd.DataFrame) -> dict:
317
+ """Build Phase 4 statistics."""
318
+
319
+ # Variant count distribution
320
+ variant_counts = variant_summary["variant_count"].value_counts().sort_index()
321
+
322
+ return {
323
+ "version": VERSION,
324
+ "timestamp": TIMESTAMP,
325
+ "input": {
326
+ "source": "phase3_product_group_mapping.csv + phase2_output.parquet",
327
+ "products": len(df),
328
+ "groups": df["group_id"].nunique(),
329
+ },
330
+ "output": {
331
+ "variants": len(variant_table),
332
+ "products_mapped": len(variant_mapping),
333
+ "groups": len(variant_summary),
334
+ },
335
+ "variant_distribution": {
336
+ "groups_with_1_variant": int((variant_summary["variant_count"] == 1).sum()),
337
+ "groups_with_2_variants": int((variant_summary["variant_count"] == 2).sum()),
338
+ "groups_with_3_variants": int((variant_summary["variant_count"] == 3).sum()),
339
+ "groups_with_4plus_variants": int((variant_summary["variant_count"] >= 4).sum()),
340
+ "max_variants_per_group": int(variant_summary["variant_count"].max()),
341
+ "avg_variants_per_group": float(variant_summary["variant_count"].mean()),
342
+ },
343
+ "variant_count_distribution": {str(k): int(v) for k, v in variant_counts.items()},
344
+ "missing_size": {
345
+ "products_with_missing_size": int(df["size"].isna().sum()),
346
+ "products_with_no_size_variant": int((df["variant_key"] == "nosize").sum()),
347
+ },
348
+ }
349
+
350
+
351
+ # ---------------------------------------------------------------------------
352
+ # 7. PEANUT BUTTER EXAMPLE
353
+ # ---------------------------------------------------------------------------
354
+
355
+ def generate_peanut_butter_example(df: pd.DataFrame) -> pd.DataFrame:
356
+ """Generate Peanut Butter variant example for audit."""
357
+ log("Generating Peanut Butter example ...")
358
+
359
+ # Find peanut butter products
360
+ pb = df[df["original_title"].str.contains("peanut butter", case=False, na=False)].copy()
361
+
362
+ # Select relevant columns
363
+ example = pb[[
364
+ "external_id", "upc", "group_id", "variant_id",
365
+ "original_title", "core_title", "brand", "size",
366
+ "variant_key", "variant_attributes", "product_line",
367
+ "identity_hash", "fat_level", "product_domain", "taxonomy"
368
+ ]].copy()
369
+
370
+ # Sort by group_id and size
371
+ example = example.sort_values(["group_id", "size"]).reset_index(drop=True)
372
+
373
+ log(f" Peanut Butter example: {len(example)} products in {example['group_id'].nunique()} groups")
374
+ return example
375
+
376
+
377
+ # ---------------------------------------------------------------------------
378
+ # MAIN
379
+ # ---------------------------------------------------------------------------
380
+
381
+ def main():
382
+ log("Starting Phase 4 (v1.0.0 — Variant Assignment)")
383
+
384
+ # Ensure output dirs exist
385
+ OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
386
+ VALIDATION_DIR.mkdir(parents=True, exist_ok=True)
387
+ STATISTICS_DIR.mkdir(parents=True, exist_ok=True)
388
+ AUDIT_DIR.mkdir(parents=True, exist_ok=True)
389
+
390
+ # ------------------------------------------------------------------
391
+ # 1. LOAD PHASE 3 OUTPUTS
392
+ # ------------------------------------------------------------------
393
+ log("Loading Phase 3 outputs ...")
394
+ mapping_path = PHASE3_DIR / "outputs" / "product_group_mapping.csv"
395
+ mapping = pd.read_csv(mapping_path)
396
+ log(f" product_group_mapping: {len(mapping)} rows")
397
+
398
+ # ------------------------------------------------------------------
399
+ # 2. LOAD PHASE 2 OUTPUT (for variant-relevant columns)
400
+ # ------------------------------------------------------------------
401
+ log("Loading Phase 2 output ...")
402
+ p2_path = PHASE2_DIR / "outputs" / "phase2_output.parquet"
403
+ p2 = pd.read_parquet(p2_path)
404
+ log(f" phase2_output: {len(p2)} rows, {len(p2.columns)} columns")
405
+
406
+ # ------------------------------------------------------------------
407
+ # 3. MERGE PHASE 3 MAPPING WITH PHASE 2 VARIANT DATA
408
+ # ------------------------------------------------------------------
409
+ log("Merging Phase 3 mapping with Phase 2 variant data ...")
410
+ variant_cols = ["external_id", "variant_attributes", "flavour", "formulation",
411
+ "fat_level", "fat_percentage", "size_amount", "size_unit", "size_qty"]
412
+ df = mapping.merge(p2[variant_cols], on="external_id", how="left")
413
+ log(f" Merged: {len(df)} rows, {len(df.columns)} columns")
414
+
415
+ # ------------------------------------------------------------------
416
+ # 4. ASSIGN VARIANTS
417
+ # ------------------------------------------------------------------
418
+ df = assign_variants(df)
419
+
420
+ # ------------------------------------------------------------------
421
+ # 5. GENERATE OUTPUTS
422
+ # ------------------------------------------------------------------
423
+ variant_table = build_variant_table(df)
424
+ variant_mapping = build_variant_mapping(df)
425
+ variant_summary = build_variant_summary(df)
426
+
427
+ # ------------------------------------------------------------------
428
+ # 6. VALIDATION
429
+ # ------------------------------------------------------------------
430
+ log("Running validation ...")
431
+ validation = validate_phase4(df, variant_table, variant_mapping, variant_summary)
432
+
433
+ # ------------------------------------------------------------------
434
+ # 7. STATISTICS
435
+ # ------------------------------------------------------------------
436
+ log("Building statistics ...")
437
+ statistics = build_statistics(df, variant_table, variant_mapping, variant_summary)
438
+
439
+ # ------------------------------------------------------------------
440
+ # 8. PEANUT BUTTER EXAMPLE
441
+ # ------------------------------------------------------------------
442
+ pb_example = generate_peanut_butter_example(df)
443
+
444
+ # ------------------------------------------------------------------
445
+ # 9. SAVE OUTPUTS
446
+ # ------------------------------------------------------------------
447
+ log("Saving outputs ...")
448
+
449
+ # Main outputs (Parquet)
450
+ variant_table.to_parquet(OUTPUT_DIR / "reference_product_variants.parquet", index=False)
451
+ log(f" Saved reference_product_variants.parquet ({len(variant_table)} rows)")
452
+
453
+ variant_mapping.to_parquet(OUTPUT_DIR / "product_variant_mapping.parquet", index=False)
454
+ log(f" Saved product_variant_mapping.parquet ({len(variant_mapping)} rows)")
455
+
456
+ variant_summary.to_parquet(OUTPUT_DIR / "reference_product_variant_summary.parquet", index=False)
457
+ log(f" Saved reference_product_variant_summary.parquet ({len(variant_summary)} rows)")
458
+
459
+ # CSV versions for inspection
460
+ variant_table.to_csv(OUTPUT_DIR / "reference_product_variants.csv", index=False)
461
+ variant_mapping.to_csv(OUTPUT_DIR / "product_variant_mapping.csv", index=False)
462
+ variant_summary.to_csv(OUTPUT_DIR / "reference_product_variant_summary.csv", index=False)
463
+
464
+ # Validation
465
+ with open(VALIDATION_DIR / "phase4_validation.json", "w") as f:
466
+ json.dump(validation, f, indent=2, default=str)
467
+ log(" Saved phase4_validation.json")
468
+
469
+ # Statistics
470
+ with open(STATISTICS_DIR / "phase4_statistics.json", "w") as f:
471
+ json.dump(statistics, f, indent=2, default=str)
472
+ log(" Saved phase4_statistics.json")
473
+
474
+ # Peanut Butter example
475
+ pb_example.to_parquet(AUDIT_DIR / "phase4_peanut_butter_example.parquet", index=False)
476
+ log(f" Saved phase4_peanut_butter_example.parquet ({len(pb_example)} rows)")
477
+
478
+ # ------------------------------------------------------------------
479
+ # SUMMARY
480
+ # ------------------------------------------------------------------
481
+ log("")
482
+ log("=== PHASE 4 COMPLETE (v1.0.0) ===")
483
+ log(f"Input: {len(df)} products, {df['group_id'].nunique()} groups")
484
+ log(f"Output: {len(variant_table)} variants, {len(variant_mapping)} products mapped")
485
+ log(f"Validation: {validation['overall']['result']}")
486
+ log("========================")
487
+
488
+ return variant_table, variant_mapping, variant_summary, validation, statistics
489
+
490
+
491
+ if __name__ == "__main__":
492
+ main()