Upload outputs/phase1_provenance.json with huggingface_hub

outputs/phase1_provenance.json

@@ -4,48 +4,53 @@
   "source_type": "HuggingFace dataset (private)",
   "access_method": "huggingface_hub.hf_hub_download",
   "original_source": "Voila.ca (Loblaw) Compliments private-label products",
-  "columns": {
-    "upc": "Universal Product Code. 1 null. 3,271 unique. Some reused across variants.",
-    "external_id": "Voila retailer product ID. 4,440 unique. No nulls.",
-    "brand": "Product brand. 11 variants of Compliments/Sensations.",
-    "title": "Raw product title from Voila. 4,375 unique.",
-    "price": "Price in CAD. 247 unique values.",
-    "price_currency": "Always 'CAD'.",
-    "size": "Raw size string from Voila. 631 unique values.",
-    "size_amount": "Parsed numeric size amount. 307 nulls (6.9%).",
-    "size_unit": "Parsed size unit. 307 nulls (6.9%).",
-    "size_qty": "Size quantity multiplier. Usually 1.",
-    "size_per_unit": "100% null. All values are NaN. DROPPED.",
-    "size_unit_norm": "Normalized size unit. 307 nulls (6.9%).",
-    "size_total": "100% null. All values are NaN. DROPPED.",
-    "image_url": "Product image URL from Voila. 4,440 unique.",
-    "source": "Always 'voila'.",
-    "source_url": "Product page URL on Voila. 4,440 unique."
+  "pipeline_version": "2.0.0",
+  "cleaning_operations": [
+    "A. String normalization: trim whitespace, empty-string-to-null on all string columns",
+    "B. Brand cleaning: whitespace trim + case normalization, preserve raw in brand_raw",
+    "C. Title cleaning: whitespace trim + collapse repeated spaces, preserve raw in title_raw",
+    "D. UPC validation: format check, null audit, reused-UPC analysis (values NOT altered)",
+    "E. external_id validation: null + duplicate check (values NOT altered)",
+    "F. 100% null columns dropped: size_per_unit, size_total (documented, no information lost)"
+  ],
+  "raw_vs_cleaned_fields": {
+    "brand_raw": "Original brand value from authoritative source, preserved for traceability",
+    "brand_clean": "Cleaned brand: whitespace trimmed, case normalized (e.g., 'COMPLIMENTS' -> 'Compliments')",
+    "title_raw": "Original title value from authoritative source, preserved for traceability",
+    "title_clean": "Cleaned title: whitespace trimmed, repeated spaces collapsed",
+    "upc_raw": "Original UPC value, preserved for traceability (values NOT altered)",
+    "external_id_raw": "Original external_id, preserved for traceability (values NOT altered)"
   },
   "dropped_columns": [
     {
       "column": "size_per_unit",
-      "reason": "100% null (4,440/4,440 values are NaN). No usable data."
+      "reason": "100% null (4,440/4,440 values are NaN)"
     },
     {
       "column": "size_total",
-      "reason": "100% null (4,440/4,440 values are NaN). No usable data."
+      "reason": "100% null (4,440/4,440 values are NaN)"
     }
   ],
-  "preserved_columns": [
-    "upc",
-    "external_id",
-    "brand",
-    "title",
-    "price",
-    "price_currency",
-    "size",
-    "size_amount",
-    "size_unit",
-    "size_qty",
-    "size_unit_norm",
-    "image_url",
-    "source",
-    "source_url"
-  ]
+  "columns": {
+    "upc": "Universal Product Code. 1 null. Some reused across variants.",
+    "upc_raw": "Original UPC from source (preserved for traceability)",
+    "external_id": "Voila retailer product ID. Unique. No nulls.",
+    "external_id_raw": "Original external_id from source (preserved for traceability)",
+    "brand": "Original brand string from source.",
+    "brand_clean": "Cleaned brand: whitespace/case normalized.",
+    "brand_raw": "Original brand from source (preserved for traceability).",
+    "title": "Original product title from source.",
+    "title_clean": "Cleaned title: whitespace normalized.",
+    "title_raw": "Original title from source (preserved for traceability).",
+    "price": "Price in CAD.",
+    "price_currency": "Always 'CAD'.",
+    "size": "Raw size string from source.",
+    "size_amount": "Parsed numeric size amount.",
+    "size_unit": "Parsed size unit.",
+    "size_qty": "Size quantity multiplier.",
+    "size_unit_norm": "Normalized size unit.",
+    "image_url": "Product image URL from source.",
+    "source": "Always 'voila'.",
+    "source_url": "Product page URL on source."
+  }
 }