Upload outputs/phase1_executed.ipynb with huggingface_hub

outputs/phase1_executed.ipynb

@@ -0,0 +1,1400 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Phase 1 — Load / Validate / Provenance\n",
+    "## Compliments Reference DB Pipeline\n",
+    "\n",
+    "**Authoritative Input:**\n",
+    "`saraNour/compliments-brand/source_of_truth/products.parquet`\n",
+    "\n",
+    "**This phase does:**\n",
+    "1. Downloads the authoritative products.parquet from HuggingFace\n",
+    "2. Validates schema, row count, nulls, duplicates\n",
+    "3. Analyzes UPC patterns, brand values, size fields\n",
+    "4. Documents provenance of every column\n",
+    "5. Drops 100% null columns with explicit documentation\n",
+    "6. Produces clean Phase 1 output + validation + statistics"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2026-07-30T14:10:05.327344Z",
+     "iopub.status.busy": "2026-07-30T14:10:05.326892Z",
+     "iopub.status.idle": "2026-07-30T14:10:06.411761Z",
+     "shell.execute_reply": "2026-07-30T14:10:06.407953Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Python: 3.13.7 (main, Mar  3 2026, 12:19:54) [GCC 15.2.0]\n",
+      "pandas: 3.0.5\n",
+      "numpy: 2.5.1\n",
+      "Timestamp: 2026-07-30T14:10:06.402163+00:00\n"
+     ]
+    }
+   ],
+   "source": [
+    "import json, os, sys\n",
+    "from datetime import datetime, timezone\n",
+    "from pathlib import Path\n",
+    "\n",
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "from huggingface_hub import hf_hub_download\n",
+    "\n",
+    "print(f\"Python: {sys.version}\")\n",
+    "print(f\"pandas: {pd.__version__}\")\n",
+    "print(f\"numpy: {np.__version__}\")\n",
+    "print(f\"Timestamp: {datetime.now(timezone.utc).isoformat()}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 1. Configuration"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2026-07-30T14:10:06.511086Z",
+     "iopub.status.busy": "2026-07-30T14:10:06.510327Z",
+     "iopub.status.idle": "2026-07-30T14:10:06.525994Z",
+     "shell.execute_reply": "2026-07-30T14:10:06.522561Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Source: saraNour/compliments-brand/source_of_truth/products.parquet\n",
+      "Version: 1.0.0\n"
+     ]
+    }
+   ],
+   "source": [
+    "HF_REPO = \"saraNour/compliments-brand\"\n",
+    "HF_FILE = \"source_of_truth/products.parquet\"\n",
+    "HF_REPO_TYPE = \"dataset\"\n",
+    "\n",
+    "VERSION = \"1.0.0\"\n",
+    "TIMESTAMP = datetime.now(timezone.utc).isoformat()\n",
+    "\n",
+    "print(f\"Source: {HF_REPO}/{HF_FILE}\")\n",
+    "print(f\"Version: {VERSION}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 2. Load Authoritative Dataset"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2026-07-30T14:10:06.532965Z",
+     "iopub.status.busy": "2026-07-30T14:10:06.530781Z",
+     "iopub.status.idle": "2026-07-30T14:10:07.287661Z",
+     "shell.execute_reply": "2026-07-30T14:10:07.285564Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Downloaded to: /home/sara/.cache/huggingface/hub/datasets--saraNour--compliments-brand/snapshots/3d9e0ded3dba963c24c623d623b07dc94592c198/source_of_truth/products.parquet\n",
+      "Shape: 4440 rows x 16 columns\n"
+     ]
+    }
+   ],
+   "source": [
+    "path = hf_hub_download(HF_REPO, HF_FILE, repo_type=HF_REPO_TYPE)\n",
+    "print(f\"Downloaded to: {path}\")\n",
+    "\n",
+    "df = pd.read_parquet(path)\n",
+    "print(f\"Shape: {df.shape[0]} rows x {df.shape[1]} columns\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 3. Schema Inspection"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2026-07-30T14:10:07.294052Z",
+     "iopub.status.busy": "2026-07-30T14:10:07.292241Z",
+     "iopub.status.idle": "2026-07-30T14:10:07.351713Z",
+     "shell.execute_reply": "2026-07-30T14:10:07.347077Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "=== Column Names ===\n",
+      "   1. upc                  str         3271 unique,     1 null\n",
+      "   2. external_id          str         4440 unique,     0 null\n",
+      "   3. brand                str           11 unique,     0 null\n",
+      "   4. title                str         4375 unique,     0 null\n",
+      "   5. price                float64      247 unique,     0 null\n",
+      "   6. price_currency       str            1 unique,     0 null\n",
+      "   7. size                 str          631 unique,     0 null\n",
+      "   8. size_amount          float64      387 unique,   307 null\n",
+      "   9. size_unit            str            5 unique,   307 null\n",
+      "  10. size_qty             int64         17 unique,     0 null\n",
+      "  11. size_per_unit        object         0 unique,  4440 null\n",
+      "  12. size_unit_norm       str            5 unique,   307 null\n",
+      "  13. size_total           object         0 unique,  4440 null\n",
+      "  14. image_url            str         4440 unique,     0 null\n",
+      "  15. source               str            1 unique,     0 null\n",
+      "  16. source_url           str         4440 unique,     0 null\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(\"=== Column Names ===\")\n",
+    "for i, col in enumerate(df.columns):\n",
+    "    print(f\"  {i+1:2d}. {col:20s} {str(df[col].dtype):10s} {df[col].nunique():5d} unique, {df[col].isna().sum():5d} null\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2026-07-30T14:10:07.366335Z",
+     "iopub.status.busy": "2026-07-30T14:10:07.365726Z",
+     "iopub.status.idle": "2026-07-30T14:10:07.420853Z",
+     "shell.execute_reply": "2026-07-30T14:10:07.418826Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "=== First 3 Rows ===\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>upc</th>\n",
+       "      <th>external_id</th>\n",
+       "      <th>brand</th>\n",
+       "      <th>title</th>\n",
+       "      <th>price</th>\n",
+       "      <th>price_currency</th>\n",
+       "      <th>size</th>\n",
+       "      <th>size_amount</th>\n",
+       "      <th>size_unit</th>\n",
+       "      <th>size_qty</th>\n",
+       "      <th>size_per_unit</th>\n",
+       "      <th>size_unit_norm</th>\n",
+       "      <th>size_total</th>\n",
+       "      <th>image_url</th>\n",
+       "      <th>source</th>\n",
+       "      <th>source_url</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>055742522167</td>\n",
+       "      <td>297608EA</td>\n",
+       "      <td>Compliments</td>\n",
+       "      <td>Compliments Cold And Sinus Plus 20 Count</td>\n",
+       "      <td>9.99</td>\n",
+       "      <td>CAD</td>\n",
+       "      <td>20 per pack</td>\n",
+       "      <td>20.000</td>\n",
+       "      <td>count</td>\n",
+       "      <td>1</td>\n",
+       "      <td>None</td>\n",
+       "      <td>count</td>\n",
+       "      <td>None</td>\n",
+       "      <td>https://voila.ca/images-v3/2d92d19c-0354-49c0-...</td>\n",
+       "      <td>voila</td>\n",
+       "      <td>https://voila.ca/products/compliments-cold-and...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>055742335200</td>\n",
+       "      <td>464324EA</td>\n",
+       "      <td>Compliments</td>\n",
+       "      <td>Compliments Twister T2 Light Bulbs 23W Soft Wh...</td>\n",
+       "      <td>11.99</td>\n",
+       "      <td>CAD</td>\n",
+       "      <td>2 per pack</td>\n",
+       "      <td>2.000</td>\n",
+       "      <td>count</td>\n",
+       "      <td>1</td>\n",
+       "      <td>None</td>\n",
+       "      <td>count</td>\n",
+       "      <td>None</td>\n",
+       "      <td>https://voila.ca/images-v3/2d92d19c-0354-49c0-...</td>\n",
+       "      <td>voila</td>\n",
+       "      <td>https://voila.ca/products/compliments-twister-...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>990002600400</td>\n",
+       "      <td>144288CS</td>\n",
+       "      <td>Compliments</td>\n",
+       "      <td>Compliments Zero Calorie Soft Drink 12 x 355ml</td>\n",
+       "      <td>5.49</td>\n",
+       "      <td>CAD</td>\n",
+       "      <td>12 x 29.583ml</td>\n",
+       "      <td>29.583</td>\n",
+       "      <td>ml</td>\n",
+       "      <td>12</td>\n",
+       "      <td>None</td>\n",
+       "      <td>ml</td>\n",
+       "      <td>None</td>\n",
+       "      <td>https://voila.ca/images-v3/2d92d19c-0354-49c0-...</td>\n",
+       "      <td>voila</td>\n",
+       "      <td>https://voila.ca/products/compliments-zero-cal...</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "            upc external_id        brand  \\\n",
+       "0  055742522167    297608EA  Compliments   \n",
+       "1  055742335200    464324EA  Compliments   \n",
+       "2  990002600400    144288CS  Compliments   \n",
+       "\n",
+       "                                               title  price price_currency  \\\n",
+       "0           Compliments Cold And Sinus Plus 20 Count   9.99            CAD   \n",
+       "1  Compliments Twister T2 Light Bulbs 23W Soft Wh...  11.99            CAD   \n",
+       "2     Compliments Zero Calorie Soft Drink 12 x 355ml   5.49            CAD   \n",
+       "\n",
+       "            size  size_amount size_unit  size_qty size_per_unit  \\\n",
+       "0    20 per pack       20.000     count         1          None   \n",
+       "1     2 per pack        2.000     count         1          None   \n",
+       "2  12 x 29.583ml       29.583        ml        12          None   \n",
+       "\n",
+       "  size_unit_norm size_total  \\\n",
+       "0          count       None   \n",
+       "1          count       None   \n",
+       "2             ml       None   \n",
+       "\n",
+       "                                           image_url source  \\\n",
+       "0  https://voila.ca/images-v3/2d92d19c-0354-49c0-...  voila   \n",
+       "1  https://voila.ca/images-v3/2d92d19c-0354-49c0-...  voila   \n",
+       "2  https://voila.ca/images-v3/2d92d19c-0354-49c0-...  voila   \n",
+       "\n",
+       "                                          source_url  \n",
+       "0  https://voila.ca/products/compliments-cold-and...  \n",
+       "1  https://voila.ca/products/compliments-twister-...  \n",
+       "2  https://voila.ca/products/compliments-zero-cal...  "
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "print(\"=== First 3 Rows ===\")\n",
+    "df.head(3)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 4. Schema Validation"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2026-07-30T14:10:07.429459Z",
+     "iopub.status.busy": "2026-07-30T14:10:07.427323Z",
+     "iopub.status.idle": "2026-07-30T14:10:07.447665Z",
+     "shell.execute_reply": "2026-07-30T14:10:07.443941Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Row count: 4440 (expected 4440) -> PASS\n",
+      "Columns: 16 (expected 16) -> PASS\n",
+      "Column order matches -> PASS\n"
+     ]
+    }
+   ],
+   "source": [
+    "EXPECTED_COLUMNS = [\n",
+    "    \"upc\", \"external_id\", \"brand\", \"title\", \"price\", \"price_currency\",\n",
+    "    \"size\", \"size_amount\", \"size_unit\", \"size_qty\", \"size_per_unit\",\n",
+    "    \"size_unit_norm\", \"size_total\", \"image_url\", \"source\", \"source_url\",\n",
+    "]\n",
+    "\n",
+    "EXPECTED_ROW_COUNT = 4440\n",
+    "\n",
+    "# Check row count\n",
+    "row_check = len(df) == EXPECTED_ROW_COUNT\n",
+    "print(f\"Row count: {len(df)} (expected {EXPECTED_ROW_COUNT}) -> {'PASS' if row_check else 'FAIL'}\")\n",
+    "\n",
+    "# Check columns\n",
+    "missing = [c for c in EXPECTED_COLUMNS if c not in df.columns]\n",
+    "extra = [c for c in df.columns if c not in EXPECTED_COLUMNS]\n",
+    "col_check = len(missing) == 0\n",
+    "print(f\"Columns: {len(df.columns)} (expected {len(EXPECTED_COLUMNS)}) -> {'PASS' if col_check else 'FAIL'}\")\n",
+    "if missing:\n",
+    "    print(f\"  MISSING: {missing}\")\n",
+    "if extra:\n",
+    "    print(f\"  EXTRA: {extra}\")\n",
+    "\n",
+    "# Check column order\n",
+    "order_check = list(df.columns) == EXPECTED_COLUMNS\n",
+    "print(f\"Column order matches -> {'PASS' if order_check else 'FAIL'}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 5. Null Analysis"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2026-07-30T14:10:07.455038Z",
+     "iopub.status.busy": "2026-07-30T14:10:07.452399Z",
+     "iopub.status.idle": "2026-07-30T14:10:07.499238Z",
+     "shell.execute_reply": "2026-07-30T14:10:07.496950Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "=== Null Analysis ===\n",
+      "  upc                 :     1 nulls (  0.0%)\n",
+      "  size_amount         :   307 nulls (  6.9%)\n",
+      "  size_unit           :   307 nulls (  6.9%)\n",
+      "  size_per_unit       :  4440 nulls (100.0%) *** 100% NULL ***\n",
+      "  size_unit_norm      :   307 nulls (  6.9%)\n",
+      "  size_total          :  4440 nulls (100.0%) *** 100% NULL ***\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>column</th>\n",
+       "      <th>null_count</th>\n",
+       "      <th>null_pct</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>upc</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0.02</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>7</th>\n",
+       "      <td>size_amount</td>\n",
+       "      <td>307</td>\n",
+       "      <td>6.91</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8</th>\n",
+       "      <td>size_unit</td>\n",
+       "      <td>307</td>\n",
+       "      <td>6.91</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>10</th>\n",
+       "      <td>size_per_unit</td>\n",
+       "      <td>4440</td>\n",
+       "      <td>100.00</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>11</th>\n",
+       "      <td>size_unit_norm</td>\n",
+       "      <td>307</td>\n",
+       "      <td>6.91</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>12</th>\n",
+       "      <td>size_total</td>\n",
+       "      <td>4440</td>\n",
+       "      <td>100.00</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "            column  null_count  null_pct\n",
+       "0              upc           1      0.02\n",
+       "7      size_amount         307      6.91\n",
+       "8        size_unit         307      6.91\n",
+       "10   size_per_unit        4440    100.00\n",
+       "11  size_unit_norm         307      6.91\n",
+       "12      size_total        4440    100.00"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "print(\"=== Null Analysis ===\")\n",
+    "null_data = []\n",
+    "for col in df.columns:\n",
+    "    n = df[col].isna().sum()\n",
+    "    pct = round(n / len(df) * 100, 2)\n",
+    "    flag = \" *** 100% NULL ***\" if n == len(df) else \"\"\n",
+    "    null_data.append({\"column\": col, \"null_count\": n, \"null_pct\": pct})\n",
+    "    if n > 0:\n",
+    "        print(f\"  {col:20s}: {n:5d} nulls ({pct:5.1f}%){flag}\")\n",
+    "\n",
+    "null_df = pd.DataFrame(null_data)\n",
+    "null_df[null_df[\"null_count\"] > 0]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2026-07-30T14:10:07.502778Z",
+     "iopub.status.busy": "2026-07-30T14:10:07.502373Z",
+     "iopub.status.idle": "2026-07-30T14:10:07.509460Z",
+     "shell.execute_reply": "2026-07-30T14:10:07.507432Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Columns that are 100% null: ['size_per_unit', 'size_total']\n",
+      "These will be DROPPED in Phase 1 output.\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Identify 100% null columns\n",
+    "cols_100pct_null = [row[\"column\"] for row in null_data if row[\"null_count\"] == len(df)]\n",
+    "print(f\"Columns that are 100% null: {cols_100pct_null}\")\n",
+    "print(f\"These will be DROPPED in Phase 1 output.\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 6. Duplicate Analysis"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2026-07-30T14:10:07.514613Z",
+     "iopub.status.busy": "2026-07-30T14:10:07.514058Z",
+     "iopub.status.idle": "2026-07-30T14:10:07.555136Z",
+     "shell.execute_reply": "2026-07-30T14:10:07.553337Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "=== Duplicate Analysis ===\n",
+      "Full row duplicates: 0 -> PASS\n",
+      "UPC duplicates:      1168 (expected: some UPCs reused across variants)\n",
+      "external_id dupes:   0 -> PASS\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(\"=== Duplicate Analysis ===\")\n",
+    "full_dupes = df.duplicated().sum()\n",
+    "upc_dupes = df[\"upc\"].duplicated().sum()\n",
+    "ext_dupes = df[\"external_id\"].duplicated().sum()\n",
+    "\n",
+    "print(f\"Full row duplicates: {full_dupes} -> {'PASS' if full_dupes == 0 else 'FAIL'}\")\n",
+    "print(f\"UPC duplicates:      {upc_dupes} (expected: some UPCs reused across variants)\")\n",
+    "print(f\"external_id dupes:   {ext_dupes} -> {'PASS' if ext_dupes == 0 else 'FAIL'}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2026-07-30T14:10:07.568353Z",
+     "iopub.status.busy": "2026-07-30T14:10:07.567601Z",
+     "iopub.status.idle": "2026-07-30T14:10:07.706138Z",
+     "shell.execute_reply": "2026-07-30T14:10:07.704546Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "=== Reused UPCs (top 10) ==="
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "Total reused UPCs: 774\n",
+      "\n",
+      "UPC 055742560770 (10 rows):\n",
+      "    403517EA     | Compliments Naturally Simple Turkey Jerky 80 g               | Compliments Naturally Simple\n",
+      "    416081EA     | Compliments Naturally Simple Crisp Crackers Multigrain 150 g | Compliments\n",
+      "    591909EA     | Compliments Naturally Simple Yogurt Tzatziki Cucumber Dip 22 | Compliments Naturally Simple\n",
+      "    498164EA     | Compliments Naturally Simple Oven Roasted Chicken Breast 175 | Compliments Naturally Simple\n",
+      "    645109EA     | Compliments Naturally Simple Rice Pilaf Wild Mushroom & Herb | Compliments\n",
+      "    403516EA     | Compliments Naturally Simple Beef Jerky Teriyaki 80 g        | Compliments Naturally Simple\n",
+      "    417169EA     | Compliments Naturally Simple Tortilla Chips Beet And Corn 19 | Compliments\n",
+      "    680994EA     | Compliments Naturally Simple Baking Mix Scone 500 g          | Compliments Naturally Simple\n",
+      "    483867EA     | Compliments Naturally Simple Crystal Mountain Trail Mix 400  | Compliments Naturally Simple\n",
+      "    403515EA     | Compliments Naturally Simple Beef Jerky Original 80 g        | Compliments Naturally Simple\n",
+      "\n",
+      "UPC 055742501926 (9 rows):\n",
+      "    316959EA     | Compliments Gift Wrap Roll 40 Inch 90 Feet                   | Compliments\n",
+      "    316979EA     | Compliments Foil Gift Wrap 30 Inch 12 Feet                   | Compliments\n",
+      "    657871EA     | Compliments 12 Inch Pizza Pan                                | Compliments\n",
+      "    24142EA      | Compliments Foil Containers With Lids 4 lb 3 Pack            | Compliments\n",
+      "    229086EA     | Compliments Aluminum Foil 12 Inch x 50 Feet                  | Compliments\n",
+      "    20712EA      | Compliments Trendy Gift Wrap 30-Inch x 57.5-Inch 1 Count     | Compliments\n",
+      "    24107EA      | Compliments Foil Containers with Lids 3-lb 5 Pack            | Compliments\n",
+      "    24066EA      | Compliments Foil Containers with Lids 2-lb 7 Pack            | Compliments\n",
+      "    316963EA     | Compliments Paper Gift Wrap 30 Inch 45 Feet                  | Compliments\n",
+      "\n",
+      "UPC 055742562125 (8 rows):\n",
+      "    539754EA     | Compliments JuJubes 350 g                                    | Compliments\n",
+      "    957046EA     | Compliments Jujubes 700 g                                    | Compliments\n",
+      "    590556EA     | Compliments Candy Jujubes 750 g                              | Compliments\n",
+      "    990350EA     | Compliments Candy Jujubes Bunnies 325 g                      | Compliments\n",
+      "    647037EA     | Compliments Candy Jujubes 800 g                              | Compliments\n",
+      "    590552EA     | Compliments Candy Jujubes 175 g                              | Compliments\n",
+      "    647035EA     | Compliments Candy Jujubes 200 g                              | Compliments\n",
+      "    587575EA     | Compliments Candy Jujubes 350 g                              | Compliments\n",
+      "\n",
+      "UPC 055742567861 (8 rows):\n",
+      "    528968EA     | Compliments Terry Kitchen Towels Yellow 2 Count              | Compliments\n",
+      "    528414EA     | Compliments Charcoal Kitchen Towels 2 EA                     | Compliments\n",
+      "    528962EA     | Compliments Terry Kitchen Towels Mint 2 EA                   | Compliments\n",
+      "    528901EA     | Compliments Terry Kitchen Towels Charcoal 2 Count            | Compliments\n",
+      "    851246EA     | Compliments Paper Towels Greencare Recycled 6 x 105 Sheets R | Compliments\n",
+      "    528448EA     | Compliments Yellow Kitchen Towels 2 EA                       | Compliments\n",
+      "    528372EA     | Compliments Beige Kitchen Towels 2 EA                        | Compliments\n",
+      "    322530EA     | Compliments Paper Towels Green Care Half Size Sheets 90 x 6  | Compliments\n",
+      "\n",
+      "UPC 055742375534 (7 rows):\n",
+      "    158294EA     | Compliments Juice Strawberry 2 L (bottle)                    | Compliments\n",
+      "    545616EA     | Compliments Juice Peach & Passion Fruit With Coconut Water 1 | Compliments\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "    521832EA     | Compliments Juice Apples & Greens 1.65 L                     | Compliments\n",
+      "    158317EA     | Compliments Juice Peach Drink  2 L (bottle)                  | Compliments\n",
+      "    656001EA     | Compliments Juice Pineapple With Coconut Water 1.54 L (Bottl | Compliments\n",
+      "    656003EA     | Compliments Juice Peach Passion Fruit Coconut Blend 1.54 L ( | Compliments\n",
+      "    521839EA     | Compliments Juice Beet & Cherry 1.65 L                       | Compliments\n",
+      "\n",
+      "UPC 055742516111 (7 rows):\n",
+      "    351251EA     | Compliments Plastic Bandages 100 EA                          | Compliments\n",
+      "    298558EA     | Compliments Plastic Beer Cup Red 16-Ounce 50 Pack            | Compliments\n",
+      "    485028EA     | Compliments Plastic Forks White Full Size 24 Pack            | Compliments\n",
+      "    265142EA     | Compliments Plastic Wrap 90 m                                | Compliments\n",
+      "    485150EA     | Compliments Plastic Wine Glasses 5 Ounce 10 Pack             | Compliments\n",
+      "    486882EA     | Compliments Plastic Beverage Glasses 7 Ounce Small 50 Pack   | Compliments\n",
+      "    280774EA     | Compliments Plastic Lice Combs 3 EA                          | Compliments\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "UPC 055742335729 (7 rows):\n",
+      "    613615EA     | Compliments Soft Drink Grape 355 ml                          | Compliments\n",
+      "    281039EA     | Compliments Soft Drink Cola 355 ml                           | Compliments\n",
+      "    613604CS     | Compliments Soft Drink Cola Blue 12 x 355 ml                 | Compliments\n",
+      "    613615CS     | Compliments Soft Drink Grape 12 x 355 ml (cans)              | Compliments\n",
+      "    503191EA     | Compliments Soft Drink Cola 2 L (bottle)                     | Compliments\n",
+      "    613604EA     | Compliments Soft Drink Blue Cola 355 ml (can)                | Compliments\n",
+      "    281039CS     | Compliments Soft Drink Cola 12 x 355 ml (cans)               | Compliments\n",
+      "\n",
+      "UPC 055742502756 (6 rows):\n",
+      "    587512EA     | Compliments Party Bites Snack Mix 150 g                      | Compliments\n",
+      "    259864EA     | Compliments Party Bites Snack Mix 225 g                      | Compliments\n",
+      "    231864EA     | Compliments Snack Party Mix 300 g                            | Compliments\n",
+      "    587555EA     | Compliments Manhattan Snack Mix 375 g                        | Compliments\n",
+      "    259901EA     | Compliments Manhattan Snack Mix 500 g                        | Compliments\n",
+      "    958118EA     | Compliments Snack Cheddar Cheese Corn 482 g                  | Compliments\n",
+      "\n",
+      "UPC 055742581607 (6 rows):\n",
+      "    259875EA     | Compliments Walnut Pieces 250 g                              | Compliments\n",
+      "    584540EA     | Compliments Walnut Pieces 150 g                              | Compliments\n",
+      "    421729EA     | Compliments Walnut Pieces 400 g                              | Compliments\n",
+      "    259934EA     | Compliments Dried Mango Slices 325 g                         | Compliments\n",
+      "    480211EA     | Compliments Walnut Pieces 750 g                              | Compliments\n",
+      "    1341149EA    | Compliments Walnut Pieces 350 g                              | Compliments\n",
+      "\n",
+      "UPC 055742575361 (6 rows):\n",
+      "    287328EA     | Compliments Apple Caramel Coffee Cake 850 g (frozen)         | Compliments\n",
+      "    636356EA     | Compliments Cinnamon Coffee Cake 500 g                       | Compliments\n",
+      "    27846EA      | Compliments Muffin Cinnamon Coffee Cake 400 g                | Compliments\n",
+      "    408285EA     | Compliments Muffins Cinnamon Swirl Coffee Cake 440 g         | Compliments\n",
+      "    657878EA     | Compliments Muffin Pan 12 Cup 1 Pack                         | Compliments\n",
+      "    636353EA     | Compliments Coffee Cake Cinnamon 500 g                       | Compliments\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Show reused UPCs (products sharing the same barcode)\n",
+    "print(\"=== Reused UPCs (top 10) ===\")\n",
+    "upc_counts = df[\"upc\"].value_counts()\n",
+    "reused = upc_counts[upc_counts > 1]\n",
+    "print(f\"Total reused UPCs: {len(reused)}\")\n",
+    "print()\n",
+    "for upc, count in reused.head(10).items():\n",
+    "    subset = df[df[\"upc\"] == upc][[\"external_id\", \"title\", \"brand\"]]\n",
+    "    print(f\"UPC {upc} ({count} rows):\")\n",
+    "    for _, row in subset.iterrows():\n",
+    "        print(f\"    {row['external_id']:12s} | {row['title'][:60]:60s} | {row['brand']}\")\n",
+    "    print()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 7. UPC Analysis"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2026-07-30T14:10:07.715374Z",
+     "iopub.status.busy": "2026-07-30T14:10:07.714883Z",
+     "iopub.status.idle": "2026-07-30T14:10:07.739920Z",
+     "shell.execute_reply": "2026-07-30T14:10:07.736865Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "=== UPC Analysis ==="
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "Total rows:          4440\n",
+      "Null UPCs:           1\n",
+      "Unique UPCs:         3271\n",
+      "Reused UPCs:         774\n",
+      "\n",
+      "Rows with null UPC:\n",
+      "    external_id                                              title        brand\n",
+      "406     24061EA  Compliments Foil Containers with Lids 5-lb 2 Pack  Compliments\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(\"=== UPC Analysis ===\")\n",
+    "print(f\"Total rows:          {len(df)}\")\n",
+    "print(f\"Null UPCs:           {df['upc'].isna().sum()}\")\n",
+    "print(f\"Unique UPCs:         {df['upc'].nunique()}\")\n",
+    "print(f\"Reused UPCs:         {len(reused)}\")\n",
+    "print()\n",
+    "\n",
+    "# Show rows with null UPC\n",
+    "null_upc = df[df[\"upc\"].isna()]\n",
+    "if len(null_upc) > 0:\n",
+    "    print(\"Rows with null UPC:\")\n",
+    "    print(null_upc[[\"external_id\", \"title\", \"brand\"]].to_string())"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 8. Brand Analysis"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2026-07-30T14:10:07.746876Z",
+     "iopub.status.busy": "2026-07-30T14:10:07.744331Z",
+     "iopub.status.idle": "2026-07-30T14:10:07.762909Z",
+     "shell.execute_reply": "2026-07-30T14:10:07.759922Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "=== Brand Distribution ===\n",
+      "Unique brands: 11\n",
+      "\n",
+      "  Compliments                        :  4170 ( 93.9%)\n",
+      "  Compliments Organic                :    90 (  2.0%)\n",
+      "  Compliments                        :    87 (  2.0%)\n",
+      "  Compliments Balance                :    42 (  0.9%)\n",
+      "  Compliments Naturally Simple       :    25 (  0.6%)\n",
+      "  Sensations                         :     8 (  0.2%)\n",
+      "  Compliments Little Ones            :     7 (  0.2%)\n",
+      "  Compliments Green Care             :     5 (  0.1%)\n",
+      "  Compliments Green                  :     4 (  0.1%)\n",
+      "  COMPLIMENTS                        :     1 (  0.0%)\n",
+      "   Compliments                       :     1 (  0.0%)\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(\"=== Brand Distribution ===\")\n",
+    "brand_counts = df[\"brand\"].value_counts()\n",
+    "print(f\"Unique brands: {len(brand_counts)}\")\n",
+    "print()\n",
+    "for brand, count in brand_counts.items():\n",
+    "    print(f\"  {brand:35s}: {count:5d} ({count/len(df)*100:5.1f}%)\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 9. Size Analysis"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2026-07-30T14:10:07.771584Z",
+     "iopub.status.busy": "2026-07-30T14:10:07.769398Z",
+     "iopub.status.idle": "2026-07-30T14:10:07.814398Z",
+     "shell.execute_reply": "2026-07-30T14:10:07.801004Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "=== Size String Analysis ===\n",
+      "Unique size strings: 631\n",
+      "Null size strings:   0\n",
+      "\n",
+      "Top 20 size strings:\n",
+      "                      :   215\n",
+      "  400g                :   133\n",
+      "  200g                :   117\n",
+      "  500g                :   111\n",
+      "  300g                :    92\n",
+      "  454g                :    69\n",
+      "  250g                :    67\n",
+      "  1kg                 :    65\n",
+      "  450g                :    64\n",
+      "  1L                  :    61\n",
+      "  340g                :    59\n",
+      "  398ml               :    59\n",
+      "  600g                :    58\n",
+      "  12 per pack         :    57\n",
+      "  100g                :    56\n",
+      "  900g                :    55\n",
+      "  375g                :    54\n",
+      "  350g                :    50\n",
+      "  680g                :    50\n",
+      "  175g                :    46\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(\"=== Size String Analysis ===\")\n",
+    "print(f\"Unique size strings: {df['size'].nunique()}\")\n",
+    "print(f\"Null size strings:   {df['size'].isna().sum()}\")\n",
+    "print()\n",
+    "print(\"Top 20 size strings:\")\n",
+    "for size, count in df[\"size\"].value_counts().head(20).items():\n",
+    "    print(f\"  {size:20s}: {count:5d}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2026-07-30T14:10:07.830225Z",
+     "iopub.status.busy": "2026-07-30T14:10:07.829314Z",
+     "iopub.status.idle": "2026-07-30T14:10:07.857284Z",
+     "shell.execute_reply": "2026-07-30T14:10:07.853647Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "=== Size Amount Analysis ===\n",
+      "Null count:  307 (6.9%)\n",
+      "Non-null:    4133\n",
+      "Min:         0.27\n",
+      "Max:         980.0\n",
+      "Mean:        283.52\n",
+      "Median:      250.00\n",
+      "\n",
+      "=== Size Unit Distribution ===\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Null count:  307 (6.9%)\n",
+      "  g         :  2312\n",
+      "  ml        :   737\n",
+      "  count     :   568\n",
+      "  L         :   261\n",
+      "  kg        :   255\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(\"=== Size Amount Analysis ===\")\n",
+    "sa = df[\"size_amount\"]\n",
+    "print(f\"Null count:  {sa.isna().sum()} ({sa.isna().sum()/len(df)*100:.1f}%)\")\n",
+    "print(f\"Non-null:    {sa.notna().sum()}\")\n",
+    "if sa.notna().any():\n",
+    "    print(f\"Min:         {sa.min()}\")\n",
+    "    print(f\"Max:         {sa.max()}\")\n",
+    "    print(f\"Mean:        {sa.mean():.2f}\")\n",
+    "    print(f\"Median:      {sa.median():.2f}\")\n",
+    "print()\n",
+    "print(\"=== Size Unit Distribution ===\")\n",
+    "su = df[\"size_unit\"]\n",
+    "print(f\"Null count:  {su.isna().sum()} ({su.isna().sum()/len(df)*100:.1f}%)\")\n",
+    "for unit, count in su.value_counts().items():\n",
+    "    print(f\"  {unit:10s}: {count:5d}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 10. Price Analysis"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2026-07-30T14:10:07.864737Z",
+     "iopub.status.busy": "2026-07-30T14:10:07.862359Z",
+     "iopub.status.idle": "2026-07-30T14:10:07.884735Z",
+     "shell.execute_reply": "2026-07-30T14:10:07.881977Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "=== Price Analysis ==="
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "Null count: 0\n",
+      "Min:        $0.01\n",
+      "Max:        $2152.80\n",
+      "Mean:       $7.56\n",
+      "Median:     $5.79\n",
+      "Unique:     247\n",
+      "Currency:   <ArrowStringArray>\n",
+      "['CAD']\n",
+      "Length: 1, dtype: str\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(\"=== Price Analysis ===\")\n",
+    "p = df[\"price\"]\n",
+    "print(f\"Null count: {p.isna().sum()}\")\n",
+    "print(f\"Min:        ${p.min():.2f}\")\n",
+    "print(f\"Max:        ${p.max():.2f}\")\n",
+    "print(f\"Mean:       ${p.mean():.2f}\")\n",
+    "print(f\"Median:     ${p.median():.2f}\")\n",
+    "print(f\"Unique:     {p.nunique()}\")\n",
+    "print(f\"Currency:   {df['price_currency'].unique()}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 11. Provenance Documentation"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2026-07-30T14:10:07.891601Z",
+     "iopub.status.busy": "2026-07-30T14:10:07.889144Z",
+     "iopub.status.idle": "2026-07-30T14:10:07.904981Z",
+     "shell.execute_reply": "2026-07-30T14:10:07.902154Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{\n",
+      "  \"source_dataset\": \"saraNour/compliments-brand/source_of_truth/products.parquet\",\n",
+      "  \"source_url\": \"https://huggingface.co/datasets/saraNour/compliments-brand/blob/main/source_of_truth/products.parquet\",\n",
+      "  \"source_type\": \"HuggingFace dataset (private)\",\n",
+      "  \"original_source\": \"Voila.ca (Loblaw) Compliments private-label products\",\n",
+      "  \"columns\": {\n",
+      "    \"upc\": \"Universal Product Code. Some reused across variants.\",\n",
+      "    \"external_id\": \"Voila retailer product ID. Unique per row.\",\n",
+      "    \"brand\": \"Product brand. 11 variants of Compliments/Sensations.\",\n",
+      "    \"title\": \"Raw product title from Voila.\",\n",
+      "    \"price\": \"Price in CAD.\",\n",
+      "    \"price_currency\": \"Always 'CAD'.\",\n",
+      "    \"size\": \"Raw size string from Voila.\",\n",
+      "    \"size_amount\": \"Parsed numeric size amount.\",\n",
+      "    \"size_unit\": \"Parsed size unit (g, kg, ml, l, count).\",\n",
+      "    \"size_qty\": \"Size quantity multiplier. Usually 1.\",\n",
+      "    \"size_per_unit\": \"100% null. DROPPED.\",\n",
+      "    \"size_unit_norm\": \"Normalized size unit.\",\n",
+      "    \"size_total\": \"100% null. DROPPED.\",\n",
+      "    \"image_url\": \"Product image URL from Voila.\",\n",
+      "    \"source\": \"Always 'voila'.\",\n",
+      "    \"source_url\": \"Product page URL on Voila.\"\n",
+      "  },\n",
+      "  \"dropped_columns\": [\n",
+      "    {\n",
+      "      \"column\": \"size_per_unit\",\n",
+      "      \"reason\": \"100% null (4,440/4,440 values are NaN).\"\n",
+      "    },\n",
+      "    {\n",
+      "      \"column\": \"size_total\",\n",
+      "      \"reason\": \"100% null (4,440/4,440 values are NaN).\"\n",
+      "    }\n",
+      "  ]\n",
+      "}\n"
+     ]
+    }
+   ],
+   "source": [
+    "provenance = {\n",
+    "    \"source_dataset\": f\"{HF_REPO}/{HF_FILE}\",\n",
+    "    \"source_url\": f\"https://huggingface.co/datasets/{HF_REPO}/blob/main/{HF_FILE}\",\n",
+    "    \"source_type\": \"HuggingFace dataset (private)\",\n",
+    "    \"original_source\": \"Voila.ca (Loblaw) Compliments private-label products\",\n",
+    "    \"columns\": {\n",
+    "        \"upc\": \"Universal Product Code. Some reused across variants.\",\n",
+    "        \"external_id\": \"Voila retailer product ID. Unique per row.\",\n",
+    "        \"brand\": \"Product brand. 11 variants of Compliments/Sensations.\",\n",
+    "        \"title\": \"Raw product title from Voila.\",\n",
+    "        \"price\": \"Price in CAD.\",\n",
+    "        \"price_currency\": \"Always 'CAD'.\",\n",
+    "        \"size\": \"Raw size string from Voila.\",\n",
+    "        \"size_amount\": \"Parsed numeric size amount.\",\n",
+    "        \"size_unit\": \"Parsed size unit (g, kg, ml, l, count).\",\n",
+    "        \"size_qty\": \"Size quantity multiplier. Usually 1.\",\n",
+    "        \"size_per_unit\": \"100% null. DROPPED.\",\n",
+    "        \"size_unit_norm\": \"Normalized size unit.\",\n",
+    "        \"size_total\": \"100% null. DROPPED.\",\n",
+    "        \"image_url\": \"Product image URL from Voila.\",\n",
+    "        \"source\": \"Always 'voila'.\",\n",
+    "        \"source_url\": \"Product page URL on Voila.\",\n",
+    "    },\n",
+    "    \"dropped_columns\": [\n",
+    "        {\"column\": \"size_per_unit\", \"reason\": \"100% null (4,440/4,440 values are NaN).\"},\n",
+    "        {\"column\": \"size_total\", \"reason\": \"100% null (4,440/4,440 values are NaN).\"},\n",
+    "    ],\n",
+    "}\n",
+    "print(json.dumps(provenance, indent=2))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 12. Build Outputs"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2026-07-30T14:10:07.909755Z",
+     "iopub.status.busy": "2026-07-30T14:10:07.909177Z",
+     "iopub.status.idle": "2026-07-30T14:10:07.928659Z",
+     "shell.execute_reply": "2026-07-30T14:10:07.925801Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Output shape: 4440 rows x 14 columns\n",
+      "Columns: ['upc', 'external_id', 'brand', 'title', 'price', 'price_currency', 'size', 'size_amount', 'size_unit', 'size_qty', 'size_unit_norm', 'image_url', 'source', 'source_url']\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Drop 100% null columns\n",
+    "df_out = df.drop(columns=cols_100pct_null)\n",
+    "print(f\"Output shape: {df_out.shape[0]} rows x {df_out.shape[1]} columns\")\n",
+    "print(f\"Columns: {list(df_out.columns)}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2026-07-30T14:10:07.935063Z",
+     "iopub.status.busy": "2026-07-30T14:10:07.933034Z",
+     "iopub.status.idle": "2026-07-30T14:10:07.948749Z",
+     "shell.execute_reply": "2026-07-30T14:10:07.946896Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Validation result: PASS\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Build validation report\n",
+    "validation = {\n",
+    "    \"version\": VERSION,\n",
+    "    \"timestamp\": TIMESTAMP,\n",
+    "    \"result\": \"PASS\",\n",
+    "    \"failures\": [],\n",
+    "    \"checks\": {\n",
+    "        \"row_count\": {\"expected\": 4440, \"actual\": len(df), \"pass\": len(df) == 4440},\n",
+    "        \"column_count\": {\"expected\": 16, \"actual\": len(df.columns), \"pass\": len(df.columns) == 16},\n",
+    "        \"columns_dropped\": cols_100pct_null,\n",
+    "        \"full_row_duplicates\": {\"count\": int(full_dupes), \"pass\": full_dupes == 0},\n",
+    "        \"external_id_duplicates\": {\"count\": int(ext_dupes), \"pass\": ext_dupes == 0},\n",
+    "    },\n",
+    "}\n",
+    "print(f\"Validation result: {validation['result']}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2026-07-30T14:10:07.954161Z",
+     "iopub.status.busy": "2026-07-30T14:10:07.953613Z",
+     "iopub.status.idle": "2026-07-30T14:10:07.966699Z",
+     "shell.execute_reply": "2026-07-30T14:10:07.965433Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{\n",
+      "  \"version\": \"1.0.0\",\n",
+      "  \"timestamp\": \"2026-07-30T14:10:06.514655+00:00\",\n",
+      "  \"input\": {\n",
+      "    \"source\": \"saraNour/compliments-brand/source_of_truth/products.parquet\",\n",
+      "    \"row_count\": 4440,\n",
+      "    \"column_count\": 16\n",
+      "  },\n",
+      "  \"output\": {\n",
+      "    \"row_count\": 4440,\n",
+      "    \"column_count\": 14,\n",
+      "    \"columns_dropped\": [\n",
+      "      \"size_per_unit\",\n",
+      "      \"size_total\"\n",
+      "    ]\n",
+      "  },\n",
+      "  \"summary\": {\n",
+      "    \"unique_upcs\": 3271,\n",
+      "    \"reused_upcs\": 774,\n",
+      "    \"null_upcs\": 1,\n",
+      "    \"unique_brands\": 11,\n",
+      "    \"unique_size_strings\": 631\n",
+      "  }\n",
+      "}\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Build statistics\n",
+    "statistics = {\n",
+    "    \"version\": VERSION,\n",
+    "    \"timestamp\": TIMESTAMP,\n",
+    "    \"input\": {\"source\": f\"{HF_REPO}/{HF_FILE}\", \"row_count\": len(df), \"column_count\": len(df.columns)},\n",
+    "    \"output\": {\"row_count\": len(df_out), \"column_count\": len(df_out.columns), \"columns_dropped\": cols_100pct_null},\n",
+    "    \"summary\": {\n",
+    "        \"unique_upcs\": int(df[\"upc\"].nunique()),\n",
+    "        \"reused_upcs\": int(len(reused)),\n",
+    "        \"null_upcs\": int(df[\"upc\"].isna().sum()),\n",
+    "        \"unique_brands\": int(brand_counts.shape[0]),\n",
+    "        \"unique_size_strings\": int(df[\"size\"].nunique()),\n",
+    "    },\n",
+    "}\n",
+    "print(json.dumps(statistics, indent=2))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 13. Save Outputs"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2026-07-30T14:10:07.992090Z",
+     "iopub.status.busy": "2026-07-30T14:10:07.989963Z",
+     "iopub.status.idle": "2026-07-30T14:10:08.085599Z",
+     "shell.execute_reply": "2026-07-30T14:10:08.083881Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Saved: outputs/phase1_output.parquet\n",
+      "Saved: validation/phase1_validation.json\n",
+      "Saved: statistics/phase1_statistics.json\n",
+      "Saved: outputs/phase1_provenance.json\n"
+     ]
+    }
+   ],
+   "source": [
+    "OUTPUT_DIR = Path(\"outputs\")\n",
+    "VALIDATION_DIR = Path(\"validation\")\n",
+    "STATISTICS_DIR = Path(\"statistics\")\n",
+    "\n",
+    "OUTPUT_DIR.mkdir(parents=True, exist_ok=True)\n",
+    "VALIDATION_DIR.mkdir(parents=True, exist_ok=True)\n",
+    "STATISTICS_DIR.mkdir(parents=True, exist_ok=True)\n",
+    "\n",
+    "df_out.to_parquet(OUTPUT_DIR / \"phase1_output.parquet\", index=False)\n",
+    "print(f\"Saved: {OUTPUT_DIR / 'phase1_output.parquet'}\")\n",
+    "\n",
+    "with open(VALIDATION_DIR / \"phase1_validation.json\", \"w\") as f:\n",
+    "    json.dump(validation, f, indent=2, default=str)\n",
+    "print(f\"Saved: {VALIDATION_DIR / 'phase1_validation.json'}\")\n",
+    "\n",
+    "with open(STATISTICS_DIR / \"phase1_statistics.json\", \"w\") as f:\n",
+    "    json.dump(statistics, f, indent=2, default=str)\n",
+    "print(f\"Saved: {STATISTICS_DIR / 'phase1_statistics.json'}\")\n",
+    "\n",
+    "with open(OUTPUT_DIR / \"phase1_provenance.json\", \"w\") as f:\n",
+    "    json.dump(provenance, f, indent=2, default=str)\n",
+    "print(f\"Saved: {OUTPUT_DIR / 'phase1_provenance.json'}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 14. Summary"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2026-07-30T14:10:08.108443Z",
+     "iopub.status.busy": "2026-07-30T14:10:08.102152Z",
+     "iopub.status.idle": "2026-07-30T14:10:08.124049Z",
+     "shell.execute_reply": "2026-07-30T14:10:08.119790Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "============================================================\n",
+      "PHASE 1 COMPLETE\n",
+      "============================================================\n",
+      "Input:    4440 rows, 16 columns\n",
+      "Output:   4440 rows, 14 columns\n",
+      "Dropped:  ['size_per_unit', 'size_total']\n",
+      "Validation: PASS\n",
+      "============================================================\n",
+      "\n",
+      "Output columns:\n",
+      "   1. upc\n",
+      "   2. external_id\n",
+      "   3. brand\n",
+      "   4. title\n",
+      "   5. price\n",
+      "   6. price_currency\n",
+      "   7. size\n",
+      "   8. size_amount\n",
+      "   9. size_unit\n",
+      "  10. size_qty\n",
+      "  11. size_unit_norm\n",
+      "  12. image_url\n",
+      "  13. source\n",
+      "  14. source_url\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(\"=\" * 60)\n",
+    "print(\"PHASE 1 COMPLETE\")\n",
+    "print(\"=\" * 60)\n",
+    "print(f\"Input:    {df.shape[0]} rows, {df.shape[1]} columns\")\n",
+    "print(f\"Output:   {df_out.shape[0]} rows, {df_out.shape[1]} columns\")\n",
+    "print(f\"Dropped:  {cols_100pct_null}\")\n",
+    "print(f\"Validation: {validation['result']}\")\n",
+    "print(\"=\" * 60)\n",
+    "print()\n",
+    "print(\"Output columns:\")\n",
+    "for i, col in enumerate(df_out.columns):\n",
+    "    print(f\"  {i+1:2d}. {col}\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.13.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}