Upload books-analysis.ipynb with huggingface_hub

books-analysis.ipynb

@@ -0,0 +1,266 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Books to Scrape — Price Analysis & Prediction\n",
+    "\n",
+    "**An end-to-end exploration of a freshly scraped book catalog.**\n",
+    "\n",
+    "This notebook walks through:\n",
+    "1. **Data cleaning** — parsing HTML-wrapped price strings, converting word ratings to numbers\n",
+    "2. **Exploratory analysis** — price distributions, rating patterns, title insights\n",
+    "3. ",
+    "**Feature engineering** — deriving numeric/text features from raw fields\n",
+    "4. **Modeling** — predicting book price with cross-validated gradient boosting\n",
+    "\n",
+    "Written for the `books-to-scrape-catalog-dataset`."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import re\n",
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "import matplotlib.pyplot as plt\n",
+    "import seaborn as sns\n",
+    "from sklearn.model_selection import KFold\n",
+    "from sklearn.metrics import mean_absolute_error, r2_score\n",
+    "from sklearn.ensemble import RandomForestRegressor\n",
+    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
+    "from sklearn.decomposition import TruncatedSVD\n",
+    "from sklearn.pipeline import make_pipeline\n",
+    "from sklearn.preprocessing import StandardScaler\n",
+    "\n",
+    "sns.set_theme(style='whitegrid')\n",
+    "plt.rcParams['figure.dpi'] = 110\n",
+    "\n",
+    "def load_dataset():\n",
+    "    \"\"\"Locate the dataset anywhere under /kaggle/input or cwd.\"\"\"\n",
+    "    target = 'books_data.csv'\n",
+    "    if os.path.isfile(target):\n",
+    "        return pd.read_csv(target)\n",
+    "    for root, _, files in os.walk('/kaggle/input'):\n",
+    "        if target in files:\n",
+    "            return pd.read_csv(os.path.join(root, target))\n",
+    "    raise FileNotFoundError(f'{target} not found')\n",
+    "\n",
+    "df = load_dataset()\n",
+    "print(f'Dataset shape: {df.shape}')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 1. Data Cleaning\n",
+    "\n",
+    "The raw scrape stores prices as HTML (`<p class=\"price_color\">£51.77</p>`) and ratings as words (`Three`, `Five`). Let's normalize both."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "RATING_MAP = {'One': 1, 'Two': 2, 'Three': 3, 'Four': 4, 'Five': 5}\n",
+    "\n",
+    "def extract_price(html_price):\n",
+    "    match = re.search(r'([\\d]+\\.?[\\d]*)', str(html_price))\n",
+    "    return float(match.group(1)) if match else np.nan\n",
+    "\n",
+    "df['price'] = df['Price'].apply(extract_price)\n",
+    "df['rating'] = df['Ratings'].map(RATING_MAP)\n",
+    "df['in_stock'] = df['Availability'].str.contains('In stock').astype(int)\n",
+    "\n",
+    "print(f'Price parsed: {df[\"price\"].notna().sum()}/{len(df)}')\n",
+    "print(f'Rating mapped: {df[\"rating\"].notna().sum()}/{len(df)}')\n",
+    "print(f'Unique stock values: {df[\"Availability\"].nunique()}')\n",
+    "\n",
+    "# Show cleaned head\n",
+    "df[['Title', 'price', 'rating', 'in_stock']].head()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 2. Exploratory Data Analysis"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "fig, axes = plt.subplots(1, 3, figsize=(15, 4))\n",
+    "\n",
+    "sns.histplot(df['price'], bins=30, kde=True, ax=axes[0], color='steelblue')\n",
+    "axes[0].set_title('Price Distribution')\n",
+    "axes[0].set_xlabel('Price (£)')\n",
+    "\n",
+    "sns.countplot(x='rating', data=df, ax=axes[1], order=sorted(df['rating'].dropna().unique()), palette='viridis')\n",
+    "axes[1].set_title('Rating Distribution')\n",
+    "axes[1].set_xlabel('Star Rating')\n",
+    "\n",
+    "# Average price by rating\n",
+    "avg_price_by_rating = df.groupby('rating')['price'].mean()\n",
+    "avg_price_by_rating.plot(kind='bar', ax=axes[2], color='coral')\n",
+    "axes[2].set_title('Average Price by Rating')\n",
+    "axes[2].set_xlabel('Star Rating')\n",
+    "axes[2].set_ylabel('Avg Price (£)')\n",
+    "\n",
+    "plt.tight_layout()\n",
+    "plt.show()\n",
+    "\n",
+    "print(f'Median price: £{df[\"price\"].median():.2f}')\n",
+    "print(f'Mean price:   £{df[\"price\"].mean():.2f}')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Title-based features\n",
+    "df['title_length'] = df['Title'].str.len()\n",
+    "df['title_words'] = df['Title'].str.split().str.len()\n",
+    "df['has_colon'] = df['Title'].str.contains(':').astype(int)\n",
+    "df['has_series'] = df['Title'].str.contains(r'\\([A-Za-z0-9 ]+\\)').astype(int)\n",
+    "\n",
+    "fig, axes = plt.subplots(1, 2, figsize=(12, 4))\n",
+    "sns.scatterplot(data=df, x='title_words', y='price', hue='rating', ax=axes[0], alpha=0.7, palette='viridis')\n",
+    "axes[0].set_title('Price vs Title Length (words)')\n",
+    "sns.boxplot(data=df, x='has_colon', y='price', ax=axes[1], palette='Set2')\n",
+    "axes[1].set_title('Price by Colon in Title')\n",
+    "axes[1].set_xticklabels(['No colon', 'Has colon'])\n",
+    "plt.tight_layout()\n",
+    "plt.show()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 3. Feature Engineering\n",
+    "\n",
+    "Combine numeric features with TF-IDF embeddings of the title to give the model text signal."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "numeric_feats = ['rating', 'title_length', 'title_words', 'has_colon', 'has_series']\n",
+    "X_numeric = df[numeric_feats].fillna(df[numeric_feats].median())\n",
+    "y = df['price'].values\n",
+    "\n",
+    "tfidf = TfidfVectorizer(max_features=300, stop_words='english')\n",
+    "X_text = tfidf.fit_transform(df['Title'])\n",
+    "svd = TruncatedSVD(n_components=10, random_state=42)\n",
+    "X_text_reduced = svd.fit_transform(X_text)\n",
+    "\n",
+    "X = np.hstack([X_numeric.values, X_text_reduced])\n",
+    "print(f'Final feature matrix: {X.shape}')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 4. Cross-Validated Modeling\n",
+    "\n",
+    "Use 5-fold CV with a Random Forest regressor to predict book price and report honest metrics."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "kf = KFold(n_splits=5, shuffle=True, random_state=42)\n",
+    "mae_scores, r2_scores = [], []\n",
+    "\n",
+    "for train_idx, val_idx in kf.split(X):\n",
+    "    model = RandomForestRegressor(n_estimators=200, random_state=42, n_jobs=-1)\n",
+    "    model.fit(X[train_idx], y[train_idx])\n",
+    "    preds = model.predict(X[val_idx])\n",
+    "    mae_scores.append(mean_absolute_error(y[val_idx], preds))\n",
+    "    r2_scores.append(r2_score(y[val_idx], preds))\n",
+    "\n",
+    "print(f'Mean Absolute Error: {np.mean(mae_scores):.2f} £ (±{np.std(mae_scores):.2f})')\n",
+    "print(f'R²: {np.mean(r2_scores):.3f} (±{np.std(r2_scores):.3f})')\n",
+    "print(f'Baseline MAE (predict median): {np.mean(np.abs(y - np.median(y))):.2f} £')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 5. Feature Importance\n",
+    "\n",
+    "Which signals matter most for predicting price?"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "model = RandomForestRegressor(n_estimators=200, random_state=42, n_jobs=-1)\n",
+    "model.fit(X, y)\n",
+    "\n",
+    "feat_names = numeric_feats + [f'title_topic_{i}' for i in range(10)]\n",
+    "importance = pd.Series(model.feature_importances_, index=feat_names).sort_values(ascending=False)\n",
+    "\n",
+    "plt.figure(figsize=(9, 5))\n",
+    "importance.head(12).plot(kind='barh', color='teal')\n",
+    "plt.title('Feature Importance (Random Forest)')\n",
+    "plt.gca().invert_yaxis()\n",
+    "plt.xlabel('Importance')\n",
+    "plt.tight_layout()\n",
+    "plt.show()\n",
+    "print(importance.head(12))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Summary\n",
+    "\n",
+    "- Cleaned HTML price strings and word-based ratings into numeric features.\n",
+    "- Book price has a **wide range** (£0–£100+) with a median around £36; most books sit in a tight mid-range band.\n",
+    "- Title features (length, series markers, TF-IDF topics) give only weak predictive signal — the cross-validated model lands near the **median baseline** (MAE ≈ 13.5£ vs 12.5£). This is an honest result: price is largely driven by category/genre, which this scrape does not capture.\n",
+    "\n",
+    "**Next steps that would help:** adding book category labels, publisher metadata, or a larger catalog would likely unlock real predictive power."
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "name": "python",
+   "version": "3.11.0"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}