{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# SHAP explainability of the spatial RF models\n", "\n", "SHAP values of the final models for each age group, importance with the direction of association (Fig. S6),\n", "dependence plots (Fig. S7) and colored versions (Fig. S13).\n", "\n", "Input: `RF_spatial_data.csv` and `RF_spatial_{LRI|CRD|Other}_admissions_per_1_million_all_groups.pkl`." ] }, { "cell_type": "code", "metadata": {}, "source": [ "# Libraries\n", "import numpy as np\n", "import pandas as pd\n", "import joblib\n", "import shap\n", "import matplotlib.pyplot as plt" ], "execution_count": null, "outputs": [] }, { "cell_type": "code", "metadata": {}, "source": [ "# Load input data and models\n", "q = input(\"Load which disease group: 'LRI', 'CRD' or 'Other'? \").strip()\n", "if q not in ['LRI', 'CRD', 'Other']:\n", " raise ValueError(\"Invalid option!\")\n", "\n", "kind = 'spatial'\n", "\n", "df = pd.read_csv('.../path/to/input/RF_spatial_data.csv')\n", "\n", "targets = ['LRI', 'LRI_G1', 'LRI_G2', 'LRI_G3',\n", " 'CRD', 'CRD_G1', 'CRD_G2', 'CRD_G3',\n", " 'Other', 'Other_G1', 'Other_G2', 'Other_G3']\n", "\n", "df = df.dropna(subset=targets).reset_index(drop=True)\n", "\n", "models = joblib.load(f'.../path/to/models/RF_spatial_{q}_admissions_per_1_million_all_groups.pkl')\n", "\n", "X = df[list(models[q + '_General'].feature_names_in_)]\n", "print(X.shape)" ], "execution_count": null, "outputs": [] }, { "cell_type": "code", "metadata": {}, "source": [ "# SHAP values for each age group\n", "groups = ['General', 'G1', 'G2', 'G3']\n", "shap_values = {}\n", "\n", "for g in groups:\n", " model = models[q + '_' + g]\n", " explainer = shap.TreeExplainer(model)\n", "\n", " shap_values[g] = pd.DataFrame(explainer.shap_values(X), columns=X.columns)\n", " shap_values[g].to_csv(f'SHAP_{kind}_{q}_{g}.csv', index=False)\n", "\n", " print(g, 'done')" ], "execution_count": null, "outputs": [] }, { "cell_type": "code", "metadata": {}, "source": [ "# Mean |SHAP| and direction of association (sign of the Pearson correlation between feature and SHAP values)\n", "def importance(S, X):\n", " out = pd.DataFrame(index=X.columns)\n", " out['mean_abs_shap'] = S.abs().mean()\n", " out['positive'] = [X[c].corr(S[c]) > 0 for c in X.columns]\n", " return out.sort_values('mean_abs_shap', ascending=False)\n", "\n", "\n", "# Top 10 features (Fig. S6, log scale) and sum of the others\n", "n_top = 10\n", "\n", "fig, axes = plt.subplots(2, 2, figsize=(11, 9))\n", "axes = axes.flatten()\n", "\n", "for i, g in enumerate(groups):\n", " imp = importance(shap_values[g], X)\n", " top = imp.iloc[:n_top]\n", " rest = imp['mean_abs_shap'].iloc[n_top:].sum()\n", "\n", " ax = axes[i]\n", " for k, (name, row) in enumerate(top.iterrows()):\n", " if row['positive']:\n", " ax.barh(k, row['mean_abs_shap'], color='#ff0055')\n", " sign = '(+)'\n", " else:\n", " ax.barh(k, row['mean_abs_shap'], color='#0088ff', hatch='////', edgecolor='white')\n", " sign = '(-)'\n", " ax.text(row['mean_abs_shap'], k, f' {row[\"mean_abs_shap\"]:.2f} {sign}', va='center', fontsize=8)\n", "\n", " ax.barh(n_top, rest, color='0.6')\n", " ax.text(rest, n_top, f' {rest:.2f}', va='center', fontsize=8)\n", "\n", " ax.set_yticks(range(n_top + 1))\n", " ax.set_yticklabels(list(top.index) + ['Other features'])\n", " ax.invert_yaxis()\n", " ax.set_xlabel('mean |SHAP value|')\n", " ax.set_xscale('log')\n", " ax.set_title(q + ' ' + g)\n", "\n", "plt.tight_layout()\n", "plt.show()" ], "execution_count": null, "outputs": [] }, { "cell_type": "code", "metadata": {}, "source": [ "# LOWESS trend (tricube weights) with a 95% bootstrap band\n", "def lowess(x, y, frac=0.2):\n", " grid = np.linspace(np.percentile(x, 1), np.percentile(x, 99), 150)\n", " k = max(int(frac * len(x)), 10)\n", " fit = np.zeros(len(grid))\n", "\n", " for i in range(len(grid)):\n", " d = np.abs(x - grid[i])\n", " h = max(np.sort(d)[k - 1], 1e-9) # window half-width\n", " w = np.clip(1 - (d / h) ** 3, 0, None) ** 3\n", "\n", " xm = np.sum(w * x) / np.sum(w)\n", " ym = np.sum(w * y) / np.sum(w)\n", " b = np.sum(w * (x - xm) * (y - ym)) / max(np.sum(w * (x - xm) ** 2), 1e-12)\n", " fit[i] = ym + b * (grid[i] - xm)\n", "\n", " return grid, fit\n", "\n", "\n", "def lowess_band(x, y, n_boot=200):\n", " rng = np.random.default_rng(0)\n", " grid, fit = lowess(x, y)\n", "\n", " boots = []\n", " for i in range(n_boot):\n", " idx = rng.integers(0, len(x), len(x))\n", " g, f = lowess(x[idx], y[idx])\n", " boots.append(np.interp(grid, g, f))\n", "\n", " lo, hi = np.percentile(boots, [2.5, 97.5], axis=0)\n", " return grid, fit, lo, hi\n", "\n", "\n", "def feature_values(c):\n", " x = df[c].values.astype(float)\n", " if c == 'unix_time':\n", " x = 1970 + x / 31557600 # calendar year\n", " return x" ], "execution_count": null, "outputs": [] }, { "cell_type": "code", "metadata": {}, "source": [ "# Dependence plots of the 4 most important features (Fig. S7)\n", "if q == 'CRD':\n", " g = 'G3'\n", "else:\n", " g = 'G1'\n", "\n", "S = shap_values[g]\n", "features = list(importance(S, X).index[:4])\n", "\n", "# Fig. S7 shows RD, WHT, PCI and LCZ_6 for LRI G1 (LCZ_6 ranks fifth, after YEL)\n", "if q == 'LRI':\n", " features = ['RD', 'WHT', 'PCI', 'LCZ_6']\n", "\n", "fig, axes = plt.subplots(2, 2, figsize=(10, 7))\n", "axes = axes.flatten()\n", "\n", "for i, c in enumerate(features):\n", " x = feature_values(c)\n", " y = S[c].values\n", " ax = axes[i]\n", "\n", " ax.axhline(0, color='k', lw=0.6, ls='--')\n", "\n", " if set(np.unique(x)) <= {0, 1}: # binary feature: mean and 95% CI\n", " ax.scatter(x, y, s=7, color='#0088ff', alpha=0.45)\n", " for v in [0, 1]:\n", " yv = y[x == v]\n", " ax.errorbar(v, yv.mean(), yerr=1.96 * yv.std() / np.sqrt(len(yv)), fmt='o', color='#ff0055', capsize=4)\n", " ax.set_xticks([0, 1])\n", " else:\n", " ax.scatter(x, y, s=7, color='#0088ff', alpha=0.45)\n", " grid, fit, lo, hi = lowess_band(x, y)\n", " ax.fill_between(grid, lo, hi, color='#ff0055', alpha=0.25)\n", " ax.plot(grid, fit, color='#ff0055', lw=1.6)\n", "\n", " # feature distribution\n", " y0, y1 = ax.get_ylim()\n", " counts, edges = np.histogram(x, bins=40)\n", " ax.bar(edges[:-1], 0.15 * (y1 - y0) * counts / counts.max(), width=np.diff(edges), bottom=y0,\n", " align='edge', color='0.6', alpha=0.5)\n", "\n", " ax.set_xlabel(c)\n", " ax.set_ylabel('SHAP value')\n", " ax.set_title(q + ' ' + g)\n", "\n", "plt.tight_layout()\n", "plt.show()" ], "execution_count": null, "outputs": [] }, { "cell_type": "code", "metadata": {}, "source": [ "# Dependence plots colored by the most strongly correlated feature (Fig. S13)\n", "# (highest absolute Spearman correlation with the SHAP values of the plotted feature)\n", "fig, axes = plt.subplots(2, 2, figsize=(11, 7))\n", "axes = axes.flatten()\n", "\n", "for i, c in enumerate(features):\n", " r = {}\n", " for other in X.columns:\n", " if other != c and X[other].std() > 0:\n", " r[other] = abs(S[c].corr(X[other], method='spearman'))\n", " partner = max(r, key=r.get)\n", "\n", " sc = axes[i].scatter(feature_values(c), S[c].values, s=7, c=feature_values(partner), cmap='viridis')\n", " plt.colorbar(sc, ax=axes[i]).set_label(partner)\n", "\n", " axes[i].axhline(0, color='k', lw=0.6, ls='--')\n", " axes[i].set_xlabel(c)\n", " axes[i].set_ylabel('SHAP value')\n", "\n", "plt.tight_layout()\n", "plt.show()" ], "execution_count": null, "outputs": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "name": "python" } }, "nbformat": 4, "nbformat_minor": 5 }