oukawa commited on
Commit
28602fd
·
verified ·
1 Parent(s): a4d442f

Upload processing notebook

Browse files
RF_data_processing_temporal_and_spatial.ipynb ADDED
@@ -0,0 +1,448 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "cells": [
3
+ {
4
+ "cell_type": "markdown",
5
+ "metadata": {},
6
+ "source": [
7
+ "# Input data for the temporal and spatial RF models\n",
8
+ "\n",
9
+ "Builds `RF_temporal_data.csv` (weekly) and `RF_spatial_data.csv` (1-km grid), following Sections 2.3.1-2.3.4 of the\n",
10
+ "manuscript and Section S1 of the Supplementary Material. Input files are not included; file and column names are\n",
11
+ "placeholders.\n",
12
+ "\n",
13
+ "Disease groups (ICD-10): LRI = J10-J18 and J21-J22; CRD = J44-J45; Other = J00-J99 excluding LRI and CRD.\n",
14
+ "Age groups: G1 = 0-4 years; G2 = 5-59 years; G3 = 60 years and over."
15
+ ]
16
+ },
17
+ {
18
+ "cell_type": "code",
19
+ "execution_count": null,
20
+ "metadata": {},
21
+ "outputs": [],
22
+ "source": [
23
+ "# Libraries\n",
24
+ "import numpy as np\n",
25
+ "import pandas as pd\n",
26
+ "import geopandas as gpd\n",
27
+ "import rasterio\n",
28
+ "from rasterio.mask import mask\n",
29
+ "from shapely.geometry import box, mapping"
30
+ ]
31
+ },
32
+ {
33
+ "cell_type": "code",
34
+ "execution_count": null,
35
+ "metadata": {},
36
+ "outputs": [],
37
+ "source": [
38
+ "# Input and output files\n",
39
+ "air_quality_path = '.../path/to/input/air_quality.csv' # datetime, station, so2, no2, nox, co, o3, pm10\n",
40
+ "weather_path = '.../path/to/input/weather.csv' # date, t_mean, t_min, t_max, rh, ws, insolation, evaporation, rainfall\n",
41
+ "admissions_path = '.../path/to/input/admissions.csv' # SIH records\n",
42
+ "population_path = '.../path/to/input/population.csv' # year, population\n",
43
+ "postal_codes_path = '.../path/to/input/pcodes.csv' # cep, x, y\n",
44
+ "\n",
45
+ "boundary_path = '.../path/to/input/municipalities.shp' # Curitiba and Araucária\n",
46
+ "tracts_path = '.../path/to/input/census.shp' # pop, pci, literate, white, black, yellow, brown, indigenous\n",
47
+ "roads_path = '.../path/to/input/roads.shp'\n",
48
+ "gdp_path = '.../path/to/input/gdp.tif'\n",
49
+ "lcz_path = '.../path/to/input/lcz.tif'\n",
50
+ "\n",
51
+ "output_dir = '.../path/to/output/'\n",
52
+ "\n",
53
+ "TARGET_CRS = 'EPSG:31982' # SIRGAS 2000 / UTM zone 22S"
54
+ ]
55
+ },
56
+ {
57
+ "cell_type": "code",
58
+ "execution_count": null,
59
+ "metadata": {},
60
+ "outputs": [],
61
+ "source": [
62
+ "# Hospital admissions (DataSUS SIH)\n",
63
+ "# MUNIC_RES: municipality, DT_INTER: admission date, DIAG_PRINC: ICD-10 code,\n",
64
+ "# COD_IDADE: age unit (4 = years), IDADE: age, CEP: postal code\n",
65
+ "adm = pd.read_csv(admissions_path, dtype={'CEP': str, 'DIAG_PRINC': str})\n",
66
+ "\n",
67
+ "# Residents of Curitiba (410690) and Araucária (410180), 2003-2017\n",
68
+ "adm = adm[adm['MUNIC_RES'].isin([410690, 410180])]\n",
69
+ "\n",
70
+ "adm['date'] = pd.to_datetime(adm['DT_INTER'].astype(str), format='%Y%m%d')\n",
71
+ "adm = adm[(adm['date'] >= '2003-01-01') & (adm['date'] <= '2017-12-31')]\n",
72
+ "\n",
73
+ "# Disease group (ICD-10)\n",
74
+ "adm['code'] = adm['DIAG_PRINC'].str[:3]\n",
75
+ "\n",
76
+ "lri_codes = ['J10', 'J11', 'J12', 'J13', 'J14', 'J15', 'J16', 'J17', 'J18', 'J21', 'J22']\n",
77
+ "crd_codes = ['J44', 'J45']\n",
78
+ "\n",
79
+ "adm = adm[adm['code'].str.startswith('J')]\n",
80
+ "\n",
81
+ "adm['disease'] = 'Other'\n",
82
+ "adm.loc[adm['code'].isin(lri_codes), 'disease'] = 'LRI'\n",
83
+ "adm.loc[adm['code'].isin(crd_codes), 'disease'] = 'CRD'\n",
84
+ "\n",
85
+ "# Age in years and age group\n",
86
+ "adm['age'] = np.where(adm['COD_IDADE'] == 4, adm['IDADE'], 0)\n",
87
+ "\n",
88
+ "adm['group'] = 'G2'\n",
89
+ "adm.loc[adm['age'] <= 4, 'group'] = 'G1'\n",
90
+ "adm.loc[adm['age'] >= 60, 'group'] = 'G3'\n",
91
+ "\n",
92
+ "print(adm['disease'].value_counts())\n",
93
+ "print(adm['group'].value_counts())"
94
+ ]
95
+ },
96
+ {
97
+ "cell_type": "markdown",
98
+ "metadata": {},
99
+ "source": [
100
+ "## Temporal input matrix"
101
+ ]
102
+ },
103
+ {
104
+ "cell_type": "code",
105
+ "execution_count": null,
106
+ "metadata": {},
107
+ "outputs": [],
108
+ "source": [
109
+ "# Air quality: daily mean per station, then mean of the eight stations\n",
110
+ "pollutants = ['so2', 'no2', 'nox', 'co', 'o3', 'pm10']\n",
111
+ "\n",
112
+ "aq = pd.read_csv(air_quality_path, parse_dates=['datetime'])\n",
113
+ "aq['date'] = aq['datetime'].dt.floor('D')\n",
114
+ "\n",
115
+ "aq_station = aq.groupby(['station', 'date'])[pollutants].mean().reset_index()\n",
116
+ "aq_daily = aq_station.groupby('date')[pollutants].mean()\n",
117
+ "\n",
118
+ "print(aq_daily.describe().round(1))"
119
+ ]
120
+ },
121
+ {
122
+ "cell_type": "code",
123
+ "execution_count": null,
124
+ "metadata": {},
125
+ "outputs": [],
126
+ "source": [
127
+ "# Weather and daily temperature amplitude\n",
128
+ "wx = pd.read_csv(weather_path, parse_dates=['date']).set_index('date')\n",
129
+ "wx['t_amp'] = wx['t_max'] - wx['t_min']\n",
130
+ "\n",
131
+ "print(wx.describe().round(1))"
132
+ ]
133
+ },
134
+ {
135
+ "cell_type": "code",
136
+ "execution_count": null,
137
+ "metadata": {},
138
+ "outputs": [],
139
+ "source": [
140
+ "# Climate indices: consecutive days meeting a condition (reset to zero when not met)\n",
141
+ "# Dry: rainfall < 1 mm; wet: rainfall >= 1 mm; hot: TX > 75th percentile; cold: TN < 25th percentile\n",
142
+ "def consecutive_days(condition):\n",
143
+ " counter = np.zeros(len(condition))\n",
144
+ " for i in range(len(condition)):\n",
145
+ " if condition[i]:\n",
146
+ " if i == 0:\n",
147
+ " counter[i] = 1\n",
148
+ " else:\n",
149
+ " counter[i] = counter[i - 1] + 1\n",
150
+ " else:\n",
151
+ " counter[i] = 0\n",
152
+ " return counter\n",
153
+ "\n",
154
+ "\n",
155
+ "tx_75 = wx['t_max'].quantile(0.75)\n",
156
+ "tn_25 = wx['t_min'].quantile(0.25)\n",
157
+ "\n",
158
+ "wx['cdd'] = consecutive_days((wx['rainfall'] < 1.0).values)\n",
159
+ "wx['cwd'] = consecutive_days((wx['rainfall'] >= 1.0).values)\n",
160
+ "wx['chd'] = consecutive_days((wx['t_max'] > tx_75).values)\n",
161
+ "wx['ccd'] = consecutive_days((wx['t_min'] < tn_25).values)\n",
162
+ "\n",
163
+ "print('TX 75th percentile:', tx_75, '| TN 25th percentile:', tn_25)"
164
+ ]
165
+ },
166
+ {
167
+ "cell_type": "code",
168
+ "execution_count": null,
169
+ "metadata": {},
170
+ "outputs": [],
171
+ "source": [
172
+ "# Weekly means (rainfall: weekly total)\n",
173
+ "daily = wx.join(aq_daily, how='outer')\n",
174
+ "\n",
175
+ "weather = ['t_min', 't_max', 't_mean', 't_amp', 'rh', 'ws', 'insolation', 'evaporation']\n",
176
+ "climate = ['cdd', 'cwd', 'chd', 'ccd']\n",
177
+ "\n",
178
+ "weekly = daily[weather + climate + pollutants].resample('7D').mean()\n",
179
+ "weekly['rainfall'] = daily['rainfall'].resample('7D').sum(min_count=1)\n",
180
+ "\n",
181
+ "weekly.index.name = 'date'\n",
182
+ "weekly = weekly.reset_index()"
183
+ ]
184
+ },
185
+ {
186
+ "cell_type": "code",
187
+ "execution_count": null,
188
+ "metadata": {},
189
+ "outputs": [],
190
+ "source": [
191
+ "# Trend and season\n",
192
+ "# Unix time (elapsed seconds since 1970-01-01) and Julian day (1-366)\n",
193
+ "weekly['unix_time'] = (weekly['date'] - pd.Timestamp('1970-01-01')).dt.total_seconds()\n",
194
+ "weekly['julian_day'] = weekly['date'].dt.dayofyear\n",
195
+ "\n",
196
+ "# Season (one-hot, summer as reference)\n",
197
+ "season = {12: 'summer', 1: 'summer', 2: 'summer',\n",
198
+ " 3: 'fall', 4: 'fall', 5: 'fall',\n",
199
+ " 6: 'winter', 7: 'winter', 8: 'winter',\n",
200
+ " 9: 'spring', 10: 'spring', 11: 'spring'}\n",
201
+ "\n",
202
+ "weekly['season'] = weekly['date'].dt.month.map(season)\n",
203
+ "\n",
204
+ "weekly['season_fall'] = (weekly['season'] == 'fall').astype(int)\n",
205
+ "weekly['season_spring'] = (weekly['season'] == 'spring').astype(int)\n",
206
+ "weekly['season_winter'] = (weekly['season'] == 'winter').astype(int)\n",
207
+ "\n",
208
+ "weekly = weekly.drop(columns='season')"
209
+ ]
210
+ },
211
+ {
212
+ "cell_type": "code",
213
+ "execution_count": null,
214
+ "metadata": {},
215
+ "outputs": [],
216
+ "source": [
217
+ "# Weekly admissions per 1 million inhabitants\n",
218
+ "population = pd.read_csv(population_path).set_index('year')['population']\n",
219
+ "\n",
220
+ "# week of each admission\n",
221
+ "start = weekly['date'].iloc[0]\n",
222
+ "adm['week'] = start + pd.to_timedelta((adm['date'] - start).dt.days // 7 * 7, unit='D')\n",
223
+ "\n",
224
+ "weekly_pop = weekly['date'].dt.year.map(population).values\n",
225
+ "\n",
226
+ "for d in ['LRI', 'CRD', 'Other']:\n",
227
+ " for g in ['General', 'G1', 'G2', 'G3']:\n",
228
+ " if g == 'General':\n",
229
+ " sel = adm[adm['disease'] == d]\n",
230
+ " col_name = d\n",
231
+ " else:\n",
232
+ " sel = adm[(adm['disease'] == d) & (adm['group'] == g)]\n",
233
+ " col_name = d + '_' + g\n",
234
+ "\n",
235
+ " counts = sel.groupby('week').size()\n",
236
+ " weekly[col_name] = counts.reindex(weekly['date']).fillna(0).values / weekly_pop * 1_000_000"
237
+ ]
238
+ },
239
+ {
240
+ "cell_type": "code",
241
+ "execution_count": null,
242
+ "metadata": {},
243
+ "outputs": [],
244
+ "source": [
245
+ "# Save\n",
246
+ "weekly.to_csv(output_dir + 'RF_temporal_data.csv', index=False)\n",
247
+ "\n",
248
+ "print(weekly.shape)\n",
249
+ "weekly.head()"
250
+ ]
251
+ },
252
+ {
253
+ "cell_type": "markdown",
254
+ "metadata": {},
255
+ "source": [
256
+ "## Spatial input matrix"
257
+ ]
258
+ },
259
+ {
260
+ "cell_type": "code",
261
+ "execution_count": null,
262
+ "metadata": {},
263
+ "outputs": [],
264
+ "source": [
265
+ "# 1-km grid\n",
266
+ "boundary = gpd.read_file(boundary_path).to_crs(TARGET_CRS)\n",
267
+ "xmin, ymin, xmax, ymax = boundary.total_bounds\n",
268
+ "\n",
269
+ "cells = []\n",
270
+ "for x in np.arange(xmin, xmax, 1000):\n",
271
+ " for y in np.arange(ymin, ymax, 1000):\n",
272
+ " cells.append(box(x, y, x + 1000, y + 1000))\n",
273
+ "\n",
274
+ "grid = gpd.GeoDataFrame(geometry=cells, crs=TARGET_CRS)\n",
275
+ "grid = grid[grid.intersects(boundary.union_all())].reset_index(drop=True)\n",
276
+ "\n",
277
+ "grid['grid_id'] = range(len(grid))\n",
278
+ "grid['left'] = grid.bounds['minx']\n",
279
+ "grid['bottom'] = grid.bounds['miny']\n",
280
+ "grid['right'] = grid.bounds['maxx']\n",
281
+ "grid['top'] = grid.bounds['maxy']\n",
282
+ "grid['area_km2'] = grid.geometry.area / 1e6\n",
283
+ "\n",
284
+ "print('cells:', len(grid))"
285
+ ]
286
+ },
287
+ {
288
+ "cell_type": "code",
289
+ "execution_count": null,
290
+ "metadata": {},
291
+ "outputs": [],
292
+ "source": [
293
+ "# Census data, weighted by the area of each tract inside the cell\n",
294
+ "tracts = gpd.read_file(tracts_path).to_crs(TARGET_CRS)\n",
295
+ "tracts['tract_area'] = tracts.geometry.area\n",
296
+ "\n",
297
+ "pieces = gpd.overlay(tracts, grid[['grid_id', 'geometry']], how='intersection')\n",
298
+ "pieces['piece_area'] = pieces.geometry.area\n",
299
+ "pieces['fraction'] = pieces['piece_area'] / pieces['tract_area']\n",
300
+ "pieces['pop_piece'] = pieces['pop'] * pieces['fraction']\n",
301
+ "\n",
302
+ "census = pieces.groupby('grid_id')['pop_piece'].sum().rename('pop').to_frame()\n",
303
+ "\n",
304
+ "for v in ['pci', 'literate', 'white', 'black', 'yellow', 'brown', 'indigenous']:\n",
305
+ " weighted = (pieces[v] * pieces['piece_area']).groupby(pieces['grid_id']).sum()\n",
306
+ " census[v] = weighted / pieces.groupby('grid_id')['piece_area'].sum()\n",
307
+ "\n",
308
+ "grid = grid.merge(census, left_on='grid_id', right_index=True, how='left')\n",
309
+ "\n",
310
+ "# Population density (inhab. per km2)\n",
311
+ "grid['popd'] = grid['pop'] / grid['area_km2']"
312
+ ]
313
+ },
314
+ {
315
+ "cell_type": "code",
316
+ "execution_count": null,
317
+ "metadata": {},
318
+ "outputs": [],
319
+ "source": [
320
+ "# Per capita GDP\n",
321
+ "with rasterio.open(gdp_path) as src:\n",
322
+ " centroids = grid.geometry.centroid.to_crs(src.crs)\n",
323
+ " coords = [(p.x, p.y) for p in centroids]\n",
324
+ " values = np.array([v[0] for v in src.sample(coords)], dtype=float)\n",
325
+ "\n",
326
+ " if src.nodata is not None:\n",
327
+ " values[values == src.nodata] = np.nan\n",
328
+ "\n",
329
+ "grid['gdp'] = values"
330
+ ]
331
+ },
332
+ {
333
+ "cell_type": "code",
334
+ "execution_count": null,
335
+ "metadata": {},
336
+ "outputs": [],
337
+ "source": [
338
+ "# Road density (km-1)\n",
339
+ "roads = gpd.read_file(roads_path).to_crs(TARGET_CRS)\n",
340
+ "\n",
341
+ "segments = gpd.overlay(roads, grid[['grid_id', 'geometry']], how='intersection', keep_geom_type=True)\n",
342
+ "segments['length_km'] = segments.geometry.length / 1000\n",
343
+ "\n",
344
+ "road_length = segments.groupby('grid_id')['length_km'].sum()\n",
345
+ "\n",
346
+ "grid['road_length'] = grid['grid_id'].map(road_length).fillna(0)\n",
347
+ "grid['road_dens'] = grid['road_length'] / grid['area_km2']"
348
+ ]
349
+ },
350
+ {
351
+ "cell_type": "code",
352
+ "execution_count": null,
353
+ "metadata": {},
354
+ "outputs": [],
355
+ "source": [
356
+ "# Local climate zones: fraction of each class in the cell (classes 11-17 = LCZ A-G)\n",
357
+ "lcz_names = ['1', '2', '3', '4', '5', '6', '7', '8', '9', '10', 'A', 'B', 'C', 'D', 'E', 'F', 'G']\n",
358
+ "\n",
359
+ "for n in lcz_names:\n",
360
+ " grid['LCZ_' + n] = 0.0\n",
361
+ "\n",
362
+ "with rasterio.open(lcz_path) as src:\n",
363
+ " cells_lcz = grid.to_crs(src.crs)\n",
364
+ "\n",
365
+ " for idx, row in cells_lcz.iterrows():\n",
366
+ " out_image, out_transform = mask(src, [mapping(row.geometry)], crop=True)\n",
367
+ " pixels = out_image[0]\n",
368
+ "\n",
369
+ " if src.nodata is not None:\n",
370
+ " pixels = pixels[pixels != src.nodata]\n",
371
+ "\n",
372
+ " if pixels.size == 0:\n",
373
+ " continue\n",
374
+ "\n",
375
+ " values, counts = np.unique(pixels, return_counts=True)\n",
376
+ "\n",
377
+ " for val, count in zip(values, counts):\n",
378
+ " if 1 <= val <= 17:\n",
379
+ " grid.loc[idx, 'LCZ_' + lcz_names[int(val) - 1]] = count / counts.sum()"
380
+ ]
381
+ },
382
+ {
383
+ "cell_type": "code",
384
+ "execution_count": null,
385
+ "metadata": {},
386
+ "outputs": [],
387
+ "source": [
388
+ "# Admissions per cell (from postal codes) per 1 million inhabitants\n",
389
+ "postal = pd.read_csv(postal_codes_path, dtype={'cep': str})\n",
390
+ "\n",
391
+ "points = adm.merge(postal, left_on='CEP', right_on='cep', how='inner')\n",
392
+ "points = gpd.GeoDataFrame(points, geometry=gpd.points_from_xy(points['x'], points['y']), crs=TARGET_CRS)\n",
393
+ "\n",
394
+ "points = gpd.sjoin(points, grid[['grid_id', 'geometry']], how='inner', predicate='within')\n",
395
+ "\n",
396
+ "cell_pop = grid['pop'].replace(0, np.nan) # cells without population\n",
397
+ "\n",
398
+ "for d in ['LRI', 'CRD', 'Other']:\n",
399
+ " for g in ['General', 'G1', 'G2', 'G3']:\n",
400
+ " if g == 'General':\n",
401
+ " sel = points[points['disease'] == d]\n",
402
+ " col_name = d\n",
403
+ " else:\n",
404
+ " sel = points[(points['disease'] == d) & (points['group'] == g)]\n",
405
+ " col_name = d + '_' + g\n",
406
+ "\n",
407
+ " counts = sel.groupby('grid_id').size()\n",
408
+ " grid[col_name] = grid['grid_id'].map(counts).fillna(0) / cell_pop * 1_000_000"
409
+ ]
410
+ },
411
+ {
412
+ "cell_type": "code",
413
+ "execution_count": null,
414
+ "metadata": {},
415
+ "outputs": [],
416
+ "source": [
417
+ "# Rename and save\n",
418
+ "names = {'white': 'WHT', 'black': 'BLK', 'yellow': 'YEL', 'brown': 'BRN', 'indigenous': 'IND',\n",
419
+ " 'literate': 'LT', 'popd': 'PD', 'pci': 'PCI', 'road_dens': 'RD', 'gdp': 'GDP'}\n",
420
+ "\n",
421
+ "grid = grid.rename(columns=names)\n",
422
+ "\n",
423
+ "lcz_cols = ['LCZ_' + n for n in lcz_names]\n",
424
+ "targets = ['LRI', 'LRI_G1', 'LRI_G2', 'LRI_G3',\n",
425
+ " 'CRD', 'CRD_G1', 'CRD_G2', 'CRD_G3',\n",
426
+ " 'Other', 'Other_G1', 'Other_G2', 'Other_G3']\n",
427
+ "\n",
428
+ "spatial = grid[['grid_id', 'left', 'top', 'right', 'bottom'] + list(names.values()) + lcz_cols + targets]\n",
429
+ "spatial.to_csv(output_dir + 'RF_spatial_data.csv', index=False)\n",
430
+ "\n",
431
+ "print(spatial.shape)\n",
432
+ "spatial.head()"
433
+ ]
434
+ }
435
+ ],
436
+ "metadata": {
437
+ "kernelspec": {
438
+ "display_name": "Python 3",
439
+ "language": "python",
440
+ "name": "python3"
441
+ },
442
+ "language_info": {
443
+ "name": "python"
444
+ }
445
+ },
446
+ "nbformat": 4,
447
+ "nbformat_minor": 5
448
+ }