{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import pymongo\n", "import json\n" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "## First we estsablish the actual elligible names:\n", "\n", "I0 = json.load(open('../maps/regions_I0.geojson','r'))\n", "\n", "all_regions = []\n", "\n", "for feature in I0['features']:\n", " all_regions.append(feature['properties']['reg_name'])" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'Piemonte': 0,\n", " \"Valle d'Aosta/Vallée d'Aoste\": 1,\n", " 'Liguria': 2,\n", " \"Valle d'Aosta\": 1}" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "regions_indexer = {item: index for index, item in enumerate(all_regions)}\n", "regions_indexer[\"Valle d'Aosta\"] = 1\n", "\n", "regions_indexer" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "all_provinces = []\n", "\n", "for region in ['../maps/provinces_R01.geojson','../maps/provinces_R02.geojson','../maps/provinces_R07.geojson']:\n", " R = json.load(open(region,'r'))\n", "\n", " for feature in R['features']:\n", " all_provinces.append(feature['properties']['prov_name'])" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'Torino': 0,\n", " 'Vercelli': 1,\n", " 'Novara': 2,\n", " 'Cuneo': 3,\n", " 'Asti': 4,\n", " 'Alessandria': 5,\n", " 'Biella': 6,\n", " 'Verbano-Cusio-Ossola': 7,\n", " \"Valle d'Aosta/Vallée d'Aoste\": 8,\n", " 'Imperia': 9,\n", " 'Savona': 10,\n", " 'Genova': 11,\n", " 'La Spezia': 12,\n", " 'Aosta': 8}" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "provinces_indexer = {item: index for index, item in enumerate(all_provinces)}\n", "provinces_indexer['Aosta'] = 8\n", "provinces_indexer" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "all_munis = []\n", "\n", "muni_files = ['../maps/municipalities_P001.geojson','../maps/municipalities_P002.geojson',\n", " '../maps/municipalities_P003.geojson','../maps/municipalities_P004.geojson',\n", " '../maps/municipalities_P005.geojson','../maps/municipalities_P006.geojson',\n", " '../maps/municipalities_P007.geojson','../maps/municipalities_P008.geojson',\n", " '../maps/municipalities_P009.geojson','../maps/municipalities_P010.geojson',\n", " '../maps/municipalities_P011.geojson','../maps/municipalities_P096.geojson',\n", " '../maps/municipalities_P103.geojson']\n", "\n", "for province in muni_files:\n", " M = json.load(open(province,'r'))\n", "\n", " for feature in M['features']:\n", " all_munis.append(feature['properties']['name'])" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [], "source": [ "munis_indexer = {item: index for index, item in enumerate(all_munis)}" ] }, { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "## Now we get the the presence vectors." ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [], "source": [ "import toml\n", "secrets = toml.load('../.streamlit/secrets.toml')" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [], "source": [ "client = pymongo.MongoClient(secrets['mongodb_string'])\n", "collection = client['poverta_educativa']['database_mappatura_1']" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [], "source": [ "presence_regioni_strings = {}\n", "\n", "cursor = collection.find()\n", "for entry in cursor:\n", " presence_regioni_strings[str(entry['_id'])] = entry['Regioni di intervento']\n", "\n", "presence_regioni_processed = {key: [item.strip() for item in value.split(';')] if isinstance(value,str) else [] for key, value in presence_regioni_strings.items()}\n", "\n", "presence_regioni_vector = {key: [0]*len(all_regions) for key in presence_regioni_processed.keys()}\n", "\n", "for key, value in presence_regioni_processed.items():\n", " for item in value:\n", " if item in regions_indexer:\n", " presence_regioni_vector[key][regions_indexer[item]] += 1" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [], "source": [ "import pickle" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [], "source": [ "pickle.dump(presence_regioni_vector, open('../data/presence_regioni_vector.pkl', 'wb'))" ] }, { "cell_type": "code", "execution_count": 29, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "['Piemonte', \"Valle d'Aosta/Vallée d'Aoste\", 'Liguria']" ] }, "execution_count": 29, "metadata": {}, "output_type": "execute_result" } ], "source": [ "all_regions" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [], "source": [ "pickle.dump(all_regions, open('../data/all_regions.pkl', 'wb'))\n", "pickle.dump(all_provinces, open('../data/all_provinces.pkl', 'wb'))\n", "pickle.dump(all_munis, open('../data/all_munis.pkl', 'wb'))" ] }, { "cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'Abruzzo',\n", " 'Basilicata',\n", " 'Calabria',\n", " 'Campania',\n", " 'Emilia-Romagna',\n", " 'Friuli-Venezia Giulia',\n", " 'Lazio',\n", " 'Liguria',\n", " 'Lombardia',\n", " 'Marche',\n", " 'Molise',\n", " 'Piemonte',\n", " 'Puglia',\n", " 'Sardegna',\n", " 'Sicilia',\n", " 'Toscana',\n", " 'Trentino-Alto Adige',\n", " 'Umbria',\n", " \"Valle d'Aosta\",\n", " 'Veneto'}" ] }, "execution_count": 14, "metadata": {}, "output_type": "execute_result" } ], "source": [ "all_regions_in = set()\n", "\n", "for values in presence_regioni_processed.values():\n", " for value in values:\n", " all_regions_in.add(value)\n", "\n", "all_regions_in" ] }, { "cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [], "source": [ "presence_province_strings = {}\n", "\n", "cursor = collection.find()\n", "for entry in cursor:\n", " presence_province_strings[str(entry['_id'])] = entry['Province di intervento']\n", "\n", "presence_province_processed = {key: [item.strip() for item in value.split(';')] if isinstance(value,str) else [] for key, value in presence_province_strings.items()}\n", "\n", "presence_province_vector = {key: [0]*len(all_provinces) for key in presence_province_processed.keys()}\n", "\n", "for key, value in presence_province_processed.items():\n", " for item in value:\n", " if item in provinces_indexer:\n", " presence_province_vector[key][provinces_indexer[item]] += 1" ] }, { "cell_type": "code", "execution_count": 28, "metadata": {}, "outputs": [], "source": [ "pickle.dump(presence_province_vector, open('../data/presence_province_vector.pkl', 'wb'))" ] }, { "cell_type": "code", "execution_count": 17, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'Agrigento',\n", " 'Alessandria',\n", " 'Ancona',\n", " 'Andria',\n", " 'Aosta',\n", " 'Arezzo',\n", " 'Ascoli Piceno',\n", " 'Asti',\n", " 'Avellino',\n", " 'Bari',\n", " 'Benevento',\n", " 'Bergamo',\n", " 'Biella',\n", " 'Bologna',\n", " 'Bolzano',\n", " 'Brescia',\n", " 'Brindisi',\n", " 'Cagliari',\n", " 'Caltanissetta',\n", " 'Campobasso',\n", " 'Caserta',\n", " 'Catania',\n", " 'Catanzaro',\n", " 'Chieti',\n", " 'Como',\n", " 'Cosenza',\n", " 'Cremona',\n", " 'Crotone',\n", " 'Cuneo',\n", " 'Enna',\n", " 'Ferrara',\n", " 'Firenze',\n", " 'Foggia',\n", " 'Frosinone',\n", " 'Genova',\n", " 'Genova, La Spezia, Imperia, Savona',\n", " 'Grosseto',\n", " 'Imperia',\n", " \"L'Aquila\",\n", " 'La Spezia',\n", " 'Latina',\n", " 'Lecce',\n", " 'Lecco',\n", " 'Livorno',\n", " 'Lodi',\n", " 'Lucca',\n", " 'Macerata',\n", " 'Mantova',\n", " 'Massa',\n", " 'Matera',\n", " 'Messina',\n", " 'Milano',\n", " 'Modena',\n", " 'Monza',\n", " 'Napoli',\n", " 'Novara',\n", " 'Nuoro',\n", " 'Padova',\n", " 'Palermo',\n", " 'Pavia',\n", " 'Perugia',\n", " 'Pesaro',\n", " 'Pescara',\n", " 'Pisa',\n", " 'Pistoia',\n", " 'Potenza',\n", " 'Prato',\n", " 'Ragusa',\n", " 'Reggio di Calabria',\n", " 'Rieti',\n", " 'Rimini',\n", " 'Roma',\n", " 'Salerno',\n", " 'Sassari',\n", " 'Savona',\n", " 'Siena',\n", " 'Siracusa',\n", " 'Sondrio',\n", " 'Sud Sardegna',\n", " 'Taranto',\n", " 'Teramo',\n", " 'Terni',\n", " 'Torino',\n", " 'Trapani',\n", " 'Trento',\n", " 'Treviso',\n", " 'Trieste',\n", " 'Udine',\n", " 'Varese',\n", " 'Venezia',\n", " 'Verbano-Cusio-Ossola',\n", " 'Vercelli',\n", " 'Verona',\n", " 'Viterbo',\n", " '_'}" ] }, "execution_count": 17, "metadata": {}, "output_type": "execute_result" } ], "source": [ "all_provinces_in = set()\n", "\n", "for values in presence_province_processed.values():\n", " for value in values:\n", " all_provinces_in.add(value)\n", "\n", "all_provinces_in" ] }, { "cell_type": "code", "execution_count": 18, "metadata": {}, "outputs": [], "source": [ "presence_muni_strings = {}\n", "\n", "cursor = collection.find()\n", "for entry in cursor:\n", " presence_muni_strings[str(entry['_id'])] = entry['Comuni di intervento']\n", "\n", "presence_muni_processed = {key: [item.strip() for item in value.split(';')] if isinstance(value,str) else [] for key, value in presence_muni_strings.items()}\n", "\n", "presence_muni_vector = {key: [0]*len(all_munis) for key in presence_muni_processed.keys()}\n", "\n", "for key, value in presence_muni_processed.items():\n", " for item in value:\n", " if item in munis_indexer:\n", " presence_muni_vector[key][munis_indexer[item]] += 1" ] }, { "cell_type": "code", "execution_count": 19, "metadata": {}, "outputs": [], "source": [ "pickle.dump(presence_muni_vector, open('../data/presence_comuni_vector.pkl', 'wb'))" ] }, { "cell_type": "code", "execution_count": 20, "metadata": {}, "outputs": [], "source": [ "import numpy as np" ] }, { "cell_type": "code", "execution_count": 21, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "array([221., 5., 135.])" ] }, "execution_count": 21, "metadata": {}, "output_type": "execute_result" } ], "source": [ "tot_regions = np.zeros(len(all_regions))\n", "\n", "for key, value in presence_regioni_vector.items():\n", " tot_regions += np.array(value)\n", "\n", "tot_regions" ] }, { "cell_type": "code", "execution_count": 22, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "array([126., 5., 14., 55., 3., 12., 10., 7., 5., 12., 40.,\n", " 80., 18.])" ] }, "execution_count": 22, "metadata": {}, "output_type": "execute_result" } ], "source": [ "tot_provinces = np.zeros(len(all_provinces))\n", "\n", "for key, value in presence_province_vector.items():\n", " tot_provinces += np.array(value)\n", "\n", "tot_provinces" ] }, { "cell_type": "code", "execution_count": 23, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "array([0., 0., 0., ..., 0., 0., 0.])" ] }, "execution_count": 23, "metadata": {}, "output_type": "execute_result" } ], "source": [ "tot_munis = np.zeros(len(all_munis))\n", "\n", "for key, value in presence_muni_vector.items():\n", " tot_munis += np.array(value)\n", "\n", "tot_munis" ] }, { "cell_type": "code", "execution_count": 24, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "630.0" ] }, "execution_count": 24, "metadata": {}, "output_type": "execute_result" } ], "source": [ "np.sum(tot_munis)" ] }, { "cell_type": "code", "execution_count": 25, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "1487" ] }, "execution_count": 25, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(tot_munis)" ] }, { "cell_type": "code", "execution_count": 26, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "244" ] }, "execution_count": 26, "metadata": {}, "output_type": "execute_result" } ], "source": [ "np.sum(tot_munis > 0)" ] }, { "cell_type": "code", "execution_count": 27, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "array([ 1., 4., 1., 1., 1., 1., 1., 2., 4., 2., 2., 1., 1.,\n", " 2., 4., 1., 1., 5., 2., 1., 2., 5., 1., 2., 4., 2.,\n", " 3., 1., 1., 1., 1., 3., 1., 1., 1., 1., 2., 1., 2.,\n", " 1., 1., 84., 1., 1., 1., 1., 1., 1., 1., 3., 1., 1.,\n", " 2., 1., 1., 1., 1., 1., 1., 1., 2., 2., 7., 1., 1.,\n", " 1., 2., 6., 1., 1., 1., 1., 1., 2., 1., 1., 2., 1.,\n", " 1., 1., 6., 4., 1., 1., 2., 1., 1., 1., 2., 1., 1.,\n", " 3., 3., 1., 1., 11., 1., 5., 1., 1., 1., 1., 1., 1.,\n", " 1., 2., 1., 1., 1., 1., 1., 3., 1., 1., 1., 7., 1.,\n", " 1., 2., 1., 1., 1., 2., 1., 2., 1., 1., 1., 1., 1.,\n", " 1., 1., 4., 1., 1., 4., 1., 1., 1., 2., 1., 2., 2.,\n", " 1., 1., 1., 1., 9., 1., 1., 1., 1., 1., 2., 1., 1.,\n", " 5., 1., 1., 3., 2., 3., 8., 7., 1., 2., 5., 3., 1.,\n", " 1., 1., 2., 1., 4., 2., 5., 2., 4., 1., 2., 2., 8.,\n", " 2., 2., 2., 2., 4., 2., 2., 1., 2., 2., 2., 2., 1.,\n", " 1., 3., 2., 2., 18., 3., 4., 1., 1., 1., 2., 7., 1.,\n", " 1., 1., 1., 1., 3., 1., 1., 69., 3., 1., 1., 2., 1.,\n", " 1., 1., 1., 10., 1., 2., 1., 1., 1., 1., 4., 2., 4.,\n", " 2., 1., 1., 1., 2., 1., 2., 1., 2., 2.])" ] }, "execution_count": 27, "metadata": {}, "output_type": "execute_result" } ], "source": [ "tot_munis[tot_munis > 0]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "druidtools-8-hbcXZL-py3.10", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.12" }, "orig_nbformat": 4 }, "nbformat": 4, "nbformat_minor": 2 }