diff --git a/.gitignore b/.gitignore index 8598178..e3670b6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +*.pyc + # Ignore config file that might contain API keys and local paths config.py diff --git a/baseline_indicators.py b/baseline_indicators.py index ff43b26..38bbaab 100644 --- a/baseline_indicators.py +++ b/baseline_indicators.py @@ -6,6 +6,8 @@ @author: doorleyr """ import json +from osm_amenity_tools import * +from indicator_tools import * import pandas as pd import urllib import pyproj @@ -14,22 +16,11 @@ import censusdata import requests -def shannon_equitability_score(species_counts): - diversity=0 - pop_size=sum(species_counts) - if ((len(species_counts)>1) and (pop_size>0)): - for count in species_counts: - pj=count/pop_size - if not pj==0: - diversity+= -pj*math.log(pj) - equitability=diversity/math.log(len(species_counts)) - return equitability - else: - return None + table_name='corktown' -OSM_URL_ROOT='https://lz4.overpass-api.de/api/interpreter?data=[out:json][bbox];node;way;out;&bbox=' +# INPUTS OSM_CONFIG_FILE_PATH='./osm_amenities.json' GROUPS_FILE_PATH='./amenity_groups.csv' TABLE_CONFIG_FILE_PATH='./tables/corktown/table_configs.json' @@ -37,33 +28,10 @@ def shannon_equitability_score(species_counts): SIM_ZONES_PATH='./tables/corktown/sim_zones.json' WAC_PATH='./tables/corktown/mi_wac_S000_JT00_2017.csv.gz' -def get_osm_amenies(bounds_all, amenity_types): - """ - takes a list representing the bounds of the area of interest and - a dictionary defining tag categories and the Oassociated OSM tags - Returns a list of amenities with their tag categories - """ - str_bounds=str(bounds_all[0])+','+str(bounds_all[1])+','+str(bounds_all[2])+','+str(bounds_all[3]) - osm_url_bbox=OSM_URL_ROOT+str_bounds - with urllib.request.urlopen(osm_url_bbox) as url: - data=json.loads(url.read().decode()) - amenities={at:0 for at in amenity_types} - for ind_record in range(len(data['elements'])): - for at in amenity_types: - # for each amenity type we're interested in: eg. restaurant, school - if 'tags' in data['elements'][ind_record]: - for record_tag in list(data['elements'][ind_record]['tags'].items()): - # check each tag in this osm record - record_tag_key, record_tag_value= record_tag[0], record_tag[1] - for osm_tag in amenity_types[at]: - # against each osm tag associated with this amenity type - osm_tag_key, osm_tag_value=osm_tag.split('=') - if (((osm_tag_value=='*') or (osm_tag_value==record_tag_value)) - and (osm_tag_key==record_tag_key)): - # lon, lat=data['elements'][ind_record]['lon'], data['elements'][ind_record]['lat'] - # x,y=pyproj.transform(wgs, projection,lon, lat) - amenities[at]+=1 - return amenities +# OUTPUTS +AMENITY_SCORES_PATH='tables/{}/amenity_scores.csv'.format(table_name) +BASIC_STATS_PATH='tables/{}/basic_stats.json'.format(table_name) +BASELINE_INDICATORS_PATH='tables/{}/baseline_indicators.json'.format(table_name) # ============================================================================= # Set-up area and configs @@ -75,9 +43,7 @@ def get_osm_amenies(bounds_all, amenity_types): amenity_groups=pd.read_csv(GROUPS_FILE_PATH) amenity_scores=amenity_groups.copy() -RESIDENTS=10000 -WORKERS=30000 -DAYTIMEPOP=35000 + MAX_RESI_PER_KM=30000 # manhattan is 27k, Dhaka most dense at 45k MAX_EMP_PER_KM=50000 @@ -130,25 +96,32 @@ def get_osm_amenies(bounds_all, amenity_types): acs_pop_data=acs_pop_data.rename(columns=census_data_columns) pop_data_full_table=acs_pop_data.sum(axis=0) -population_all=pop_data_full_table['population'] +population_all=int(pop_data_full_table['population']) wac_data['block_group']=wac_data.apply(lambda row: str(row['w_geocode'])[:12], axis=1) wac_data=wac_data.loc[wac_data['block_group'].isin(table_geoids)] wac_data_full_table=wac_data.sum(axis=0) -n_jobs_all=wac_data_full_table['C000'] +n_jobs_all=int(wac_data_full_table['C000']) total_people=n_jobs_all+population_all + +max_residents= MAX_RESI_PER_KM * AREA +max_employees= MAX_EMP_PER_KM * AREA + +#save the basic stats for use in the resposive module +json.dump({'residents': population_all, 'employees': n_jobs_all, + 'max_residents':max_residents, 'max_employees': max_employees, + 'area': AREA}, open(BASIC_STATS_PATH, 'w')) # ============================================================================= # Density # ============================================================================= # density of residential: census # density of employment: Workplace Area Characteristics file -max_residents= MAX_RESI_PER_KM * AREA -max_employees= MAX_EMP_PER_KM * AREA + # 3rd day, 3rd night: cat scores # education, cultural: sub-cat scores -amenity_scores['quota']=DAYTIMEPOP*amenity_scores['quota_per_k_people']/1000 +amenity_scores['quota']=total_people*amenity_scores['quota_per_k_people']/1000 amenity_scores['num_present']=amenity_scores.apply(lambda row: amenities[row['sub_sub_cat']], axis=1) @@ -166,6 +139,7 @@ def get_osm_amenies(bounds_all, amenity_types): residential_density_score=population_all/max_residents employment_density_score=n_jobs_all/max_residents +resi_employ_ratio=min(population_all, n_jobs_all)/max(population_all, n_jobs_all) # ============================================================================= # Diversity @@ -173,8 +147,6 @@ def get_osm_amenies(bounds_all, amenity_types): # working pop income, employment sector from WAC # live-work ratio # cultural, education: diversity of sub-cats -cultural_pop=list(amenity_scores.loc[amenity_scores['sub_cat']=='Culture', 'num_present'].values) - sub_cat_diversity=amenity_scores.groupby('sub_cat').agg( {'num_present': shannon_equitability_score}).rename(columns={'num_present': 'diversity_score'}) # 3rd places: diversity of categories @@ -197,6 +169,7 @@ def get_osm_amenies(bounds_all, amenity_types): {'name': 'Educational Inst Density','category': 'Density', 'value': cat_scores.loc['Educational', 'score']}, {'name': 'Cultural Inst Density','category': 'Density', 'value': sub_cat_scores.loc['Culture', 'score']}, {'name': 'Cultural Inst Diversity','category': 'Diversity', 'value': sub_cat_diversity.loc['Culture', 'diversity_score']}, + {'name': 'Residential/Employment Ratio','category': 'Diversity', 'value': resi_employ_ratio}, {'name': 'Educational Inst Diversity','category': 'Diversity', 'value': sub_cat_diversity.loc['Educational', 'diversity_score']}, {'name': '3rd Places Diversity','category': 'Diversity', 'value': cat_diversity.loc['3rd places Day', 'diversity_score']}, {'name': 'Job Type Diversity','category': 'Diversity', 'value': job_type_diversity}, @@ -215,10 +188,16 @@ def get_osm_amenies(bounds_all, amenity_types): 'category': 'Energy', 'value': random.random()}) indicators.append({'name': '{} Embodied Energy'.format(energy_ind), 'category': 'Energy', 'value': random.random()}) +# ============================================================================= +# Save results +# ============================================================================= +amenity_scores.to_csv('tables/corktown/amenity_scores.csv', index=False) host='https://cityio.media.mit.edu/' cityIO_output_path=host+'api/table/update/'+table_name r = requests.post(cityIO_output_path+'/indicators', data = json.dumps(indicators)) -print(r) \ No newline at end of file +print(r) + +json.dump(indicators, open(BASELINE_INDICATORS_PATH, 'w')) \ No newline at end of file diff --git a/indicator_tools.py b/indicator_tools.py new file mode 100644 index 0000000..abe7d10 --- /dev/null +++ b/indicator_tools.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +Created on Mon Mar 2 11:23:12 2020 + +@author: doorleyr +""" +import math + +def shannon_equitability_score(species_counts): + diversity=0 + pop_size=sum(species_counts) + if ((len(species_counts)>1) and (pop_size>0)): + for count in species_counts: + pj=count/pop_size + if not pj==0: + diversity+= -pj*math.log(pj) + equitability=diversity/math.log(len(species_counts)) + return equitability + else: + return None \ No newline at end of file diff --git a/indicators_module.py b/indicators_module.py new file mode 100644 index 0000000..0eac10d --- /dev/null +++ b/indicators_module.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +Created on Mon Mar 2 12:20:41 2020 + +@author: doorleyr +""" +import json +from osm_amenity_tools import * +from indicator_tools import * +import pandas as pd +import urllib +from time import sleep +import requests + +# Which table? +import sys + +if len(sys.argv)>1: + table_name=sys.argv[1] +else: + table_name='corktown' + + +# ============================================================================= +# Define Functions +# ============================================================================= +def initialise(): + """ + Steps that only need to be performed once when the model starts running + """ + print('Initialising') + global geogrid, amenity_scores, basic_stats, baseline_indicators + with urllib.request.urlopen(cityIO_get_url+'/GEOGRID') as url: + #get the GEOGRID from cityI/O + geogrid=json.loads(url.read().decode()) + amenity_scores=pd.read_csv(AMENITY_SCORES_PATH) + baseline_indicators=json.load(open(BASELINE_INDICATORS_PATH)) + basic_stats=json.load(open(BASIC_STATS_PATH)) + + + +def perform_updates(output_name, geogrid_data): + """ + Steps that take place every time a change is detected in the + city_io grid data + """ + print('Performing updates') + + residents=basic_stats['residents'] + employees=basic_stats['employees'] + indicators=baseline_indicators.copy() + + for cell in geogrid_data: + if 'Residential' in cell['name']: + residents+=PEOPLE_PER_RESI_BLD + elif 'Office' in cell['name']: + employees+=PEOPLE_PER_OFFICE_BLD + + residential_density_score=residents/basic_stats['max_residents'] + employment_density_score=employees/basic_stats['max_employees'] + + for ind in indicators: + if ind['name']=='Residential Density': + ind['value']=residential_density_score + elif ind['name']=='Employment Density': + ind['value']=employment_density_score + elif ind['name']=='Residential/Employment Ratio': + ind['value']=min(residents, employees)/max(residents, employees) + r = requests.post(cityIO_post_url+'/'+output_name, data = json.dumps(indicators)) + print(r) + + +# INPUTS +AMENITY_SCORES_PATH='tables/{}/amenity_scores.csv'.format(table_name) +BASIC_STATS_PATH='tables/{}/basic_stats.json'.format(table_name) +BASELINE_INDICATORS_PATH='tables/{}/baseline_indicators.json'.format(table_name) + +# CITY I/O +host='https://cityio.media.mit.edu/' +table_name='corktown' +cityIO_get_url=host+'api/table/'+table_name +cityIO_post_url=host+'api/table/update/'+table_name + + +SLEEP_TIME=0.1 # seconds to sleep between checkinh cityI/O + +PEOPLE_PER_RESI_BLD=200 +PEOPLE_PER_OFFICE_BLD=200 + + +initialise() + +# ============================================================================= +# Update Loop +# ============================================================================= +lastId=0 +while True: +#check if grid data changed + try: + with urllib.request.urlopen(cityIO_get_url+'/meta/hashes/GEOGRIDDATA') as url: + hash_id=json.loads(url.read().decode()) + except: + print('Cant access cityIO GEOGRIDDATA hash') + hash_id=1 + if hash_id==lastId: + sleep(SLEEP_TIME) + else: + try: + with urllib.request.urlopen(cityIO_get_url+'/GEOGRIDDATA') as url: + geogrid_data=json.loads(url.read().decode()) + perform_updates('indicators', geogrid_data) + lastId=hash_id + except: + print('Got city_IO GEOGRIDDATA hash but couldnt get data') + sleep(SLEEP_TIME) + diff --git a/osm_amenity_tools.py b/osm_amenity_tools.py new file mode 100644 index 0000000..2300cf1 --- /dev/null +++ b/osm_amenity_tools.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +Created on Mon Mar 2 11:16:01 2020 + +@author: doorleyr +""" +import urllib +import json + +def get_osm_amenies(bounds_all, amenity_types): + """ + takes a list representing the bounds of the area of interest and + a dictionary defining tag categories and the associated OSM tags + Returns a list of amenities with their tag categories + """ + OSM_URL_ROOT='https://lz4.overpass-api.de/api/interpreter?data=[out:json][bbox];node;way;out;&bbox=' + + str_bounds=str(bounds_all[0])+','+str(bounds_all[1])+','+str(bounds_all[2])+','+str(bounds_all[3]) + osm_url_bbox=OSM_URL_ROOT+str_bounds + with urllib.request.urlopen(osm_url_bbox) as url: + data=json.loads(url.read().decode()) + amenities={at:0 for at in amenity_types} + for ind_record in range(len(data['elements'])): + for at in amenity_types: + # for each amenity type we're interested in: eg. restaurant, school + if 'tags' in data['elements'][ind_record]: + for record_tag in list(data['elements'][ind_record]['tags'].items()): + # check each tag in this osm record + record_tag_key, record_tag_value= record_tag[0], record_tag[1] + for osm_tag in amenity_types[at]: + # against each osm tag associated with this amenity type + osm_tag_key, osm_tag_value=osm_tag.split('=') + if (((osm_tag_value=='*') or (osm_tag_value==record_tag_value)) + and (osm_tag_key==record_tag_key)): + # lon, lat=data['elements'][ind_record]['lon'], data['elements'][ind_record]['lat'] + # x,y=pyproj.transform(wgs, projection,lon, lat) + amenities[at]+=1 + return amenities \ No newline at end of file diff --git a/tables/corktown/amenity_scores.csv b/tables/corktown/amenity_scores.csv new file mode 100644 index 0000000..d0b5e18 --- /dev/null +++ b/tables/corktown/amenity_scores.csv @@ -0,0 +1,25 @@ +category,sub_cat,sub_sub_cat,quota_per_k_people,quota,num_present,score +3rd places Day,Hotels,hotels,0.14,1.41036,3,1.0 +3rd places Day,Restaurants,restaurants,2.0,20.148,29,1.0 +3rd places Day,Leisure and Wellness,pharmacies,0.31,3.1229400000000003,0,0.0 +3rd places Day,Leisure and Wellness,dentist,0.27,2.71998,0,0.0 +3rd places Day,Leisure and Wellness,ophthalmologists,0.1,1.0074,0,0.0 +3rd places Day,Leisure and Wellness,gyms,0.07,0.70518,0,0.0 +3rd places Day,Leisure and Wellness,spas,0.02,0.20148000000000002,0,0.0 +3rd places Day,Leisure and Wellness,clinic,0.045,0.45333,8,1.0 +3rd places Day,Leisure and Wellness,daycare,1.38,13.902119999999998,0,0.0 +3rd places Day,Leisure and Wellness,hospital,0.016,0.161184,0,0.0 +3rd places Day,Leisure and Wellness,nursery,6.38,64.27212,0,0.0 +3rd places Day,Culture,libraries,0.04,0.40296000000000004,1,1.0 +3rd places Day,Culture,museums,0.05,0.5037,0,0.0 +3rd places Day,Culture,cinema,0.126,1.2693240000000001,0,0.0 +3rd places Day,Culture,bookstore,0.102,1.027548,1,0.9731905468163046 +3rd places Day,Culture,art-gallery,0.0181,0.1823394,0,0.0 +3rd places Day,Gym and Sports,sports_facilities,0.6,6.0443999999999996,11,1.0 +3rd places Day,Gym and Sports,gyms,0.07,0.70518,0,0.0 +3rd places Day,Gym and Sports,sports_shop,0.15,1.5110999999999999,0,0.0 +3rd places Night,nightlife,nightlife,0.89,8.965860000000001,7,0.7807393825020689 +Educational,Educational,schools,0.03,0.30222,13,1.0 +Educational,Educational,universities,0.02,0.20148000000000002,1,1.0 +Educational,Educational,kindergarten,0.027999999999999997,0.28207199999999993,0,0.0 +Educational,Educational,further_education,0.05,0.5037,1,1.0 diff --git a/tables/corktown/baseline_indicators.json b/tables/corktown/baseline_indicators.json new file mode 100644 index 0000000..0dbfbfd --- /dev/null +++ b/tables/corktown/baseline_indicators.json @@ -0,0 +1 @@ +[{"name": "Residential Density", "category": "Density", "value": 0.0323}, {"name": "Employment Density", "category": "Density", "value": 0.05165}, {"name": "3rd Places Day Density", "category": "Density", "value": 0.567816510761541}, {"name": "3rd Places Night Density", "category": "Density", "value": 0.7807393825020689}, {"name": "Educational Inst Density", "category": "Density", "value": 0.75}, {"name": "Cultural Inst Density", "category": "Density", "value": 0.3946381093632609}, {"name": "Cultural Inst Diversity", "category": "Diversity", "value": 0.43067655807339306}, {"name": "Educational Inst Diversity", "category": "Diversity", "value": 0.3499214199431193}, {"name": "3rd Places Diversity", "category": "Diversity", "value": 0.7629451472200595}, {"name": "Job Type Diversity", "category": "Diversity", "value": 0.6111269487221378}, {"name": "Income Level Diversity", "category": "Diversity", "value": 0.9126869136380995}, {"name": "Proximity to Employment", "category": "Proximity", "value": 0.22331241589367745}, {"name": "Proximity to Education", "category": "Proximity", "value": 0.11788791567898416}, {"name": "Proximity to Housing", "category": "Proximity", "value": 0.5378882103487862}, {"name": "Proximity to 3rd Places", "category": "Proximity", "value": 0.9498267714988761}, {"name": "Proximity to Parks", "category": "Proximity", "value": 0.5857501458943148}, {"name": "Proximity to Healthcare", "category": "Proximity", "value": 0.6546832261298203}, {"name": "Buildings Energy Efficiency", "category": "Energy", "value": 0.4677640695283458}, {"name": "Buildings Embodied Energy", "category": "Energy", "value": 0.37409620919900455}, {"name": "Mobility Energy Efficiency", "category": "Energy", "value": 0.22333458263430495}, {"name": "Mobility Embodied Energy", "category": "Energy", "value": 0.3125549498232869}] \ No newline at end of file diff --git a/tables/corktown/basic_stats.json b/tables/corktown/basic_stats.json new file mode 100644 index 0000000..87fa386 --- /dev/null +++ b/tables/corktown/basic_stats.json @@ -0,0 +1 @@ +{"residents": 3876, "employees": 6198, "max_residents": 120000, "max_employees": 200000, "area": 4} \ No newline at end of file