gitignore to config.py and directory with local shapefiles

.gitignore

@@ -1,3 +1,5 @@
+*.pyc
+
 # Ignore config file that might contain API keys and local paths
 config.py
 

baseline_indicators.py

@@ -6,6 +6,8 @@
 @author: doorleyr
 """
 import json
+from osm_amenity_tools import *
+from indicator_tools import *
 import pandas as pd
 import urllib
 import pyproj
@@ -14,56 +16,22 @@
 import censusdata
 import requests
 
-def shannon_equitability_score(species_counts):
-    diversity=0
-    pop_size=sum(species_counts)
-    if ((len(species_counts)>1) and (pop_size>0)):        
-        for count in species_counts:
-            pj=count/pop_size
-            if not pj==0:
-                diversity+= -pj*math.log(pj)
-        equitability=diversity/math.log(len(species_counts))
-        return equitability
-    else:
-        return None
+
 
 table_name='corktown'
 
-OSM_URL_ROOT='https://lz4.overpass-api.de/api/interpreter?data=[out:json][bbox];node;way;out;&bbox='
+# INPUTS
 OSM_CONFIG_FILE_PATH='./osm_amenities.json'
 GROUPS_FILE_PATH='./amenity_groups.csv'
 TABLE_CONFIG_FILE_PATH='./tables/corktown/table_configs.json'
 SIM_AREA_GEOM_PATH='./tables/corktown/table_area.geojson'
 SIM_ZONES_PATH='./tables/corktown/sim_zones.json'
 WAC_PATH='./tables/corktown/mi_wac_S000_JT00_2017.csv.gz'
 
-def get_osm_amenies(bounds_all, amenity_types):
-    """
-    takes a list representing the bounds of the area of interest and
-    a dictionary defining tag categories and the Oassociated OSM tags 
-    Returns a list of amenities with their tag categories
-    """
-    str_bounds=str(bounds_all[0])+','+str(bounds_all[1])+','+str(bounds_all[2])+','+str(bounds_all[3])
-    osm_url_bbox=OSM_URL_ROOT+str_bounds
-    with urllib.request.urlopen(osm_url_bbox) as url:
-        data=json.loads(url.read().decode())
-    amenities={at:0 for at in amenity_types}
-    for ind_record in range(len(data['elements'])):
-        for at in amenity_types:
-            # for each amenity type we're interested in: eg. restaurant, school
-            if 'tags' in data['elements'][ind_record]:
-                for record_tag in list(data['elements'][ind_record]['tags'].items()):
-                    # check each tag in this osm record
-                    record_tag_key, record_tag_value= record_tag[0], record_tag[1]
-                    for osm_tag in amenity_types[at]:
-                        # against each osm tag associated with this amenity type
-                        osm_tag_key, osm_tag_value=osm_tag.split('=')
-                        if (((osm_tag_value=='*') or (osm_tag_value==record_tag_value)) 
-                                and (osm_tag_key==record_tag_key)):
-    #                        lon, lat=data['elements'][ind_record]['lon'], data['elements'][ind_record]['lat']
-    #                        x,y=pyproj.transform(wgs, projection,lon, lat)
-                            amenities[at]+=1
-    return amenities        
+# OUTPUTS
+AMENITY_SCORES_PATH='tables/{}/amenity_scores.csv'.format(table_name)
+BASIC_STATS_PATH='tables/{}/basic_stats.json'.format(table_name)
+BASELINE_INDICATORS_PATH='tables/{}/baseline_indicators.json'.format(table_name)
 
 # =============================================================================
 # Set-up area and configs
@@ -75,9 +43,7 @@ def get_osm_amenies(bounds_all, amenity_types):
 amenity_groups=pd.read_csv(GROUPS_FILE_PATH)
 amenity_scores=amenity_groups.copy()
 
-RESIDENTS=10000
-WORKERS=30000
-DAYTIMEPOP=35000
+
 MAX_RESI_PER_KM=30000
 # manhattan is 27k, Dhaka most dense at 45k
 MAX_EMP_PER_KM=50000
@@ -130,25 +96,32 @@ def get_osm_amenies(bounds_all, amenity_types):
 
 acs_pop_data=acs_pop_data.rename(columns=census_data_columns)
 pop_data_full_table=acs_pop_data.sum(axis=0)
-population_all=pop_data_full_table['population']
+population_all=int(pop_data_full_table['population'])
 
 wac_data['block_group']=wac_data.apply(lambda row: str(row['w_geocode'])[:12], axis=1)
 wac_data=wac_data.loc[wac_data['block_group'].isin(table_geoids)]
 
 wac_data_full_table=wac_data.sum(axis=0)
-n_jobs_all=wac_data_full_table['C000']
+n_jobs_all=int(wac_data_full_table['C000'])
 
 total_people=n_jobs_all+population_all
+
+max_residents= MAX_RESI_PER_KM * AREA
+max_employees= MAX_EMP_PER_KM * AREA
+
+#save the basic stats for use in the resposive module
+json.dump({'residents': population_all, 'employees': n_jobs_all,
+           'max_residents':max_residents, 'max_employees': max_employees,
+            'area': AREA}, open(BASIC_STATS_PATH, 'w'))
 # =============================================================================
 # Density 
 # =============================================================================
 # density of residential: census
 # density of employment: Workplace Area Characteristics file
-max_residents= MAX_RESI_PER_KM * AREA
-max_employees= MAX_EMP_PER_KM * AREA
+
 # 3rd day, 3rd night: cat scores
 # education, cultural: sub-cat scores
-amenity_scores['quota']=DAYTIMEPOP*amenity_scores['quota_per_k_people']/1000
+amenity_scores['quota']=total_people*amenity_scores['quota_per_k_people']/1000
 
 amenity_scores['num_present']=amenity_scores.apply(lambda row: 
     amenities[row['sub_sub_cat']], axis=1)
@@ -166,15 +139,14 @@ def get_osm_amenies(bounds_all, amenity_types):
 
 residential_density_score=population_all/max_residents
 employment_density_score=n_jobs_all/max_residents
+resi_employ_ratio=min(population_all, n_jobs_all)/max(population_all, n_jobs_all)
 
 # =============================================================================
 # Diversity
 # =============================================================================
 # working pop income, employment sector from WAC
 # live-work ratio
 # cultural, education: diversity of sub-cats
-cultural_pop=list(amenity_scores.loc[amenity_scores['sub_cat']=='Culture', 'num_present'].values)
-
 sub_cat_diversity=amenity_scores.groupby('sub_cat').agg(
         {'num_present': shannon_equitability_score}).rename(columns={'num_present': 'diversity_score'})
 # 3rd places: diversity of categories    
@@ -197,6 +169,7 @@ def get_osm_amenies(bounds_all, amenity_types):
             {'name': 'Educational Inst Density','category': 'Density', 'value': cat_scores.loc['Educational', 'score']},
             {'name': 'Cultural Inst Density','category': 'Density', 'value': sub_cat_scores.loc['Culture', 'score']},
             {'name': 'Cultural Inst Diversity','category': 'Diversity', 'value': sub_cat_diversity.loc['Culture', 'diversity_score']},
+            {'name': 'Residential/Employment Ratio','category': 'Diversity', 'value': resi_employ_ratio},
             {'name': 'Educational Inst Diversity','category': 'Diversity', 'value': sub_cat_diversity.loc['Educational', 'diversity_score']},
             {'name': '3rd Places Diversity','category': 'Diversity', 'value': cat_diversity.loc['3rd places Day', 'diversity_score']},
             {'name': 'Job Type Diversity','category': 'Diversity', 'value': job_type_diversity},
@@ -215,10 +188,16 @@ def get_osm_amenies(bounds_all, amenity_types):
                       'category': 'Energy', 'value': random.random()})
     indicators.append({'name': '{} Embodied Energy'.format(energy_ind), 
                       'category': 'Energy', 'value': random.random()})
+# =============================================================================
+# Save results
+# =============================================================================
+amenity_scores.to_csv('tables/corktown/amenity_scores.csv', index=False)
 
 
 host='https://cityio.media.mit.edu/'
 cityIO_output_path=host+'api/table/update/'+table_name
 
 r = requests.post(cityIO_output_path+'/indicators', data = json.dumps(indicators))
-print(r)
+print(r)
+
+json.dump(indicators, open(BASELINE_INDICATORS_PATH, 'w'))

indicator_tools.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Mon Mar  2 11:23:12 2020
+
+@author: doorleyr
+"""
+import math
+
+def shannon_equitability_score(species_counts):
+    diversity=0
+    pop_size=sum(species_counts)
+    if ((len(species_counts)>1) and (pop_size>0)):        
+        for count in species_counts:
+            pj=count/pop_size
+            if not pj==0:
+                diversity+= -pj*math.log(pj)
+        equitability=diversity/math.log(len(species_counts))
+        return equitability
+    else:
+        return None

indicators_module.py

@@ -0,0 +1,117 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Mon Mar  2 12:20:41 2020
+
+@author: doorleyr
+"""
+import json
+from osm_amenity_tools import *
+from indicator_tools import *
+import pandas as pd
+import urllib
+from time import sleep
+import requests
+
+# Which table?
+import sys
+
+if len(sys.argv)>1:
+    table_name=sys.argv[1]
+else:
+    table_name='corktown'
+
+
+# =============================================================================
+# Define Functions
+# =============================================================================
+def initialise():
+    """
+    Steps that only need to be performed once when the model starts running
+    """
+    print('Initialising')
+    global geogrid, amenity_scores, basic_stats, baseline_indicators
+    with urllib.request.urlopen(cityIO_get_url+'/GEOGRID') as url:
+    #get the GEOGRID from cityI/O
+        geogrid=json.loads(url.read().decode())
+    amenity_scores=pd.read_csv(AMENITY_SCORES_PATH)
+    baseline_indicators=json.load(open(BASELINE_INDICATORS_PATH))
+    basic_stats=json.load(open(BASIC_STATS_PATH))
+
+
+
+def perform_updates(output_name, geogrid_data):
+    """
+    Steps that take place every time a change is detected in the 
+    city_io grid data
+    """
+    print('Performing updates')
+
+    residents=basic_stats['residents']
+    employees=basic_stats['employees']
+    indicators=baseline_indicators.copy()
+
+    for cell in geogrid_data:
+        if 'Residential' in cell['name']:
+            residents+=PEOPLE_PER_RESI_BLD
+        elif 'Office' in cell['name']:
+            employees+=PEOPLE_PER_OFFICE_BLD
+
+    residential_density_score=residents/basic_stats['max_residents']
+    employment_density_score=employees/basic_stats['max_employees']
+
+    for ind in indicators:
+        if ind['name']=='Residential Density':
+            ind['value']=residential_density_score
+        elif ind['name']=='Employment Density':
+            ind['value']=employment_density_score  
+        elif ind['name']=='Residential/Employment Ratio':
+            ind['value']=min(residents, employees)/max(residents, employees) 
+    r = requests.post(cityIO_post_url+'/'+output_name, data = json.dumps(indicators))
+    print(r)
+
+
+# INPUTS
+AMENITY_SCORES_PATH='tables/{}/amenity_scores.csv'.format(table_name)
+BASIC_STATS_PATH='tables/{}/basic_stats.json'.format(table_name)
+BASELINE_INDICATORS_PATH='tables/{}/baseline_indicators.json'.format(table_name)
+
+# CITY I/O
+host='https://cityio.media.mit.edu/'
+table_name='corktown'
+cityIO_get_url=host+'api/table/'+table_name
+cityIO_post_url=host+'api/table/update/'+table_name
+
+
+SLEEP_TIME=0.1 # seconds to sleep between checkinh cityI/O
+
+PEOPLE_PER_RESI_BLD=200
+PEOPLE_PER_OFFICE_BLD=200
+
+
+initialise()
+
+# =============================================================================
+# Update Loop
+# =============================================================================
+lastId=0
+while True:
+#check if grid data changed
+    try:
+        with urllib.request.urlopen(cityIO_get_url+'/meta/hashes/GEOGRIDDATA') as url:
+            hash_id=json.loads(url.read().decode())
+    except:
+        print('Cant access cityIO GEOGRIDDATA hash')
+        hash_id=1
+    if hash_id==lastId:
+        sleep(SLEEP_TIME)
+    else:
+        try:
+            with urllib.request.urlopen(cityIO_get_url+'/GEOGRIDDATA') as url:
+                geogrid_data=json.loads(url.read().decode())
+            perform_updates('indicators', geogrid_data)
+            lastId=hash_id
+        except:
+            print('Got city_IO GEOGRIDDATA hash but couldnt get data')
+            sleep(SLEEP_TIME) 
+

osm_amenity_tools.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Mon Mar  2 11:16:01 2020
+
+@author: doorleyr
+"""
+import urllib
+import json
+
+def get_osm_amenies(bounds_all, amenity_types):
+    """
+    takes a list representing the bounds of the area of interest and
+    a dictionary defining tag categories and the associated OSM tags 
+    Returns a list of amenities with their tag categories
+    """
+    OSM_URL_ROOT='https://lz4.overpass-api.de/api/interpreter?data=[out:json][bbox];node;way;out;&bbox='
+
+    str_bounds=str(bounds_all[0])+','+str(bounds_all[1])+','+str(bounds_all[2])+','+str(bounds_all[3])
+    osm_url_bbox=OSM_URL_ROOT+str_bounds
+    with urllib.request.urlopen(osm_url_bbox) as url:
+        data=json.loads(url.read().decode())
+    amenities={at:0 for at in amenity_types}
+    for ind_record in range(len(data['elements'])):
+        for at in amenity_types:
+            # for each amenity type we're interested in: eg. restaurant, school
+            if 'tags' in data['elements'][ind_record]:
+                for record_tag in list(data['elements'][ind_record]['tags'].items()):
+                    # check each tag in this osm record
+                    record_tag_key, record_tag_value= record_tag[0], record_tag[1]
+                    for osm_tag in amenity_types[at]:
+                        # against each osm tag associated with this amenity type
+                        osm_tag_key, osm_tag_value=osm_tag.split('=')
+                        if (((osm_tag_value=='*') or (osm_tag_value==record_tag_value)) 
+                                and (osm_tag_key==record_tag_key)):
+    #                        lon, lat=data['elements'][ind_record]['lon'], data['elements'][ind_record]['lat']
+    #                        x,y=pyproj.transform(wgs, projection,lon, lat)
+                            amenities[at]+=1
+    return amenities 

tables/corktown/amenity_scores.csv

@@ -0,0 +1,25 @@
+category,sub_cat,sub_sub_cat,quota_per_k_people,quota,num_present,score
+3rd places Day,Hotels,hotels,0.14,1.41036,3,1.0
+3rd places Day,Restaurants,restaurants,2.0,20.148,29,1.0
+3rd places Day,Leisure and Wellness,pharmacies,0.31,3.1229400000000003,0,0.0
+3rd places Day,Leisure and Wellness,dentist,0.27,2.71998,0,0.0
+3rd places Day,Leisure and Wellness,ophthalmologists,0.1,1.0074,0,0.0
+3rd places Day,Leisure and Wellness,gyms,0.07,0.70518,0,0.0
+3rd places Day,Leisure and Wellness,spas,0.02,0.20148000000000002,0,0.0
+3rd places Day,Leisure and Wellness,clinic,0.045,0.45333,8,1.0
+3rd places Day,Leisure and Wellness,daycare,1.38,13.902119999999998,0,0.0
+3rd places Day,Leisure and Wellness,hospital,0.016,0.161184,0,0.0
+3rd places Day,Leisure and Wellness,nursery,6.38,64.27212,0,0.0
+3rd places Day,Culture,libraries,0.04,0.40296000000000004,1,1.0
+3rd places Day,Culture,museums,0.05,0.5037,0,0.0
+3rd places Day,Culture,cinema,0.126,1.2693240000000001,0,0.0
+3rd places Day,Culture,bookstore,0.102,1.027548,1,0.9731905468163046
+3rd places Day,Culture,art-gallery,0.0181,0.1823394,0,0.0
+3rd places Day,Gym and Sports,sports_facilities,0.6,6.0443999999999996,11,1.0
+3rd places Day,Gym and Sports,gyms,0.07,0.70518,0,0.0
+3rd places Day,Gym and Sports,sports_shop,0.15,1.5110999999999999,0,0.0
+3rd places Night,nightlife,nightlife,0.89,8.965860000000001,7,0.7807393825020689
+Educational,Educational,schools,0.03,0.30222,13,1.0
+Educational,Educational,universities,0.02,0.20148000000000002,1,1.0
+Educational,Educational,kindergarten,0.027999999999999997,0.28207199999999993,0,0.0
+Educational,Educational,further_education,0.05,0.5037,1,1.0

tables/corktown/baseline_indicators.json

@@ -0,0 +1 @@
+[{"name": "Residential Density", "category": "Density", "value": 0.0323}, {"name": "Employment Density", "category": "Density", "value": 0.05165}, {"name": "3rd Places Day Density", "category": "Density", "value": 0.567816510761541}, {"name": "3rd Places Night Density", "category": "Density", "value": 0.7807393825020689}, {"name": "Educational Inst Density", "category": "Density", "value": 0.75}, {"name": "Cultural Inst Density", "category": "Density", "value": 0.3946381093632609}, {"name": "Cultural Inst Diversity", "category": "Diversity", "value": 0.43067655807339306}, {"name": "Educational Inst Diversity", "category": "Diversity", "value": 0.3499214199431193}, {"name": "3rd Places Diversity", "category": "Diversity", "value": 0.7629451472200595}, {"name": "Job Type Diversity", "category": "Diversity", "value": 0.6111269487221378}, {"name": "Income Level Diversity", "category": "Diversity", "value": 0.9126869136380995}, {"name": "Proximity to Employment", "category": "Proximity", "value": 0.22331241589367745}, {"name": "Proximity to Education", "category": "Proximity", "value": 0.11788791567898416}, {"name": "Proximity to Housing", "category": "Proximity", "value": 0.5378882103487862}, {"name": "Proximity to 3rd Places", "category": "Proximity", "value": 0.9498267714988761}, {"name": "Proximity to Parks", "category": "Proximity", "value": 0.5857501458943148}, {"name": "Proximity to Healthcare", "category": "Proximity", "value": 0.6546832261298203}, {"name": "Buildings Energy Efficiency", "category": "Energy", "value": 0.4677640695283458}, {"name": "Buildings Embodied Energy", "category": "Energy", "value": 0.37409620919900455}, {"name": "Mobility Energy Efficiency", "category": "Energy", "value": 0.22333458263430495}, {"name": "Mobility Embodied Energy", "category": "Energy", "value": 0.3125549498232869}]

tables/corktown/basic_stats.json

@@ -0,0 +1 @@
+{"residents": 3876, "employees": 6198, "max_residents": 120000, "max_employees": 200000, "area": 4}