work on geog refactor #17 and removed nomis table name from queries, cloing #23

Author: virgesmith

R/Nomisweb.R

@@ -28,7 +28,7 @@ queryMetadata = function() {
 #' geoCodeLookup(censusapi, "MSOA11")
 #' @export
 geoCodeLookup = function(api, geoCodeString) {
-  return(as.integer(api$GeoCodeLookup[geoCodeString]))
+  return(as.character(api$GeoCodeLookup[geoCodeString]))
 }
 
 #' getMetadata()
@@ -76,9 +76,9 @@ getMetadata = function(api, tableName) {
 #' getData(censusapi, table, meta$nomis_table, queryParams)
 #' }
 #' @export
-getData = function(api, tableName, internalName, query) {
+getData = function(api, tableName, query) {
   # returned value is filename (or error) to avoid data frame compatibility issues
-  filename = api$get_data(tableName, internalName, query, TRUE)
+  filename = api$get_data(tableName, query, TRUE)
   # TODO check that string isnt an error!
   return(read.csv(filename, sep="\t", stringsAsFactors = FALSE))
 }
@@ -113,7 +113,7 @@ getLADCodes = function(api, laNames) {
 #' @export
 geoCodes = function(api, coverage, resolution) {
   # force correct types
-  return(api$get_geo_codes(as.integer(coverage), as.integer(resolution)))
+  return(api$get_geo_codes(as.integer(coverage), resolution))
 }
 
 #' contextify

inst/examples/contextify.R

@@ -11,7 +11,6 @@ cacheDir = "/tmp/UKCensusAPI"
 # Here's a predefined query, to which we add contextual data
 
 table = "KS401EW"
-table_internal = "NM_618_1"
 queryParams = list(
   date = "latest",
   RURAL_URBAN = "0",
@@ -24,7 +23,7 @@ queryParams = list(
 api = instance(cacheDir)
 
 # Fetch the data
-KS401EW = getData(api, table, table_internal, queryParams)
+KS401EW = getData(api, table, queryParams)
 
 # Add the context...
 KS401EW = contextify(api, table, "CELL", KS401EW)

inst/examples/contextify.py

@@ -16,7 +16,6 @@ def main():
 
   # Heres predefined query on a small geographical area
   table = "KS401EW"
-  table_internal = "NM_618_1"
   query_params = {}
   query_params["CELL"] = "7...13"
   query_params["date"] = "latest"
@@ -25,7 +24,7 @@ def main():
   query_params["geography"] = "1245710558...1245710560"
   query_params["MEASURES"] = "20100"
 
-  ks401 = api.get_data(table, table_internal, query_params)
+  ks401 = api.get_data(table, query_params)
   # display the first ten rows
   print(ks401.head(10))
 

inst/examples/geoquery.R

@@ -11,7 +11,6 @@ cacheDir = "/tmp/UKCensusAPI"
 # Here's a predefined query using Leeds at MSOA resolution,
 # but we want to change the geographical area and refine the resolution
 table = "KS401EW"
-table_internal = "NM_618_1"
 queryParams = list(
   date = "latest",
   RURAL_URBAN = "0",
@@ -24,15 +23,15 @@ queryParams = list(
 api = instance(cacheDir)
 
 # Define the new region and resolution
-coverage = c("City of London")
+coverage = c("City of London", "Westminster")
 resolution = geoCodeLookup(api, "OA11") # OA 2011 - see NomiswebApi.py
 
 # Modify the query
 coverageCodes = getLADCodes(api, coverage)
 queryParams["geography"] = geoCodes(api, coverageCodes, resolution)
 
 # Fetch the new data
-KS401EW = getData(api, table, table_internal, queryParams)
+KS401EW = getData(api, table, queryParams)
 
 # End of example
 

inst/examples/geoquery.py

@@ -11,7 +11,6 @@ def main():
   # In the previous example we had a predefined query using Leeds at MSOA resolution,
   # but we want to expand the geographical area and refine the resolution
   table = "KS401EW"
-  table_internal = "NM_618_1"
   query_params = {}
   query_params["CELL"] = "7...13"
   query_params["date"] = "latest"
@@ -29,15 +28,15 @@ def main():
   # replace the geography value in the query
   query_params["geography"] = api.get_geo_codes(coverage_codes, resolution)
   # get the data
-  ks401fine = api.get_data(table, table_internal, query_params)
+  ks401fine = api.get_data(table, query_params)
   print(ks401fine.head(5))
 
   # Now widen the coverage to England & Wales and coarsen the resolution to LA
   coverage_codes = [Api.Nomisweb.GeoCodeLookup["EnglandWales"]]
   resolution = Api.Nomisweb.GeoCodeLookup["LAD"]
   query_params["geography"] = api.get_geo_codes(coverage_codes, resolution)
   # get the data
-  ks401broad = api.get_data(table, table_internal, query_params)
+  ks401broad = api.get_data(table, query_params)
   print(ks401broad.head(5))
 
 if __name__ == "__main__":

tests/test_all.py

@@ -52,22 +52,21 @@ def test_get_url(self):
 
   def test_get_data(self):
     table = "KS401EW"
-    table_internal = "NM_618_1"
+#    table_internal = "NM_618_1"
     query_params = {}
     query_params["CELL"] = "7...13"
     query_params["date"] = "latest"
     query_params["RURAL_URBAN"] = "0"
     query_params["select"] = "GEOGRAPHY_CODE,CELL,OBS_VALUE"
     query_params["geography"] = "1245710558...1245710560"
     query_params["MEASURES"] = "20100"
-    table = self.api.get_data(table, table_internal, query_params)
+    table = self.api.get_data(table, query_params)
     self.assertEqual(table.shape, (21, 3))
     self.assertEqual(sum(table.OBS_VALUE), 8214)
 
   def test_get_and_add_descriptive_column(self):
 
     table_name = "KS401EW"
-    meta = self.api.load_metadata(table_name)
 
     query_params = {}
     query_params["CELL"] = "7...13"
@@ -76,7 +75,7 @@ def test_get_and_add_descriptive_column(self):
     query_params["select"] = "GEOGRAPHY_CODE,CELL,OBS_VALUE"
     query_params["geography"] = "1245710558...1245710560"
     query_params["MEASURES"] = "20100"
-    table = self.api.get_data(table_name, meta["nomis_table"], query_params)
+    table = self.api.get_data(table_name, query_params)
     self.assertEqual(table.shape, (21, 3))
     self.assertEqual(sum(table.OBS_VALUE), 8214)
 

tests/testthat/test-all.R

@@ -14,10 +14,10 @@ skip_if_no_python_api = function() {
 
 # simply checks we can get nomis geo codes back
 test_that("geoCodeLookup", {
-  expect_true(UKCensusAPI::geoCodeLookup(api, "MSOA11") == 297)
-  expect_true(UKCensusAPI::geoCodeLookup(api, "LSOA01") == 304)
-  expect_true(UKCensusAPI::geoCodeLookup(api, "LAD") == 464)
-  expect_true(UKCensusAPI::geoCodeLookup(api, "EnglandWales") == 2092957703)
+  expect_true(UKCensusAPI::geoCodeLookup(api, "MSOA11") == "TYPE297")
+  expect_true(UKCensusAPI::geoCodeLookup(api, "LSOA01") == "TYPE304")
+  expect_true(UKCensusAPI::geoCodeLookup(api, "LAD") == "TYPE464")
+  expect_true(UKCensusAPI::geoCodeLookup(api, "EnglandWales") == "2092957703")
 })
 
 # simply checks we get data back
@@ -31,14 +31,13 @@ test_that("getMetadata", {
 test_that("getData", {
   skip_if_no_python_api()
   table = "KS401EW"
-  table_internal = "NM_618_1"
   query = list(date = "latest",
                geography = "1245714681...1245714688",
                CELL = "7...13",
                RURAL_URBAN="0",
                measures = "20100",
                select = "GEOGRAPHY_CODE,CELL,OBS_VALUE")
-  expect_true(class(UKCensusAPI::getData(api, table, table_internal, query)) == "data.frame")
+  expect_true(class(UKCensusAPI::getData(api, table, query)) == "data.frame")
 
 })
 
@@ -65,45 +64,44 @@ test_that("getLADCodes", {
 
 test_that("geoCodes empty", {
   skip_if_no_python_api()
-  expect_true(geoCodes(api, c(), 999) == "")
+  expect_true(geoCodes(api, c(), "TYPE999") == "")
 })
 
 test_that("geoCodes invalid", {
   skip_if_no_python_api()
-  expect_true(geoCodes(api, c(999), 999) == "")
+  expect_true(geoCodes(api, c(999), "TYPE999") == "")
 })
 
 test_that("geoCodes single LA", {
   skip_if_no_python_api()
-  expect_true(geoCodes(api, 1946157124, 464) == "1946157124")
+  expect_true(geoCodes(api, 1946157124, "TYPE464") == "1946157124")
 })
 
 test_that("geoCodes multi MSOA", {
   skip_if_no_python_api()
-  expect_true(geoCodes(api, c(1946157124, 1946157128), 297) == "1245710411...1245710471,1245710661...1245710705")
+  expect_true(geoCodes(api, c(1946157124, 1946157128), "TYPE297") == "1245710411...1245710471,1245710661...1245710705")
 })
 
 test_that("geoCodes multi LSOA", {
   skip_if_no_python_api()
-  expect_true(geoCodes(api, c(1946157124, 1946157128), 298) == "1249912854...1249913154,1249913980...1249914188,1249935357...1249935365")
+  expect_true(geoCodes(api, c(1946157124, 1946157128), "TYPE298") == "1249912854...1249913154,1249913980...1249914188,1249935357...1249935365")
 })
 
 test_that("geoCodes single OA", {
   skip_if_no_python_api()
-  expect_true(geoCodes(api, 1946157124, 299) == "1254148629...1254150034,1254267588...1254267709")
+  expect_true(geoCodes(api, 1946157124, "TYPE299") == "1254148629...1254150034,1254267588...1254267709")
 })
 
 test_that("contextify", {
   skip_if_no_python_api()
   table = "KS401EW"
-  table_internal = "NM_618_1"
   query = list(date = "latest",
                geography = "1245714681...1245714688",
                CELL = "7...13",
                RURAL_URBAN="0",
                measures = "20100",
                select = "GEOGRAPHY_CODE,CELL,OBS_VALUE")
-  data = UKCensusAPI::getData(api, table, table_internal, query)
+  data = UKCensusAPI::getData(api, table, query)
   column = "CELL"
 
   data = contextify(api, table, column, data)

ukcensusapi/Nomisweb.py

@@ -32,17 +32,18 @@ class Nomisweb:
   # https://www.nomisweb.co.uk/api/v01/dataset/NM_144_1/geography/2092957703TYPE464.def.sdmx.json
   # https://www.nomisweb.co.uk/api/v01/dataset/NM_1_1/geography/2092957703TYPE464.def.sdmx.json
   GeoCodeLookup = {
-    "LAD": 464,
-    "MSOA11": 297,
-    "LSOA11": 298,
-    "OA11": 299,
-    "MSOA01": 305,
-    "LSOA01": 304,
-    "OA01": 310,   
-    "England": 2092957699,
-    "EnglandWales": 2092957703,
-    "GB": 2092957698,
-    "UK": 2092957697
+    # give meaning to some common nomis geography types/codes
+    "LAD": "TYPE464",
+    "MSOA11": "TYPE297",
+    "LSOA11": "TYPE298",
+    "OA11": "TYPE299",
+    "MSOA01": "TYPE305",
+    "LSOA01": "TYPE304",
+    "OA01": "TYPE310",       
+    "England": "2092957699",
+    "EnglandWales": "2092957703",
+    "GB": "2092957698",
+    "UK": "2092957697"
   }
 
   # initialise, supplying a location to cache downloads
@@ -61,7 +62,8 @@ def __init__(self, cache_dir):
       os.mkdir(self.cache_dir)
       # TODO check dir created
     if Nomisweb.KEY is None:
-      raise RuntimeError("no API key found. Giving up since downloads may be truncated.\n" \
+      raise RuntimeError("No API key found. Whilst downloads still work, they may be truncated,\n" \
+                         "causing potentially unforseen problems in any modelling/analysis.\n" \
                          "Set the key value in the environment variable NOMIS_API_KEY.\n" \
                          "Register at www.nomisweb.co.uk to obtain a key")
 
@@ -91,8 +93,7 @@ def get_geo_codes(self, la_codes, code_type):
 
     geo_codes = []
     for i in range(0, len(la_codes)):
-      path = "api/v01/dataset/NM_144_1/geography/" + str(la_codes[i]) + "TYPE" \
-           + str(code_type) + ".def.sdmx.json?"
+      path = "api/v01/dataset/NM_144_1/geography/" + str(la_codes[i]) + code_type + ".def.sdmx.json?"
       rawdata = self.__fetch_json(path, {})
 
       # use try-catch block to deal with any issues arising from the returned json
@@ -144,7 +145,7 @@ def get_url(self, table_internal, query_params):
   # Two reasons for this:
   # - pandas/R dataframes conversion is done via matrix (which drops col names)
   # - reporting errors to R is useful (print statements aren't displayed in R(Studio))
-  def get_data(self, table, table_internal, query_params, r_compat=False):
+  def get_data(self, table, query_params, r_compat=False):
     """Downloads or retrieves data given a table and query parameters.
     Args:
        table: ONS table name
@@ -154,14 +155,15 @@ def get_data(self, table, table_internal, query_params, r_compat=False):
         a dataframe containing the data. If downloaded, the data is also cached to a file
     """
     query_params["uid"] = Nomisweb.KEY
-    query_string = self.get_url(table_internal, query_params)
+    metadata = self.load_metadata(table)
+    query_string = self.get_url(metadata["nomis_table"], query_params)
 
     filename = self.cache_dir + table + "_" + hashlib.md5(query_string.encode()).hexdigest()+".tsv"
 
     # retrieve if not in cache
     if not os.path.isfile(filename):
-      meta = self.get_metadata(table)
-      self.write_metadata(table, meta)
+      meta = self.load_metadata(table)
+      #self.write_metadata(table, meta)
       print("Downloading and cacheing data: " + filename)
       request.urlretrieve(query_string, filename) #, timeout = Nomisweb.Timeout)
 
@@ -247,6 +249,9 @@ def get_metadata(self, table_name):
               "fields": fields,
               "geographies": geogs }
 
+    # save a copy
+    self.write_metadata(table_name, result)
+
     return result
 
   # loads metadata from cached json if available, otherwises downloads from nomisweb.

ukcensusapi/Query.py

@@ -49,22 +49,20 @@ def table(self):
 
     add_geog = input("Add geography? (y/N): ") == "y"
     if add_geog:
-      query_params["geography"] = self.__add_geog()
+      query_params["geography"] = self.__add_geog(meta)
       #print(query_params)
 
       get_data = input("Get data now? (y/N): ") == "y"
       if get_data:
         print("\n\nGetting data...")
 
         # Fetch (and cache) data
-        self.api.get_data(table, meta["nomis_table"], query_params)
+        self.api.get_data(table, query_params)
 
     # Remove API key in example code (lest it be accidentally committed)
     if "uid" in query_params:
       del query_params["uid"]
 
-    self.api.write_metadata(table, meta)
-
     self.write_code_snippets(table, meta, query_params)
 
   # returns a geography string that can be inserted into an existing query
@@ -77,16 +75,10 @@ def get_geog_from_names(self, coverage, resolution):
     coverage_codes = self.api.get_lad_codes(coverage)
     return self.api.get_geo_codes(coverage_codes, resolution)
 
-  def __add_geog(self):
+  def __add_geog(self, metadata):
 
     coverage = input("\nGeographical coverage\nE/EW/GB/UK or LAD codes(s)/name(s), comma separated: ")
 
-    resolution = input("Resolution (LAD/MSOA11/LSOA11/OA11/MSOA01/LSOA01/OA01): ")
-    while not resolution in Api.Nomisweb.GeoCodeLookup.keys():
-      print(resolution + " is not valid")
-      resolution = input("Resolution (LAD/MSOA11/LSOA11/OA11/MSOA01/LSOA01/OA01): ")
-    resolution = Api.Nomisweb.GeoCodeLookup[resolution]
-
     if coverage == "E":
       coverage_codes = [Api.Nomisweb.GeoCodeLookup["England"]]
     elif coverage == "EW":
@@ -98,6 +90,18 @@ def __add_geog(self):
     else:
       coverage_codes = self.api.get_lad_codes(coverage.split(","))
 
+    #print(metadata)
+    for key in metadata["geographies"]:
+      print(key, metadata["geographies"][key])
+
+    resolution_valid = False
+    while not resolution_valid:
+      resolution = input("Select Resolution: ")
+      if resolution in metadata["geographies"].keys():
+        resolution_valid = True
+      else:
+        print(resolution + " is not valid")
+
     area_codes = self.api.get_geo_codes(coverage_codes, resolution)
     return area_codes
 
@@ -121,7 +125,7 @@ def write_code_snippets(self, table, meta, query_params):
         py_file.write("\nquery_params[\""+key+"\"] = \""+query_params[key]+"\"")
       if not "geography" in query_params:
         py_file.write("\n# TODO query_params[\"geography\"] = ...")
-      py_file.write("\n" + table + " = api.get_data(table, table_internal, query_params)\n")
+      py_file.write("\n" + table + " = api.get_data(table, query_params)\n")
 
     print("\nWriting R code snippet to " + self.api.cache_dir + table + ".R")
     with open(self.api.cache_dir + table + ".R", "w") as r_file:
@@ -146,4 +150,4 @@ def write_code_snippets(self, table, meta, query_params):
       if not "geography" in query_params:
         r_file.write("\n  # TODO add geography parameter to this query...")
       r_file.write("\n)")
-      r_file.write("\n" + table + " = UKCensusAPI::getData(api, table, table_internal, queryParams)\n")
+      r_file.write("\n" + table + " = UKCensusAPI::getData(api, table, queryParams)\n")