Skip to content

Commit 1d7013b

Browse files
jeffdefactojeffdefacto
committed
Add support for geopackage
This adds geopackage support to batch-machine. I've tested it locally and its working as expected.
1 parent 25d0ec1 commit 1d7013b

6 files changed

Lines changed: 178 additions & 5 deletions

File tree

‎README.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,3 +60,5 @@ openaddr-process-one --skip-preview --layer addresses --layersource state exampl
6060
```
6161

6262
Review https://github.com/openaddresses/openaddresses/blob/master/CONTRIBUTING.md for input json syntax.
63+
64+
Supported conform formats include `shapefile`, `geojson`, `csv`, `xml`, `gdb`, and `gpkg`.

‎openaddr/conform.py‎

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -404,6 +404,28 @@ def find_source_path(data_source, source_paths):
404404
return c
405405
_L.warning("Source names file %s but could not find it", source_file_name)
406406
return None
407+
elif format_string == "gpkg":
408+
candidates = []
409+
for fn in source_paths:
410+
basename, ext = os.path.splitext(fn)
411+
if ext.lower() == ".gpkg":
412+
candidates.append(fn)
413+
if len(candidates) == 0:
414+
_L.warning("No GPKG found in %s", source_paths)
415+
return None
416+
elif len(candidates) == 1:
417+
_L.debug("Selected %s for source", candidates[0])
418+
return candidates[0]
419+
else:
420+
if "file" not in conform:
421+
_L.warning("Multiple GPKGs found, but source has no file attribute.")
422+
return None
423+
source_file_name = conform["file"]
424+
for c in candidates:
425+
if source_file_name == os.path.basename(c):
426+
return c
427+
_L.warning("Source names file %s but could not find it", source_file_name)
428+
return None
407429
elif format_string == "xml":
408430
# Return file if it's specified, else return the first .gml file we find
409431
if "file" in conform:
@@ -425,7 +447,7 @@ def find_source_path(data_source, source_paths):
425447
return None
426448

427449
class ConvertToGeojsonTask(object):
428-
known_types = ('.shp', '.json', '.csv', '.kml', '.gdb')
450+
known_types = ('.shp', '.json', '.csv', '.kml', '.gdb', '.gpkg')
429451

430452
def convert(self, source_config, source_paths, workdir):
431453
"Convert a list of source_paths and write results in workdir"
@@ -1151,7 +1173,7 @@ def extract_to_source_csv(source_config, source_path, extract_path):
11511173
format_string = source_config.data_source["conform"]['format']
11521174
protocol_string = source_config.data_source['protocol']
11531175

1154-
if format_string in ("shapefile", "xml", "gdb"):
1176+
if format_string in ("shapefile", "xml", "gdb", "gpkg"):
11551177
ogr_source_path = normalize_ogr_filename_case(source_path)
11561178
ogr_source_to_csv(source_config, ogr_source_path, extract_path)
11571179
elif format_string == "csv":
@@ -1194,7 +1216,7 @@ def conform_cli(source_config, source_path, dest_path):
11941216

11951217
format_string = source_config.data_source["conform"].get('format')
11961218

1197-
if not format_string in ["shapefile", "geojson", "csv", "xml", "gdb"]:
1219+
if not format_string in ["shapefile", "geojson", "csv", "xml", "gdb", "gpkg"]:
11981220
_L.warning("Skipping file with unknown conform: %s", source_path)
11991221
return 1
12001222

‎openaddr/tests/conform.py‎

Lines changed: 32 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1776,6 +1776,36 @@ def test_lake_man_gdb(self):
17761776
self.assertEqual(rows[5]['properties']['number'], '5115')
17771777
self.assertEqual(rows[5]['properties']['street'], 'OLD MILL RD')
17781778

1779+
def test_lake_man_gpkg(self):
1780+
with open(os.path.join(self.conforms_dir, "lake-man-gpkg.json")) as file:
1781+
source_config = SourceConfig(json.load(file), "addresses", "default")
1782+
source_path = os.path.join(self.conforms_dir, "lake-man.gpkg")
1783+
dest_path = os.path.join(self.testdir, 'lake-man-gpkg-conformed.csv')
1784+
1785+
rc = conform_cli(source_config, source_path, dest_path)
1786+
self.assertEqual(0, rc)
1787+
1788+
with open(dest_path) as fp:
1789+
rows = list(map(json.loads, list(fp)))
1790+
1791+
self.assertEqual('Point', rows[0]['geometry']['type'])
1792+
self.assertAlmostEqual(-122.2592497, rows[0]['geometry']['coordinates'][0], places=4)
1793+
self.assertAlmostEqual(37.8026126, rows[0]['geometry']['coordinates'][1], places=4)
1794+
1795+
self.assertEqual(6, len(rows))
1796+
self.assertEqual(rows[0]['properties']['number'], '5115')
1797+
self.assertEqual(rows[0]['properties']['street'], 'FRUITED PLAINS LN')
1798+
self.assertEqual(rows[1]['properties']['number'], '5121')
1799+
self.assertEqual(rows[1]['properties']['street'], 'FRUITED PLAINS LN')
1800+
self.assertEqual(rows[2]['properties']['number'], '5133')
1801+
self.assertEqual(rows[2]['properties']['street'], 'FRUITED PLAINS LN')
1802+
self.assertEqual(rows[3]['properties']['number'], '5126')
1803+
self.assertEqual(rows[3]['properties']['street'], 'FRUITED PLAINS LN')
1804+
self.assertEqual(rows[4]['properties']['number'], '5120')
1805+
self.assertEqual(rows[4]['properties']['street'], 'FRUITED PLAINS LN')
1806+
self.assertEqual(rows[5]['properties']['number'], '5115')
1807+
self.assertEqual(rows[5]['properties']['street'], 'OLD MILL RD')
1808+
17791809
def test_lake_man_split(self):
17801810
rc, dest_path = self._run_conform_on_source('lake-man-split', 'shp')
17811811
self.assertEqual(0, rc)
@@ -2309,8 +2339,8 @@ def test_srs(self):
23092339
self.assertEqual(r[0], u'n,s,{GEOM_FIELDNAME}'.format(**globals()))
23102340

23112341
r = r[1].split(',')
2312-
self.assertEquals(r[0], '3203')
2313-
self.assertEquals(r[1], 'SE WOODSTOCK BLVD')
2342+
self.assertEqual(r[0], '3203')
2343+
self.assertEqual(r[1], 'SE WOODSTOCK BLVD')
23142344

23152345
x,y = wkt_pt(r[2])
23162346
self.assertAlmostEqual(-122.630842186651, x, places=4)

‎openaddr/tests/conforms/lake-man-gml.gfs‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
<GMLFeatureClassList>
2+
<GMLFeatureClass>
3+
<Name>lake_man</Name>
4+
<ElementPath>lake_man</ElementPath>
5+
<GeometryName>geometryProperty</GeometryName>
6+
<GeometryElementPath>geometryProperty</GeometryElementPath>
7+
<!--POINT-->
8+
<GeometryType>1</GeometryType>
9+
<DatasetSpecificInfo>
10+
<FeatureCount>6</FeatureCount>
11+
<ExtentXMin>-122.25925</ExtentXMin>
12+
<ExtentXMax>-122.25672</ExtentXMax>
13+
<ExtentYMin>37.80071</ExtentYMin>
14+
<ExtentYMax>37.80436</ExtentYMax>
15+
</DatasetSpecificInfo>
16+
<PropertyDefn>
17+
<Name>MSTRID</Name>
18+
<ElementPath>MSTRID</ElementPath>
19+
<Type>Integer</Type>
20+
</PropertyDefn>
21+
<PropertyDefn>
22+
<Name>CATEGORY</Name>
23+
<ElementPath>CATEGORY</ElementPath>
24+
<Type>String</Type>
25+
<Width>7</Width>
26+
</PropertyDefn>
27+
<PropertyDefn>
28+
<Name>ADDRESSID</Name>
29+
<ElementPath>ADDRESSID</ElementPath>
30+
<Type>Integer</Type>
31+
</PropertyDefn>
32+
<PropertyDefn>
33+
<Name>BASEID</Name>
34+
<ElementPath>BASEID</ElementPath>
35+
<Type>Integer</Type>
36+
</PropertyDefn>
37+
<PropertyDefn>
38+
<Name>REVDATE</Name>
39+
<ElementPath>REVDATE</ElementPath>
40+
<Type>String</Type>
41+
<Width>10</Width>
42+
</PropertyDefn>
43+
<PropertyDefn>
44+
<Name>FLOOR</Name>
45+
<ElementPath>FLOOR</ElementPath>
46+
<Type>Integer</Type>
47+
</PropertyDefn>
48+
<PropertyDefn>
49+
<Name>NUMBER</Name>
50+
<ElementPath>NUMBER</ElementPath>
51+
<Type>Integer</Type>
52+
</PropertyDefn>
53+
<PropertyDefn>
54+
<Name>ZIP</Name>
55+
<ElementPath>ZIP</ElementPath>
56+
<Type>Integer</Type>
57+
</PropertyDefn>
58+
<PropertyDefn>
59+
<Name>ACTIVE</Name>
60+
<ElementPath>ACTIVE</ElementPath>
61+
<Type>String</Type>
62+
<Width>3</Width>
63+
</PropertyDefn>
64+
<PropertyDefn>
65+
<Name>MAILING</Name>
66+
<ElementPath>MAILING</ElementPath>
67+
<Type>String</Type>
68+
<Width>3</Width>
69+
</PropertyDefn>
70+
<PropertyDefn>
71+
<Name>FENAME</Name>
72+
<ElementPath>FENAME</ElementPath>
73+
<Type>String</Type>
74+
<Width>14</Width>
75+
</PropertyDefn>
76+
<PropertyDefn>
77+
<Name>FNAME</Name>
78+
<ElementPath>FNAME</ElementPath>
79+
<Type>String</Type>
80+
<Width>14</Width>
81+
</PropertyDefn>
82+
<PropertyDefn>
83+
<Name>FTYPE</Name>
84+
<ElementPath>FTYPE</ElementPath>
85+
<Type>String</Type>
86+
<Width>2</Width>
87+
</PropertyDefn>
88+
<PropertyDefn>
89+
<Name>STRNAME</Name>
90+
<ElementPath>STRNAME</ElementPath>
91+
<Type>String</Type>
92+
<Width>17</Width>
93+
</PropertyDefn>
94+
<PropertyDefn>
95+
<Name>id</Name>
96+
<ElementPath>id</ElementPath>
97+
<Type>String</Type>
98+
<Width>9</Width>
99+
</PropertyDefn>
100+
</GMLFeatureClass>
101+
</GMLFeatureClassList>

‎openaddr/tests/conforms/lake-man-gpkg.json‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
{
2+
"schema": 2,
3+
"layers": {
4+
"addresses": [{
5+
"name": "default",
6+
"data": "http://fake-web/lake-man.gpkg",
7+
"cache": "http://fake-cache/lake-man.gpkg",
8+
"protocol": "http",
9+
"conform": {
10+
"lon": "X",
11+
"lat": "Y",
12+
"number": "NUMBER",
13+
"street": "STRNAME",
14+
"format": "gpkg"
15+
}
16+
}]
17+
}
18+
}

‎openaddr/tests/conforms/lake-man.gpkg‎

104 KB
Binary file not shown.

0 commit comments

Comments
 (0)