Skip to content

Commit 01faaf3

Browse files
fpurcellfpurcell
committed
fix(init): initial add...
0 parents  commit 01faaf3

File tree

9 files changed

+587
-0
lines changed

9 files changed

+587
-0
lines changed

‎CHANGES.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
0.0.1 (2020-07-20)
2+
------------------
3+
- initial

‎LICENSE‎

Lines changed: 362 additions & 0 deletions
Large diffs are not rendered by default.

‎README.md‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
omt_server
2+
==========
3+
4+
The omt_server project contains:
5+
1. [blah](README.md) ...
6+
7+
install:
8+
1. install python 3.7 (works with py versions >= 2.7), zc.buildout and git
9+
1. git clone https://github.com/OpenTransitTools/omt_server.git
10+
1. cd omt_server
11+
1. buildout
12+
13+
run:
14+
1. bin/test
15+
1. ...

‎buildout.cfg‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
[buildout]
2+
extends = versions.cfg
3+
update-versions-file = versions.cfg
4+
parts = dev prod testrunner pydev
5+
develop = . ../utils/
6+
app-egg-name = ott.omt_server
7+
newest = false
8+
versions = versions
9+
include-site-packages = true
10+
allowed-eggs-from-site-packages = psycopg2 PyCrypto cryptography distribute Setuptools zc.buildout
11+
prefer-final = true
12+
13+
[dev]
14+
recipe = zc.recipe.egg
15+
dependent-scripts = true
16+
interpreter = python
17+
eggs = ott.omt_server[dev]
18+
19+
[prod]
20+
recipe = zc.recipe.egg
21+
dependent-scripts = true
22+
interpreter = python
23+
eggs = ott.omt_server
24+
25+
[testrunner]
26+
recipe = zc.recipe.testrunner
27+
eggs = ${dev:eggs}
28+
script = test
29+
30+
[pydev]
31+
recipe = pb.recipes.pydev
32+
eggs = ${dev:eggs}

‎ott/__init__.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
__import__('pkg_resources').declare_namespace(__name__)

‎ott/omt_server/__init__.py‎

Whitespace-only changes.

‎ott/omt_server/converter.py‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
from ott.utils import object_utils
2+
3+
from ott.osm.osm_cache import OsmCache
4+
from ott.osm.osm2pgsql.osm2pgsql import Osm2pgsql
5+
from ott.loader.gtfs.gtfs_cache import GtfsCache
6+
from ott.loader.otp.graph.otp_exporter import OtpExporter
7+
from ott.loader.otp.graph.otp_builder import OtpBuilder
8+
from ott.loader.gtfsdb.gtfsdb_loader import GtfsdbLoader
9+
from ott.loader.gtfsdb.gtfsdb_exporter import GtfsdbExporter
10+
from ott.loader.gtfsdb_realtime.gtfsdb_realtime_loader import GtfsdbRealtimeLoader
11+
from ott.loader.sum.sum_cache import SumCache
12+
from ott.loader.solr.solr_loader import SolrLoader
13+
from ott.loader.gtfs.fix import Fix
14+
15+
import logging
16+
log = logging.getLogger(__file__)
17+
18+
19+
def download_data():
20+
""" just download data
21+
22+
does the following:
23+
1. update GTFS feeds in cache
24+
2. update OSM data in cache
25+
3. update SUM data in cache
26+
"""
27+
force_update = object_utils.is_force_update()
28+
if force_update:
29+
log.info("step 0 IMPORTANT: loading due to 'force_update'!")
30+
31+
log.info("step 1: cache latest gtfs feeds")
32+
gtfs = GtfsCache()
33+
updated_feeds = gtfs.check_cached_feeds(force_update=force_update)
34+
if updated_feeds and len(updated_feeds) > 0:
35+
log.info("step 1 IMPORTANT: loading on GTFS changes in feed(s): {}!".format(updated_feeds))
36+
force_update = True
37+
38+
# step 1b: TODO temp gtfs_fix until OTP is fixed -- commented due to fixed OTP - April 2020
39+
#f = Fix("TRIMET.zip")
40+
#f.remove_deadhead_stop_times(stop="8169", cull=True)
41+
42+
log.info("step 2: cache latest osm data")
43+
updated_osm = OsmCache.update(force_update=force_update)
44+
if updated_osm:
45+
log.info("step 2 IMPORTANT: loading due to OSM changes!")
46+
force_update = True
47+
pg = Osm2pgsql()
48+
pg.run()
49+
50+
log.info("step 3: download SUM data (BIKETOWN)")
51+
sum_update = SumCache.load
52+
53+
return force_update
54+
55+
56+
def load_all():
57+
""" will load OTP and gtfsdb
58+
59+
load gtfsdb
60+
pg_dump an export file for gtfsdb
61+
62+
build OTP graph
63+
test OTP graph
64+
65+
load SOLR with cached data
66+
"""
67+
force_update = object_utils.is_force_update()
68+
69+
# steps 1 - 3: download data (see above)
70+
# import pdb; pdb.set_trace()
71+
download_data()
72+
73+
log.info("step 4: load gtfsdb")
74+
db = GtfsdbLoader()
75+
db.check_db(force_update=force_update)
76+
77+
log.info("step 5: load otp (build new graph)")
78+
otp = OtpBuilder(force_update=force_update)
79+
otp.build_and_test_graphs(force_update=force_update)
80+
81+
log.info("step 6: load various data layers into SOLR")
82+
solr_load = SolrLoader.load()
83+
84+
85+
def export_all():
86+
"""
87+
"""
88+
log.info("step 1: scp any gtfsdb pg_dump files created in the load_all() call ")
89+
GtfsdbExporter.scp()
90+
91+
log.info("step 2: load any new OSM database snapshot")
92+
osm = OsmCache()
93+
#osm.deploy_db_snapshot()
94+
95+
log.info("step 3: export otp graph")
96+
OtpExporter.export()
97+
98+
log.info("step 4: export... SOLR updates")
99+
#solr_load = SolrLoader.load
100+
101+
102+
def load_and_export():
103+
log.info("***load and build things, then export them and scp' them to production servers ***")
104+
load_all()
105+
export_all()
106+
107+
108+
def restore_production():
109+
""" load (production) new database extracts and deploy new otp graphs
110+
111+
does the following:
112+
1. will any postgres exports into local (production) db
113+
2. enable new OTP graph
114+
115+
"""
116+
log.info("step 1: restore gtfsdb from export")
117+
GtfsdbLoader.restore()
118+
119+
log.info("step 2: load any new OSM database snapshot")
120+
osm = OsmCache()
121+
#osm.restore_db_snapshots()
122+
123+
log.info("step 4: export... SOLR updates")
124+
#solr_load = SolrLoader.load

‎setup.py‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
import os
2+
import sys
3+
from setuptools import setup, find_packages
4+
5+
here = os.path.abspath(os.path.dirname(__file__))
6+
README = open(os.path.join(here, 'README.md')).read()
7+
CHANGES = open(os.path.join(here, 'CHANGES.txt')).read()
8+
9+
requires = [
10+
'ott.utils',
11+
'simplejson',
12+
]
13+
14+
extras_require = dict(
15+
dev=[],
16+
)
17+
18+
19+
setup(
20+
name='ott.omt_server',
21+
version='0.1.1',
22+
description='Open Transit Tools - OpenMapTiles Data/Style/etc... Server',
23+
long_description=README + '\n\n' + CHANGES,
24+
classifiers=[
25+
"Programming Language :: Python",
26+
"Topic :: Internet :: WWW/HTTP",
27+
],
28+
author="Open Transit Tools",
29+
author_email="[email protected]",
30+
dependency_links=[
31+
'git+https://github.com/OpenTransitTools/utils.git#egg=ott.utils-0.1.0',
32+
],
33+
license="Mozilla-derived (http://opentransittools.com)",
34+
url='http://opentransittools.com',
35+
keywords='ott, osm, otp, gtfs, vector, tiles, maps, transit',
36+
packages=find_packages(),
37+
include_package_data=True,
38+
zip_safe=False,
39+
install_requires=requires,
40+
extras_require=extras_require,
41+
tests_require=requires,
42+
test_suite="ott.omt_server.tests",
43+
entry_points="""
44+
[console_scripts]
45+
download_data = ott.omt_server.converter:download_data
46+
""",
47+
)

‎versions.cfg‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
[versions]
2+
3+

0 commit comments

Comments
 (0)