Implement management of OSM data

.gitignore

@@ -2,6 +2,8 @@
 db.sqlite
 settings.py
 .idea
+media
+**.pbf
 
 
 # Byte-compiled / optimized / DLL files

Dockerfile

@@ -1,27 +1,34 @@
 FROM docker.io/debian:buster-slim
 
 RUN apt update
-RUN apt install -y --no-install-recommends pipenv uwsgi uwsgi-plugin-python3 python3-psycopg2 python3-setuptools
+RUN apt install -y --no-install-recommends pipenv uwsgi uwsgi-plugin-python3 python3-psycopg2 python3-setuptools \
+    nginx supervisor
 
 # install project dependencies and add sources
 ADD Pipfile Pipfile.lock /app/src/
 WORKDIR /app/src
 RUN pipenv install --system --deploy --ignore-pipfile
-ADD . /app/src/
+ADD manage.py /app/src/
+ADD docker /app/src/docker
+ADD tileservermapping /app/src/tileservermapping
 
 # put configuration in correct places
 RUN mkdir -p /app/config
 RUN cp /app/src/tileservermapping/settings.py.example /app/config/settings.py
 RUN ln -sf /app/config/settings.py /app/src/tileservermapping/settings.py
 RUN ln -s /app/src/docker/uwsgi.ini /etc/uwsgi/tileservermapping.ini
+RUN ln -s /app/src/docker/supervisor.conf /etc/supervisor/conf.d/app.conf
+RUN ln -sf /app/src/docker/nginx.conf /etc/nginx/sites-enabled/default
 
 # collect staticfiles
 RUN ./manage.py collectstatic --no-input
 
 # container metadata
-CMD uwsgi /etc/uwsgi/tileservermapping.ini
+ENTRYPOINT ["/app/src/docker/entrypoint.sh"]
+CMD ["supervisord", "-n", "-c", "/etc/supervisor/supervisord.conf", "-u", "root"]
 ENV LANG='en_US.UTF-8'
 # http
 EXPOSE 8000/tcp
 # uwsgi
 EXPOSE 3003/tcp
+VOLUME /app/media

README.md

@@ -41,6 +41,7 @@ TM\_DEBUG | *empty* | Enables django debug mode when not empty
 TM\_SECRET\_KEY | v€ry $ecret key | [**Change this in production**](https://docs.djangoproject.com/en/3.0/ref/settings/#std:setting-SECRET_KEY)
 TM\_HOSTS | localhost | Comma sperated list of hostnames which this server responds to
 TM\_DB\_HOST | localhost | Hostname of the postgresql database server
+TM\_MEDIA\_ROOT | /app/media | Where uploaded files are stored
 TM\_DB\_PORT | 5432 | Database port
 TM\_DB\_NAME | osm_tileservermapping | Which database to use on that postgresql server
 TM\_DB\_USER | osm_tileservermapping | User used to authenticate at the database

docker/entrypoint.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+set -e
+
+/app/src/manage.py migrate
+exec "$@"

docker/nginx.conf

@@ -0,0 +1,18 @@
+server {
+    listen 8000;
+    server_name _;
+
+    location / {
+        include uwsgi_params;
+        uwsgi_pass localhost:3003;
+        client_max_body_size 0;
+    }
+
+    location /media {
+        alias /app/media;
+    }
+
+    location /static {
+        alias /app/static;
+    }
+}

docker/supervisor.conf

@@ -0,0 +1,15 @@
+[program:nginx]
+command=nginx -g "daemon off;"
+autorestart=true
+stdout_logfile = /dev/stdout
+stdout_logfile_maxbytes = 0
+stderr_logfile = /dev/stderr
+stderr_logfile_maxbytes = 0
+
+[program:uwsgi]
+command=uwsgi --ini /etc/uwsgi/tileservermapping.ini
+autorestart=true
+stdout_logfile = /dev/stdout
+stdout_logfile_maxbytes = 0
+stderr_logfile = /dev/stderr
+stderr_logfile_maxbytes = 0

docker/uwsgi.ini

@@ -1,7 +1,6 @@
 [uwsgi]
 master = true
 socket = :3003
-http-socket = :8000
 
 plugins = python3
 chdir = /app/src
@@ -13,5 +12,3 @@ cheaper = 2
 
 ; disable uWSGI request logging
 disable-logging = true
-
-static-map = /static=/app/static

tileservermapping/osm_data/admin.py

@@ -0,0 +1,5 @@
+from django.contrib import admin
+from . import models
+
+admin.site.register(models.PlanetDump)
+admin.site.register(models.SqlDump)

tileservermapping/osm_data/apps.py

@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class OsmDataConfig(AppConfig):
+    name = 'tileservermapping.osm_data'

tileservermapping/osm_data/migrations/0001_initial.py

@@ -0,0 +1,42 @@
+# Generated by Django 3.0.6 on 2020-05-28 18:34
+
+from django.db import migrations, models
+import tileservermapping.osm_data.models
+import tileservermapping.osm_data.storage
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='SqlDump',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('x', models.IntegerField(help_text='Slippy map coordinate X')),
+                ('y', models.IntegerField(help_text='Slippy map coordinate Z')),
+                ('z', models.IntegerField(help_text='Slippy map coordinate Z')),
+                ('file', models.FileField(default=None, null=True, storage=tileservermapping.osm_data.storage.OverwriteStorage(), upload_to=tileservermapping.osm_data.models.gen_sql_dump_location)),
+            ],
+            options={
+                'unique_together': {('x', 'y', 'z')},
+            },
+        ),
+        migrations.CreateModel(
+            name='PlanetDump',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('x', models.IntegerField(help_text='Slippy map coordinate X')),
+                ('y', models.IntegerField(help_text='Slippy map coordinate Y')),
+                ('z', models.IntegerField(help_text='Slippy map coordinate Z (zoom)')),
+                ('file', models.FileField(default=None, null=True, storage=tileservermapping.osm_data.storage.OverwriteStorage(), upload_to=tileservermapping.osm_data.models.gen_planet_dump_location)),
+            ],
+            options={
+                'unique_together': {('x', 'y', 'z')},
+            },
+        ),
+    ]

tileservermapping/osm_data/models.py

@@ -0,0 +1,64 @@
+import posixpath
+from typing import *
+
+from django.db import models
+
+from tileservermapping.osm_data.storage import OverwriteStorage
+
+
+def gen_planet_dump_location(instance, filename) -> str:
+    """
+    Generate uploaded filename based on coordinates of the instance
+
+    :param PlanetDump instance: Instance of the newly created database object
+    :param str filename: Originally uploaded file name
+    """
+    return f'planet_dumps/{instance.z}_{instance.x}_{instance.y}.pbf'
+
+
+def gen_sql_dump_location(instance, filename) -> str:
+    """
+    Generate uploaded filename based on coordinates of the file
+
+    :param SqlDump instance: Instance of the newly created database object
+    :param str filename: Originally uploaded file name
+    """
+    return f'sql_dumps/{instance.z}_{instance.x}_{instance.y}.pg_dump'
+
+
+class PlanetDump(models.Model):
+    """
+    Planet dump file encoded in PBF format.
+    Each database object is mapped to one file in the file storage and can be used to manage that file.
+
+    One PlanetDump does not always store the whole planet but only a smaller portion
+    defined by the `x y` and `z` coordinates.
+    """
+
+    x = models.IntegerField(help_text='Slippy map coordinate X')
+    y = models.IntegerField(help_text='Slippy map coordinate Y')
+    z = models.IntegerField(help_text='Slippy map coordinate Z (zoom)')
+    file = models.FileField(upload_to=gen_planet_dump_location, null=True, default=None, storage=OverwriteStorage())
+
+    class Meta:
+        unique_together = [['x', 'y', 'z']]
+
+    def __str__(self):
+        return f'{self.__class__.__name__} of x:{self.x} y:{self.y} z:{self.z}'
+
+
+class SqlDump(models.Model):
+    """
+    PostgreSQL Dump files which hold all the data a Tileserver needs.
+    """
+
+    x = models.IntegerField(help_text='Slippy map coordinate X')
+    y = models.IntegerField(help_text='Slippy map coordinate Z')
+    z = models.IntegerField(help_text='Slippy map coordinate Z')
+    file = models.FileField(upload_to=gen_sql_dump_location, null=True, default=None, storage=OverwriteStorage())
+
+    class Meta:
+        unique_together = [['x', 'y', 'z']]
+
+    def __str__(self):
+        return f'{self.__class__.__name__} of x:{self.x} y:{self.y} z:{self.z}'

tileservermapping/osm_data/serializers.py

@@ -0,0 +1,30 @@
+from rest_framework import serializers, validators
+from . import models
+
+
+class PlanetDumpSerializer(serializers.HyperlinkedModelSerializer):
+    class Meta:
+        model = models.PlanetDump
+        fields = ['url', 'id', 'x', 'y', 'z', 'file']
+        validators = [
+            validators.UniqueTogetherValidator(
+                queryset=models.PlanetDump.objects.all(),
+                fields=['x', 'y', 'z']
+            )
+        ]
+
+    # TODO: Validate uploaded pbf file
+
+
+class SqlDumpSerializer(serializers.HyperlinkedModelSerializer):
+    class Meta:
+        model = models.SqlDump
+        fields = ['url', 'id', 'x', 'y', 'z', 'file']
+        validators = [
+            validators.UniqueTogetherValidator(
+                queryset=models.SqlDump.objects.all(),
+                fields=['x', 'y', 'z']
+            )
+        ]
+
+    # TODO: Validate uploaded postgresql-dump file

tileservermapping/osm_data/storage.py

@@ -0,0 +1,11 @@
+from django.core.files.storage import FileSystemStorage
+
+
+class OverwriteStorage(FileSystemStorage):
+    def _save(self, name, content):
+        if self.exists(name):
+            self.delete(name)
+        return super(OverwriteStorage, self)._save(name, content)
+
+    def get_available_name(self, name, max_length=None):
+        return name

tileservermapping/osm_data/tests.py

@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.

tileservermapping/osm_data/urls.py

@@ -0,0 +1,9 @@
+from django.urls import path, include, re_path
+from rest_framework import routers
+
+from . import views
+
+router = routers.DefaultRouter()
+router.register(r'planet_dumps', views.PlanetDumpViewset)
+router.register(r'postgresql_dumps', views.SqlDumpViewset)
+urlpatterns = router.urls

tileservermapping/osm_data/views.py

@@ -0,0 +1,36 @@
+from rest_framework import viewsets, permissions, parsers
+from . import models, serializers
+
+
+class PlanetDumpViewset(viewsets.ModelViewSet):
+    queryset = models.PlanetDump.objects.all()
+    serializer_class = serializers.PlanetDumpSerializer
+
+    def get_parsers(self):
+        if self.request.method == 'POST' or self.request.method == 'PATCH' or self.request.method == 'PUT':
+            return [*super(PlanetDumpViewset, self).get_parsers(), parsers.MultiPartParser()]
+        else:
+            return super().get_parsers()
+
+    def get_permissions(self):
+        if self.action == 'list':
+            return [permissions.AllowAny()]
+        else:
+            return super(PlanetDumpViewset, self).get_permissions()
+
+
+class SqlDumpViewset(viewsets.ModelViewSet):
+    queryset = models.SqlDump.objects.all()
+    serializer_class = serializers.SqlDumpSerializer
+
+    def get_parsers(self):
+        if self.request.method == 'POST' or self.request.method == 'PATCH' or self.request.method == 'PUT':
+            return [*super(SqlDumpViewset, self).get_parsers(), parsers.MultiPartParser()]
+        else:
+            return super().get_parsers()
+
+    def get_permissions(self):
+        if self.action == 'list':
+            return [permissions.AllowAny()]
+        else:
+            return super(SqlDumpViewset, self).get_permissions()

tileservermapping/settings.py.example

@@ -36,3 +36,4 @@ DATABASES = {
 }
 
 STATIC_ROOT = os.path.join('/', 'app', 'static')
+MEDIA_ROOT = os.getenv('TM_MEDIA_ROOT', os.path.join('/', 'app', 'media'))

tileservermapping/settings_base.py

@@ -34,6 +34,7 @@
     'drf_yasg',
     'tileservermapping.mapping',
     'tileservermapping.service_accounts',
+    'tileservermapping.osm_data',
 ]
 
 MIDDLEWARE = [
@@ -110,6 +111,7 @@
 # https://docs.djangoproject.com/en/2.0/howto/static-files/
 
 STATIC_URL = '/static/'
+MEDIA_URL = '/media/'
 
 
 # Django Rest Framework and extensions
@@ -123,9 +125,8 @@
         'rest_framework.authentication.SessionAuthentication',
         'tileservermapping.service_accounts.authentication_classes.ServiceAccountTokenAuthentication'
     ],
-    "DEFAULT_PERMISSION_CLASSES": [
-        "rest_framework.permissions.IsAuthenticated"
-    ],
+    "DEFAULT_PERMISSION_CLASSES": ["rest_framework.permissions.IsAuthenticated"],
+    "DEFAULT_PARSER_CLASSES": ["rest_framework.parsers.JSONParser"]
 }
 
 

tileservermapping/urls.py

@@ -25,12 +25,13 @@
 
 urlpatterns = [
     path("api/<str:version>/", include("tileservermapping.mapping.urls")),
+    path("api/<str:version>/", include("tileservermapping.osm_data.urls")),
     path("api/<str:version>/", include("tileservermapping.service_accounts.urls")),
     path("mappings/", include("tileservermapping.mapping.urls")),   # included for compatibility to old url schema
 
     path("admin/", admin.site.urls),
     path("schema<str:format>/", schema_view.without_ui(cache_timeout=0), name="schema-json"),
     path("docs/", schema_view.with_ui("swagger", cache_timeout=0), name="schema-swagger-ui"),
 
-    path("", RedirectView.as_view(url="/docs"))
+    path("", RedirectView.as_view(url="/docs")),
 ]