Implement management of OSM data

.gitignore

@@ -2,6 +2,8 @@
 db.sqlite
 settings.py
 .idea
+media
+**.pbf
 
 
 # Byte-compiled / optimized / DLL files

Dockerfile

@@ -1,7 +1,8 @@
 FROM docker.io/debian:buster-slim
 
 RUN apt update
-RUN apt install -y --no-install-recommends pipenv uwsgi uwsgi-plugin-python3 python3-psycopg2 python3-setuptools
+RUN apt install -y --no-install-recommends pipenv uwsgi uwsgi-plugin-python3 python3-psycopg2 python3-setuptools \
+    nginx supervisor
 
 # install project dependencies and add sources
 ADD Pipfile Pipfile.lock /app/src/
@@ -14,14 +15,18 @@ RUN mkdir -p /app/config
 RUN cp /app/src/tileservermapping/settings.py.example /app/config/settings.py
 RUN ln -sf /app/config/settings.py /app/src/tileservermapping/settings.py
 RUN ln -s /app/src/docker/uwsgi.ini /etc/uwsgi/tileservermapping.ini
+RUN ln -s /app/src/docker/supervisor.conf /etc/supervisor/conf.d/app.conf
+RUN ln -sf /app/src/docker/nginx.conf /etc/nginx/sites-enabled/default
 
 # collect staticfiles
 RUN ./manage.py collectstatic --no-input
 
 # container metadata
-CMD uwsgi /etc/uwsgi/tileservermapping.ini
+ENTRYPOINT ["/app/src/docker/entrypoint.sh"]
+CMD ["supervisord", "-n", "-c", "/etc/supervisor/supervisord.conf", "-u", "root"]
 ENV LANG='en_US.UTF-8'
 # http
 EXPOSE 8000/tcp
 # uwsgi
 EXPOSE 3003/tcp
+VOLUME /app/media

README.md

@@ -16,6 +16,7 @@ TM\_DEBUG | *empty* | Enables django debug mode when not empty
 TM\_SECRET\_KEY | v€ry $ecret key | [**Change this in production**](https://docs.djangoproject.com/en/3.0/ref/settings/#std:setting-SECRET_KEY)
 TM\_HOSTS | localhost | Comma sperated list of hostnames which this server responds to
 TM\_DB\_HOST | localhost | Hostname of the postgresql database server
+TM\_MEDIA\_ROOT | /app/media | Where uploaded files are stored
 TM\_DB\_PORT | 5432 | Database port
 TM\_DB\_NAME | osm_tileservermapping | Which database to use on that postgresql server
 TM\_DB\_USER | osm_tileservermapping | User used to authenticate at the database

docker/entrypoint.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+set -e
+
+/app/src/manage.py migrate
+exec "$@"

docker/nginx.conf

@@ -0,0 +1,18 @@
+server {
+    listen 8000;
+    server_name _;
+
+    location / {
+        include uwsgi_params;
+        uwsgi_pass localhost:3003;
+        client_max_body_size 0;
+    }
+
+    location /media {
+        alias /app/media;
+    }
+
+    location /static {
+        alias /app/static;
+    }
+}

docker/supervisor.conf

@@ -0,0 +1,15 @@
+[program:nginx]
+command=nginx -g "daemon off;"
+autorestart=true
+stdout_logfile = /dev/stdout
+stdout_logfile_maxbytes = 0
+stderr_logfile = /dev/stderr
+stderr_logfile_maxbytes = 0
+
+[program:uwsgi]
+command=uwsgi --ini /etc/uwsgi/tileservermapping.ini
+autorestart=true
+stdout_logfile = /dev/stdout
+stdout_logfile_maxbytes = 0
+stderr_logfile = /dev/stderr
+stderr_logfile_maxbytes = 0

docker/uwsgi.ini

@@ -1,7 +1,6 @@
 [uwsgi]
 master = true
 socket = :3003
-http-socket = :8000
 
 plugins = python3
 chdir = /app/src
@@ -13,5 +12,3 @@ cheaper = 2
 
 ; disable uWSGI request logging
 disable-logging = true
-
-static-map = /static=/app/static

tileservermapping/osm_data/admin.py

@@ -0,0 +1,4 @@
+from django.contrib import admin
+from . import models
+
+admin.site.register(models.PlanetDump)

tileservermapping/osm_data/apps.py

@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class OsmDataConfig(AppConfig):
+    name = 'tileservermapping.osm_data'

tileservermapping/osm_data/migrations/0001_initial_squashed_0004_auto_20200524_0937.py

@@ -0,0 +1,32 @@
+# Generated by Django 3.0.5 on 2020-05-24 10:02
+
+from django.db import migrations, models
+import tileservermapping.osm_data.models
+
+
+class Migration(migrations.Migration):
+
+    replaces = [('osm_data', '0001_initial'), ('osm_data', '0002_auto_20200524_0924'), ('osm_data', '0003_auto_20200524_0930'), ('osm_data', '0004_auto_20200524_0937')]
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='PlanetDump',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('x', models.IntegerField(help_text='Slippy map coordinate X')),
+                ('y', models.IntegerField(help_text='Slippy map coordinate Y')),
+                ('z', models.IntegerField(help_text='Slippy map coordinate Z (zoom)')),
+                ('file', models.FileField(upload_to=tileservermapping.osm_data.models.generate_file_name)),
+            ],
+            options={
+                'unique_together': set(),
+            },
+        ),
+        migrations.AddConstraint(
+            model_name='planetdump',
+            constraint=models.UniqueConstraint(fields=('x', 'y', 'z'), name='unique_coordinates'),
+        ),
+    ]

tileservermapping/osm_data/migrations/0002_alter_planet_dump_file_field.py

@@ -0,0 +1,27 @@
+# Generated by Django 3.0.5 on 2020-05-25 15:37
+
+from django.db import migrations, models
+import tileservermapping.osm_data.models
+import tileservermapping.osm_data.storage
+
+
+class Migration(migrations.Migration):
+
+    replaces = [('osm_data', '0002_auto_20200525_1451'), ('osm_data', '0003_auto_20200525_1530')]
+
+    dependencies = [
+        ('osm_data', '0001_initial_squashed_0004_auto_20200524_0937'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='planetdump',
+            name='file',
+            field=models.FileField(default=None, null=True, upload_to=tileservermapping.osm_data.models.generate_file_name),
+        ),
+        migrations.AlterField(
+            model_name='planetdump',
+            name='file',
+            field=models.FileField(default=None, null=True, storage=tileservermapping.osm_data.storage.OverwriteStorage(), upload_to=tileservermapping.osm_data.models.generate_file_name),
+        ),
+    ]

tileservermapping/osm_data/models.py

@@ -0,0 +1,39 @@
+import posixpath
+from django.db import models
+
+from tileservermapping.osm_data.storage import OverwriteStorage
+
+
+def generate_file_name(instance, filename):
+    """
+    Generate uploaded filename based on coordinates of the file
+
+    :param instance: Instance of the newly created database object
+    :type instance: PlanetDump
+    :param filename: Originally uploaded file name
+    :type filename: str
+    """
+    return posixpath.join('planet_dumps', f'{instance.z}_{instance.x}_{instance.y}.pbf')
+
+
+class PlanetDump(models.Model):
+    """
+    Planet dump file encoded in PBF format.
+    Each database object is mapped to one file in the file storage and can be used to manage that file.
+
+    One PlanetDump does not always store the whole planet but only a smaller portion
+    defined by the `x y` and `z` coordinates.
+    """
+
+    x = models.IntegerField(help_text='Slippy map coordinate X')
+    y = models.IntegerField(help_text='Slippy map coordinate Y')
+    z = models.IntegerField(help_text='Slippy map coordinate Z (zoom)')
+    file = models.FileField(upload_to=generate_file_name, null=True, default=None, storage=OverwriteStorage())
+
+    class Meta:
+        constraints = [
+            models.UniqueConstraint(fields=['x', 'y', 'z'], name='unique_coordinates')
+        ]
+
+    def __str__(self):
+        return f'{self.__class__.__name__} of x:{self.x} y:{self.y} z:{self.z}'

tileservermapping/osm_data/serializers.py

@@ -0,0 +1,14 @@
+from rest_framework import serializers, validators
+from . import models
+
+
+class PlanetDumpSerializer(serializers.HyperlinkedModelSerializer):
+    class Meta:
+        model = models.PlanetDump
+        fields = ['url', 'id', 'x', 'y', 'z', 'file']
+        validators = [
+            validators.UniqueTogetherValidator(
+                queryset=models.PlanetDump.objects.all(),
+                fields=['x', 'y', 'z']
+            )
+        ]

tileservermapping/osm_data/storage.py

@@ -0,0 +1,11 @@
+from django.core.files.storage import FileSystemStorage
+
+
+class OverwriteStorage(FileSystemStorage):
+    def _save(self, name, content):
+        if self.exists(name):
+            self.delete(name)
+        return super(OverwriteStorage, self)._save(name, content)
+
+    def get_available_name(self, name, max_length=None):
+        return name

tileservermapping/osm_data/tests.py

@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.

tileservermapping/osm_data/urls.py

@@ -0,0 +1,8 @@
+from django.urls import path, include, re_path
+from rest_framework import routers
+
+from . import views
+
+router = routers.DefaultRouter()
+router.register(r'planet_dumps', views.PlanetDumpViewset)
+urlpatterns = router.urls

tileservermapping/osm_data/views.py

@@ -0,0 +1,19 @@
+from rest_framework import viewsets, permissions, parsers
+from . import models, serializers
+
+
+class PlanetDumpViewset(viewsets.ModelViewSet):
+    queryset = models.PlanetDump.objects.all()
+    serializer_class = serializers.PlanetDumpSerializer
+
+    def get_parsers(self):
+        if self.request.method == 'POST' or self.request.method == 'PATCH' or self.request.method == 'PUT':
+            return [*super(PlanetDumpViewset, self).get_parsers(), parsers.MultiPartParser()]
+        else:
+            return super().get_parsers()
+
+    def get_permissions(self):
+        if self.action == 'list':
+            return [permissions.AllowAny()]
+        else:
+            return super(PlanetDumpViewset, self).get_permissions()

tileservermapping/settings.py.example

@@ -36,3 +36,4 @@ DATABASES = {
 }
 
 STATIC_ROOT = os.path.join('/', 'app', 'static')
+MEDIA_ROOT = os.getenv('TM_MEDIA_ROOT', os.path.join('/', 'app', 'media'))

tileservermapping/settings_base.py

@@ -31,6 +31,7 @@
     'rest_framework',
     'drf_yasg',
     'tileservermapping.mapping',
+    'tileservermapping.osm_data',
 ]
 
 MIDDLEWARE = [
@@ -107,6 +108,7 @@
 # https://docs.djangoproject.com/en/2.0/howto/static-files/
 
 STATIC_URL = '/static/'
+MEDIA_URL = '/media/'
 
 
 # Django Rest Framework and extensions
@@ -116,4 +118,5 @@
     "ALLOWED_VERSIONS": ["v1"],
     "DEFAULT_VERSIONING_CLASS": "rest_framework.versioning.URLPathVersioning",
     "DEFAULT_PERMISSION_CLASSES": ["rest_framework.permissions.IsAuthenticated"],
+    "DEFAULT_PARSER_CLASSES": ["rest_framework.parsers.JSONParser"]
 }

tileservermapping/urls.py

@@ -25,11 +25,12 @@
 
 urlpatterns = [
     path("api/<str:version>/", include("tileservermapping.mapping.urls")),
+    path("api/<str:version>/", include("tileservermapping.osm_data.urls")),
     path("mappings/", include("tileservermapping.mapping.urls")),   # included for compatibility to old url schema
 
     path("admin/", admin.site.urls),
     path("schema<str:format>/", schema_view.without_ui(cache_timeout=0), name="schema-json"),
     path("docs/", schema_view.with_ui("swagger", cache_timeout=0), name="schema-swagger-ui"),
 
-    path("", RedirectView.as_view(url="/docs"))
+    path("", RedirectView.as_view(url="/docs")),
 ]