diff --git a/.gitignore b/.gitignore index 9066f341..1e183cd3 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,6 @@ *.egg-info .settings .idea +mapillary_run_provider.yml +docker/dockercontext/mapillary +gitversion diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 00000000..6d145bb0 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,7 @@ +#!groovy +@Library('mapillary-pipeline') _ +com.mapillary.pipeline.Pipeline.builder(this, steps) + .defaultPipeline() + .skipSystemStage() + .build() + .execute() diff --git a/docker/Makefile b/docker/Makefile new file mode 100644 index 00000000..d04d2262 --- /dev/null +++ b/docker/Makefile @@ -0,0 +1,13 @@ +# MISC +NAME = mapillary_tools + +# BOOT +RUN_DIR=/mapillary_source +RUN_CMD=script/run upload + +export + +DOCKER_CONTEXT=../ + +MAPILLARY_CLUSTER_DIR ?= /opt/mapillary_cluster +include $(MAPILLARY_CLUSTER_DIR)/build/app/Makefile diff --git a/docker/dockercontext/Dockerfile b/docker/dockercontext/Dockerfile new file mode 100644 index 00000000..e68f8d2a --- /dev/null +++ b/docker/dockercontext/Dockerfile @@ -0,0 +1,23 @@ +FROM ubuntu:16.04 + +# SETUP +RUN \ + apt-get -qq update && \ + apt-get -yqq install \ + git \ + python-pip && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +RUN \ + pip install --upgrade pip + +COPY . /source/mapillary_tools + +WORKDIR /source/mapillary_tools + +RUN pip install -r requirements.txt + +COPY docker/dockercontext/mapillary/ /mapillary + +ENV PATH="/mapillary:${PATH}" diff --git a/mapillary_tools/uploader.py b/mapillary_tools/uploader.py index 985fcfaf..e5f80785 100644 --- a/mapillary_tools/uploader.py +++ b/mapillary_tools/uploader.py @@ -19,7 +19,7 @@ import sys import processing -MAPILLARY_UPLOAD_URL = "https://d22zcsn13kp53w.cloudfront.net/" +MAPILLARY_UPLOAD_URL = "http://awss3provider:4569/mapillary.uploads.images" MAPILLARY_DIRECT_UPLOAD_URL = "https://s3-eu-west-1.amazonaws.com/mapillary.uploads.images" PERMISSION_HASH = "eyJleHBpcmF0aW9uIjoiMjAyMC0wMS0wMVQwMDowMDowMFoiLCJjb25kaXRpb25zIjpbeyJidWNrZXQiOiJtYXBpbGxhcnkudXBsb2Fkcy5pbWFnZXMifSxbInN0YXJ0cy13aXRoIiwiJGtleSIsIiJdLHsiYWNsIjoicHJpdmF0ZSJ9LFsic3RhcnRzLXdpdGgiLCIkQ29udGVudC1UeXBlIiwiIl0sWyJjb250ZW50LWxlbmd0aC1yYW5nZSIsMCwyMDQ4NTc2MF1dfQ==" SIGNATURE_HASH = "f6MHj3JdEq8xQ/CmxOOS7LvMxoI=" diff --git a/requirements.txt b/requirements.txt index 4138fc0b..ba790317 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ -e . -git+https://github.com/mapillary/Piexif \ No newline at end of file +git+https://github.com/mapillary/Piexif +redis diff --git a/script/run b/script/run new file mode 100755 index 00000000..1ee65d74 --- /dev/null +++ b/script/run @@ -0,0 +1,13 @@ +#!/bin/bash + +source map_cluster_init + +if [ "$1" = "upload" ]; +then + map_cluster_wait_for_it $S3_WAIT + while : + do + mapillary_tools upload --import_path tests/integration/mapillary_tools_upload_provider/data/ + sleep 60 + done +fi diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/integration/mapillary_tools_upload_provider/__init__.py b/tests/integration/mapillary_tools_upload_provider/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/integration/mapillary_tools_upload_provider/data/2016_05_14_11_50_34_383.jpg b/tests/integration/mapillary_tools_upload_provider/data/2016_05_14_11_50_34_383.jpg new file mode 100644 index 00000000..b2ab97b4 Binary files /dev/null and b/tests/integration/mapillary_tools_upload_provider/data/2016_05_14_11_50_34_383.jpg differ diff --git a/tests/integration/mapillary_tools_upload_provider/data/2018_07_19_17_33_15_820_+0200.jpg b/tests/integration/mapillary_tools_upload_provider/data/2018_07_19_17_33_15_820_+0200.jpg new file mode 100644 index 00000000..07db5d65 Binary files /dev/null and b/tests/integration/mapillary_tools_upload_provider/data/2018_07_19_17_33_15_820_+0200.jpg differ diff --git a/tests/integration/mapillary_tools_upload_provider/data/__init__.py b/tests/integration/mapillary_tools_upload_provider/data/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/integration/mapillary_tools_upload_provider/test_upload_provider.py b/tests/integration/mapillary_tools_upload_provider/test_upload_provider.py new file mode 100644 index 00000000..004f1db9 --- /dev/null +++ b/tests/integration/mapillary_tools_upload_provider/test_upload_provider.py @@ -0,0 +1,20 @@ +import os +from time import sleep + +from mapillary_messi.fixtures.psql_main_fixtures.psql_main_user_fixture import PsqlMainUserFixture +from mapillary_messi.models import User +from testtools import TestCase + +UPLOADED_FILENAME = "2016_05_14_11_50_34_383.jpg" + +class UploadProviderTestCase(TestCase): + + def test_images_uploaded_are_harvested(self): + user_a = User() + self.useFixture(PsqlMainUserFixture([user_a])) + + current_dir = os.path.abspath(__file__) + data_dir = os.path.dirname(current_dir) + image_path = os.path.join(data_dir, 'data/{}'.format(UPLOADED_FILENAME)) + + sleep(3000)