diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..113bf4f11 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,42 @@ +# Build artifacts +target/ +build/ +.gradle/ + +# IDE files +.idea/ +*.iml +.vscode/ +.project +.classpath +.settings/ + +# OS files +.DS_Store +Thumbs.db + +# Version control +.git/ +.gitignore + +# Documentation +docs/ +*.md +!README.md + +# Test data and temporary files +photon_data/ +*.tmp +*.log + +# Docker files (except the ones we need) +docker-compose.override.yml +.dockerignore + +# Node modules if any +node_modules/ + +# Other temporary files +*.swp +*.swo +*~ \ No newline at end of file diff --git a/.gitignore b/.gitignore index 8dc69496f..b3f7afaad 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ photon_data/ dependency-reduced-pom.xml .gradle/ app/*/build +build/ .idea/ *.iml diff --git a/DOCKER.md b/DOCKER.md new file mode 100644 index 000000000..4ac949591 --- /dev/null +++ b/DOCKER.md @@ -0,0 +1,399 @@ +# Photon Docker Multi-Country Deployment + +This Docker implementation provides comprehensive support for deploying Photon with multi-country configurations, allowing you to run country-specific geocoding services or world-wide instances. + +## Features + +- **Multi-country support**: Deploy separate instances for different countries or regions +- **Automatic data download**: Automatically downloads and imports official Photon JSONL dumps +- **Flexible configuration**: Environment-based configuration for easy customization +- **Health checks**: Built-in health monitoring for all services +- **Load balancing**: Optional nginx reverse proxy for distributing requests +- **Resource optimization**: Configurable memory and performance settings +- **Volume management**: Persistent data storage with Docker volumes + +## Quick Start + +### Simple Single-Country Deployment + +For a quick start with a single country (e.g., Germany): + +```bash +# Clone the repository +git clone +cd photon-docker + +# Start with Germany data +docker-compose -f docker-compose.simple.yml up -d + +# The service will be available at http://localhost:2322 +``` + +### Multi-Country Deployment + +For a complete multi-country setup: + +```bash +# Start all services (this will take time to download and import data) +docker-compose up -d + +# Services will be available at: +# - World-wide: http://localhost:2322 +# - Europe: http://localhost:2323 +# - North America: http://localhost:2324 +# - Germany: http://localhost:2325 +``` + +## Configuration + +### Environment Variables + +| Variable | Default | Description | +|----------|---------|-------------| +| `PHOTON_COUNTRIES` | `""` | Comma-separated country codes (ISO 3166-1 alpha-2). Empty = world-wide | +| `PHOTON_LANGUAGES` | `"en,de,fr,it"` | Comma-separated language codes | +| `PHOTON_DATA_DIR` | `"/photon/data"` | Directory for Photon database files | +| `PHOTON_LISTEN_IP` | `"0.0.0.0"` | IP address to bind to | +| `PHOTON_LISTEN_PORT` | `"2322"` | Port to listen on | +| `PHOTON_CORS_ANY` | `"false"` | Enable CORS for any origin | +| `PHOTON_UPDATE_API` | `"false"` | Enable update API endpoint | +| `PHOTON_MAX_RESULTS` | `"50"` | Maximum number of search results | +| `PHOTON_AUTO_DOWNLOAD` | `"true"` | Automatically download data if not present | +| `PHOTON_SYNONYM_FILE` | `""` | Path to synonym file | +| `PHOTON_EXTRA_TAGS` | `""` | Extra OSM tags to include | +| `PHOTON_CUSTOM_ARGS` | `""` | Additional command line arguments | +| `JAVA_OPTS` | `""` | JVM options (e.g., `-Xmx4g`) | + +### Country Codes + +Use ISO 3166-1 alpha-2 country codes. Examples: + +- **Single country**: `PHOTON_COUNTRIES=de` +- **Multiple countries**: `PHOTON_COUNTRIES=de,fr,it,es` +- **Region examples**: + - Europe: `de,fr,it,es,nl,be,at,ch,pl,cz` + - North America: `us,ca,mx` + - Nordic: `se,no,dk,fi` + +### Memory Requirements + +Recommended memory settings based on data size: + +- **Single small country** (e.g., Netherlands): `-Xmx1g` +- **Single large country** (e.g., Germany, France): `-Xmx2g` +- **Multiple countries**: `-Xmx3g-4g` +- **World-wide**: `-Xmx8g+` (requires significant memory) + +## Docker Compose Examples + +### Custom Country Configuration + +Create your own `docker-compose.override.yml`: + +```yaml +version: '3.8' + +services: + photon-custom: + build: . + container_name: photon-custom + ports: + - "2326:2322" + environment: + - PHOTON_COUNTRIES=se,no,dk,fi # Nordic countries + - PHOTON_LANGUAGES=sv,no,da,fi,en + - PHOTON_CORS_ANY=true + - JAVA_OPTS=-Xmx2g + volumes: + - photon-custom-data:/photon/data + - photon-custom-dumps:/photon/dumps + restart: unless-stopped + +volumes: + photon-custom-data: + photon-custom-dumps: +``` + +### Production Configuration + +For production deployments: + +```yaml +version: '3.8' + +services: + photon-prod: + build: . + container_name: photon-prod + ports: + - "2322:2322" + environment: + - PHOTON_COUNTRIES=de,fr,it,es,nl,be,at,ch + - PHOTON_LANGUAGES=de,fr,it,es,nl,en + - PHOTON_CORS_ANY=false + - PHOTON_UPDATE_API=true + - PHOTON_MAX_RESULTS=25 + - JAVA_OPTS=-Xmx6g -XX:+UseG1GC -XX:MaxGCPauseMillis=200 + volumes: + - /opt/photon/data:/photon/data + - /opt/photon/dumps:/photon/dumps + - /opt/photon/logs:/photon/logs + restart: unless-stopped + logging: + driver: "json-file" + options: + max-size: "100m" + max-file: "5" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:2322/status"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 300s +``` + +## Data Management + +### Data Sources + +The Docker implementation automatically downloads data from official Photon sources: +- World-wide: `https://download1.graphhopper.com/public/experimental/` +- Country extracts: `https://download1.graphhopper.com/public/experimental/extracts/` + +### Manual Data Import + +To use your own data files: + +1. Disable auto-download: `PHOTON_AUTO_DOWNLOAD=false` +2. Mount your data files to `/photon/dumps/` +3. Name your file `photon-db-latest.bz2` or `photon-db-latest.jsonl` + +```yaml +services: + photon: + # ... other configuration + environment: + - PHOTON_AUTO_DOWNLOAD=false + volumes: + - ./my-data.bz2:/photon/dumps/photon-db-latest.bz2:ro + - photon-data:/photon/data +``` + +### Data Updates + +For production environments with update capability: + +```bash +# Enable update API +PHOTON_UPDATE_API=true + +# Trigger updates via API +curl http://localhost:2322/nominatim-update + +# Check update status +curl http://localhost:2322/nominatim-update/status +``` + +## Networking and Load Balancing + +### With Nginx Proxy + +To use the included nginx load balancer: + +```bash +docker-compose --profile with-proxy up -d +``` + +This provides: +- Default world-wide service on port 80 +- Region-specific routing based on hostname +- CORS headers +- Health check endpoints + +### Custom Load Balancing + +For custom load balancing, you can: + +1. Use external load balancers (HAProxy, Traefik, etc.) +2. Route based on query parameters +3. Implement geographic routing + +## Monitoring and Logging + +### Health Checks + +All services include health checks via `/status` endpoint: + +```bash +# Check service health +curl http://localhost:2322/status + +# Docker health status +docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}" +``` + +### Logs + +Access container logs: + +```bash +# View logs +docker-compose logs photon-germany + +# Follow logs +docker-compose logs -f photon-europe + +# View all logs +docker-compose logs +``` + +### Monitoring with External Tools + +Example with Prometheus monitoring: + +```yaml +services: + photon: + # ... existing configuration + labels: + - "prometheus.scrape=true" + - "prometheus.port=2322" + - "prometheus.path=/status" +``` + +## Troubleshooting + +### Common Issues + +1. **Out of Memory** + ```bash + # Increase JVM heap size + JAVA_OPTS=-Xmx4g + ``` + +2. **Slow Download/Import** + ```bash + # Check download progress + docker-compose logs -f photon-germany + + # Monitor disk space + docker system df + ``` + +3. **Port Conflicts** + ```bash + # Change port mapping + ports: + - "2323:2322" # Use different host port + ``` + +4. **Data Corruption** + ```bash + # Remove data and restart + docker-compose down + docker volume rm photon-docker_photon-data + docker-compose up -d + ``` + +### Performance Tuning + +1. **JVM Tuning**: + ``` + JAVA_OPTS=-Xmx4g -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:+DisableExplicitGC + ``` + +2. **Disk I/O**: + - Use SSD storage for volumes + - Consider dedicated volume drivers + +3. **Network**: + - Use host networking for better performance + - Adjust nginx worker processes + +## Security Considerations + +1. **CORS Configuration**: + - Set `PHOTON_CORS_ANY=false` in production + - Configure specific origins if needed + +2. **Update API**: + - Only enable `PHOTON_UPDATE_API=true` if needed + - Protect with reverse proxy authentication + +3. **Resource Limits**: + ```yaml + services: + photon: + deploy: + resources: + limits: + cpus: '2.0' + memory: 4G + reservations: + memory: 2G + ``` + +4. **Network Security**: + - Use internal networks for service communication + - Expose only necessary ports + +## Examples and Use Cases + +### 1. Regional Service Provider + +Deploy separate instances for different regions: + +```yaml +services: + photon-eu: + # European countries + environment: + - PHOTON_COUNTRIES=de,fr,it,es,nl,be,at,ch,pl,cz,hu,sk,si,hr,bg,ro,ee,lv,lt + + photon-na: + # North American countries + environment: + - PHOTON_COUNTRIES=us,ca,mx + + photon-asia: + # Asian countries + environment: + - PHOTON_COUNTRIES=jp,kr,cn,in,th,sg,my,id,ph,vn +``` + +### 2. Language-Specific Services + +Deploy instances optimized for specific languages: + +```yaml +services: + photon-german: + environment: + - PHOTON_COUNTRIES=de,at,ch + - PHOTON_LANGUAGES=de,en + + photon-french: + environment: + - PHOTON_COUNTRIES=fr,be,ch,ca + - PHOTON_LANGUAGES=fr,en +``` + +### 3. Development Environment + +Quick setup for development: + +```yaml +services: + photon-dev: + build: . + ports: + - "2322:2322" + environment: + - PHOTON_COUNTRIES=de # Small dataset for testing + - PHOTON_CORS_ANY=true + - JAVA_OPTS=-Xmx1g + volumes: + - photon-dev-data:/photon/data +``` + +This Docker implementation provides a complete solution for multi-country Photon deployments with automatic data management, flexible configuration, and production-ready features. \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..e4c2867f5 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,71 @@ +# Multi-stage build for Photon with multi-country support +FROM gradle:8.14.3-jdk11 AS builder + +# Set working directory +WORKDIR /app + +# Copy source code +COPY . . + +# Build the application (with SSL workaround for restricted environments) +RUN ./gradlew shadowJar --no-daemon --no-build-cache || \ + (echo "Gradle build failed, checking for existing JAR..." && \ + find target -name "photon-*.jar" -type f | head -1 | xargs -I {} cp {} /app/photon.jar) || \ + (echo "No existing JAR found, trying alternative build..." && \ + GRADLE_OPTS="-Dorg.gradle.internal.http.connectionTimeout=60000 -Dorg.gradle.internal.http.socketTimeout=60000" ./gradlew shadowJar --no-daemon --offline --no-build-cache) || \ + (echo "Build failed, but continuing with manual JAR assembly if available" && ls -la target/ || true) + +# Ensure we have a JAR file +RUN find target -name "photon-*.jar" -type f | head -1 | xargs -I {} cp {} /app/photon.jar || \ + (echo "ERROR: No JAR file found after build" && exit 1) + +# Runtime stage +FROM openjdk:11-jre-slim + +# Install required packages for downloading and extracting data +RUN apt-get update && apt-get install -y \ + curl \ + wget \ + bzip2 \ + pbzip2 \ + jq \ + bash \ + && rm -rf /var/lib/apt/lists/* + +# Create photon user and directories +RUN useradd -m -s /bin/bash photon && \ + mkdir -p /photon/data /photon/dumps /photon/logs && \ + chown -R photon:photon /photon + +# Copy the built JAR from builder stage +COPY --from=builder /app/photon.jar /photon/photon.jar + +# Copy scripts +COPY docker/entrypoint.sh /photon/entrypoint.sh +COPY docker/download-data.sh /photon/download-data.sh + +# Make scripts executable +RUN chmod +x /photon/entrypoint.sh /photon/download-data.sh + +# Set working directory +WORKDIR /photon + +# Switch to photon user +USER photon + +# Expose port +EXPOSE 2322 + +# Set default environment variables +ENV PHOTON_COUNTRIES="" +ENV PHOTON_LANGUAGES="en,de,fr,it" +ENV PHOTON_DATA_DIR="/photon/data" +ENV PHOTON_LISTEN_IP="0.0.0.0" +ENV PHOTON_LISTEN_PORT="2322" +ENV PHOTON_CORS_ANY="false" +ENV PHOTON_UPDATE_API="false" +ENV PHOTON_MAX_RESULTS="50" +ENV PHOTON_AUTO_DOWNLOAD="true" + +# Default command +ENTRYPOINT ["/photon/entrypoint.sh"] \ No newline at end of file diff --git a/Dockerfile.prebuilt b/Dockerfile.prebuilt new file mode 100644 index 000000000..31a2b4a82 --- /dev/null +++ b/Dockerfile.prebuilt @@ -0,0 +1,50 @@ +# Simple Dockerfile using pre-built JAR for Photon with multi-country support +FROM openjdk:11-jre-slim + +# Install required packages for downloading and extracting data +RUN apt-get update && apt-get install -y \ + curl \ + wget \ + bzip2 \ + pbzip2 \ + jq \ + bash \ + && rm -rf /var/lib/apt/lists/* + +# Create photon user and directories +RUN useradd -m -s /bin/bash photon && \ + mkdir -p /photon/data /photon/dumps /photon/logs && \ + chown -R photon:photon /photon + +# Copy the pre-built JAR +COPY target/photon-*.jar /photon/photon.jar + +# Copy scripts +COPY docker/entrypoint.sh /photon/entrypoint.sh +COPY docker/download-data.sh /photon/download-data.sh + +# Make scripts executable +RUN chmod +x /photon/entrypoint.sh /photon/download-data.sh + +# Set working directory +WORKDIR /photon + +# Switch to photon user +USER photon + +# Expose port +EXPOSE 2322 + +# Set default environment variables +ENV PHOTON_COUNTRIES="" +ENV PHOTON_LANGUAGES="en,de,fr,it" +ENV PHOTON_DATA_DIR="/photon/data" +ENV PHOTON_LISTEN_IP="0.0.0.0" +ENV PHOTON_LISTEN_PORT="2322" +ENV PHOTON_CORS_ANY="false" +ENV PHOTON_UPDATE_API="false" +ENV PHOTON_MAX_RESULTS="50" +ENV PHOTON_AUTO_DOWNLOAD="true" + +# Default command +ENTRYPOINT ["/photon/entrypoint.sh"] \ No newline at end of file diff --git a/README.md b/README.md index ee2ccfade..5a2614dfb 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,32 @@ the server. See instructions below.* # Installation and Usage +## Docker Multi-Country Deployment + +**NEW**: Photon now supports Docker deployment with multi-country configurations! This is the **easiest way** to get started with Photon. + +### Quick Docker Start + +For a single country (e.g., Germany): +```bash +git clone +cd photon-docker +docker-compose -f docker-compose.simple.yml up -d +``` + +For multi-country deployment: +```bash +docker-compose up -d +``` + +This will automatically: +- Download official per-country Photon JSONL dumps +- Import data for specified countries +- Start geocoding services on different ports +- Provide health monitoring and load balancing + +**See [DOCKER.md](DOCKER.md) for comprehensive Docker deployment documentation.** + ## photon ElasticSearch vs. photon OpenSearch photon was originally built on ElasticSearch. For technical reasons, we are diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml new file mode 100644 index 000000000..f374c71cc --- /dev/null +++ b/docker-compose.dev.yml @@ -0,0 +1,61 @@ +version: '3.8' + +# Development environment with fast startup +# Uses small country datasets for quick testing +services: + photon-dev-small: + build: . + container_name: photon-dev-small + ports: + - "2322:2322" + environment: + # Small country for fast download and import + - PHOTON_COUNTRIES=lu # Luxembourg (very small dataset) + - PHOTON_LANGUAGES=de,fr,en + - PHOTON_CORS_ANY=true + - PHOTON_MAX_RESULTS=50 + - JAVA_OPTS=-Xmx512m # Minimal memory for development + volumes: + - photon-dev-data:/photon/data + - photon-dev-dumps:/photon/dumps + restart: "no" # Don't auto-restart in dev + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:2322/status"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 60s # Shorter start period for small dataset + + photon-dev-medium: + build: . + container_name: photon-dev-medium + ports: + - "2323:2322" + environment: + # Medium-sized country for more comprehensive testing + - PHOTON_COUNTRIES=nl # Netherlands (medium dataset) + - PHOTON_LANGUAGES=nl,en,de + - PHOTON_CORS_ANY=true + - PHOTON_UPDATE_API=true # Enable for testing updates + - PHOTON_MAX_RESULTS=50 + - JAVA_OPTS=-Xmx1g + volumes: + - photon-dev-medium-data:/photon/data + - photon-dev-medium-dumps:/photon/dumps + restart: "no" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:2322/status"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 120s + +volumes: + photon-dev-data: + driver: local + photon-dev-dumps: + driver: local + photon-dev-medium-data: + driver: local + photon-dev-medium-dumps: + driver: local \ No newline at end of file diff --git a/docker-compose.simple.yml b/docker-compose.simple.yml new file mode 100644 index 000000000..5dde4235b --- /dev/null +++ b/docker-compose.simple.yml @@ -0,0 +1,34 @@ +version: '3.8' + +# Simple single-country deployment example +services: + photon: + build: . + container_name: photon-simple + ports: + - "2322:2322" + environment: + # Set to your desired country codes (comma-separated) + # Leave empty for world-wide data + - PHOTON_COUNTRIES=de + - PHOTON_LANGUAGES=de,en + - PHOTON_CORS_ANY=true + - PHOTON_MAX_RESULTS=50 + # Adjust memory based on your needs + - JAVA_OPTS=-Xmx2g + volumes: + - photon-data:/photon/data + - photon-dumps:/photon/dumps + restart: unless-stopped + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:2322/status"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 120s + +volumes: + photon-data: + driver: local + photon-dumps: + driver: local \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..c2655c6d7 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,134 @@ +version: '3.8' + +services: + # Example: World-wide Photon instance + photon-world: + build: . + container_name: photon-world + ports: + - "2322:2322" + environment: + - PHOTON_COUNTRIES= + - PHOTON_LANGUAGES=en,de,fr,it + - PHOTON_CORS_ANY=true + - PHOTON_MAX_RESULTS=50 + - JAVA_OPTS=-Xmx4g + volumes: + - photon-world-data:/photon/data + - photon-world-dumps:/photon/dumps + restart: unless-stopped + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:2322/status"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 300s + + # Example: European countries Photon instance + photon-europe: + build: . + container_name: photon-europe + ports: + - "2323:2322" + environment: + - PHOTON_COUNTRIES=de,fr,it,es,nl,be,at,ch,pl,cz + - PHOTON_LANGUAGES=en,de,fr,it,es,nl,pl,cs + - PHOTON_CORS_ANY=true + - PHOTON_MAX_RESULTS=50 + - JAVA_OPTS=-Xmx2g + volumes: + - photon-europe-data:/photon/data + - photon-europe-dumps:/photon/dumps + restart: unless-stopped + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:2322/status"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 180s + + # Example: North America Photon instance + photon-north-america: + build: . + container_name: photon-north-america + ports: + - "2324:2322" + environment: + - PHOTON_COUNTRIES=us,ca,mx + - PHOTON_LANGUAGES=en,es,fr + - PHOTON_CORS_ANY=true + - PHOTON_MAX_RESULTS=50 + - JAVA_OPTS=-Xmx2g + volumes: + - photon-north-america-data:/photon/data + - photon-north-america-dumps:/photon/dumps + restart: unless-stopped + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:2322/status"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 180s + + # Example: Single country (Germany) instance + photon-germany: + build: . + container_name: photon-germany + ports: + - "2325:2322" + environment: + - PHOTON_COUNTRIES=de + - PHOTON_LANGUAGES=de,en + - PHOTON_CORS_ANY=true + - PHOTON_MAX_RESULTS=50 + - PHOTON_UPDATE_API=true + - JAVA_OPTS=-Xmx1g + volumes: + - photon-germany-data:/photon/data + - photon-germany-dumps:/photon/dumps + restart: unless-stopped + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:2322/status"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 120s + + # Nginx reverse proxy for load balancing (optional) + nginx: + image: nginx:alpine + container_name: photon-nginx + ports: + - "80:80" + volumes: + - ./docker/nginx.conf:/etc/nginx/nginx.conf:ro + depends_on: + - photon-world + - photon-europe + - photon-north-america + - photon-germany + restart: unless-stopped + profiles: + - with-proxy + +volumes: + photon-world-data: + driver: local + photon-world-dumps: + driver: local + photon-europe-data: + driver: local + photon-europe-dumps: + driver: local + photon-north-america-data: + driver: local + photon-north-america-dumps: + driver: local + photon-germany-data: + driver: local + photon-germany-dumps: + driver: local + +networks: + default: + name: photon-network \ No newline at end of file diff --git a/docker/download-data.sh b/docker/download-data.sh new file mode 100755 index 000000000..3544b344d --- /dev/null +++ b/docker/download-data.sh @@ -0,0 +1,184 @@ +#!/bin/bash +set -e + +# Configuration +BASE_URL_WORLDWIDE="https://download1.graphhopper.com/public/experimental" +BASE_URL_EXTRACTS="https://download1.graphhopper.com/public/experimental/extracts" +DUMP_DIR="/photon/dumps" +MAX_RETRIES=3 +RETRY_DELAY=5 + +# Logging function +log() { + echo "[$(date '+%Y-%m-%d %H:%M:%S')] DOWNLOAD: $1" +} + +# Function to download file with retries +download_with_retry() { + local url="$1" + local output="$2" + local retries=0 + + while [ $retries -lt $MAX_RETRIES ]; do + log "Downloading ${url} (attempt $((retries + 1))/${MAX_RETRIES})" + + if wget --progress=dot:giga -T 60 -t 1 "${url}" -O "${output}"; then + log "Successfully downloaded ${output}" + return 0 + else + retries=$((retries + 1)) + if [ $retries -lt $MAX_RETRIES ]; then + log "Download failed, retrying in ${RETRY_DELAY} seconds..." + sleep $RETRY_DELAY + fi + fi + done + + log "ERROR: Failed to download ${url} after ${MAX_RETRIES} attempts" + return 1 +} + +# Function to get file size +get_file_size() { + local url="$1" + wget --spider --server-response "$url" 2>&1 | grep "Content-Length" | tail -1 | awk '{print $2}' || echo "0" +} + +# Function to format bytes +format_bytes() { + local bytes=$1 + if [ $bytes -gt 1073741824 ]; then + echo "$(( bytes / 1073741824 ))GB" + elif [ $bytes -gt 1048576 ]; then + echo "$(( bytes / 1048576 ))MB" + elif [ $bytes -gt 1024 ]; then + echo "$(( bytes / 1024 ))KB" + else + echo "${bytes}B" + fi +} + +# Create dump directory +mkdir -p "${DUMP_DIR}" + +# Parse countries argument +COUNTRIES="$1" + +if [ -z "${COUNTRIES}" ]; then + # Download world-wide data + log "Downloading world-wide Photon data..." + + WORLD_URL="${BASE_URL_WORLDWIDE}/photon-db-latest.bz2" + OUTPUT_FILE="${DUMP_DIR}/photon-db-latest.bz2" + + # Check if file exists and get size + if wget --spider "${WORLD_URL}" 2>/dev/null; then + file_size=$(get_file_size "${WORLD_URL}") + log "World-wide database size: $(format_bytes $file_size)" + + download_with_retry "${WORLD_URL}" "${OUTPUT_FILE}" + + log "World-wide data download completed!" + log "Downloaded file: ${OUTPUT_FILE}" + else + log "ERROR: World-wide database not available at ${WORLD_URL}" + exit 1 + fi +else + # Download country-specific data + log "Downloading country-specific data for: ${COUNTRIES}" + + # Convert comma-separated countries to array + IFS=',' read -ra COUNTRY_ARRAY <<< "${COUNTRIES}" + + # Create a temporary file for merging country data + TEMP_MERGED="${DUMP_DIR}/merged.jsonl" + > "${TEMP_MERGED}" # Clear file + + downloaded_countries=() + failed_countries=() + + for country in "${COUNTRY_ARRAY[@]}"; do + # Trim whitespace + country=$(echo "${country}" | tr -d ' ') + country_lower=$(echo "${country}" | tr '[:upper:]' '[:lower:]') + + log "Processing country: ${country}" + + # Try different naming conventions for country files + potential_files=( + "${country_lower}.bz2" + "${country}.bz2" + "${country_lower}.jsonl.bz2" + "${country}.jsonl.bz2" + ) + + downloaded=false + for filename in "${potential_files[@]}"; do + country_url="${BASE_URL_EXTRACTS}/${filename}" + + log "Checking ${country_url}..." + if wget --spider "${country_url}" 2>/dev/null; then + log "Found ${filename} for ${country}" + + file_size=$(get_file_size "${country_url}") + log "File size: $(format_bytes $file_size)" + + temp_file="${DUMP_DIR}/${country}_${filename}" + + if download_with_retry "${country_url}" "${temp_file}"; then + # Extract and append to merged file + log "Extracting ${filename}..." + if [[ "${filename}" == *.bz2 ]]; then + bzip2 -dc "${temp_file}" >> "${TEMP_MERGED}" + else + cat "${temp_file}" >> "${TEMP_MERGED}" + fi + + # Clean up temporary file + rm -f "${temp_file}" + + downloaded_countries+=("${country}") + downloaded=true + break + else + log "Failed to download ${filename}" + rm -f "${temp_file}" + fi + fi + done + + if [ "$downloaded" = false ]; then + log "WARNING: Could not find data for country: ${country}" + failed_countries+=("${country}") + fi + done + + # Check if we downloaded any countries + if [ ${#downloaded_countries[@]} -eq 0 ]; then + log "ERROR: No country data could be downloaded!" + rm -f "${TEMP_MERGED}" + exit 1 + fi + + # Move merged file to final location + mv "${TEMP_MERGED}" "${DUMP_DIR}/photon-db-latest.jsonl" + + log "Country-specific data download completed!" + log "Successfully downloaded: ${downloaded_countries[*]}" + if [ ${#failed_countries[@]} -gt 0 ]; then + log "Failed to download: ${failed_countries[*]}" + fi + log "Merged data file: ${DUMP_DIR}/photon-db-latest.jsonl" +fi + +# Show final file info +if [ -f "${DUMP_DIR}/photon-db-latest.bz2" ]; then + size=$(stat -c%s "${DUMP_DIR}/photon-db-latest.bz2") + log "Final compressed database size: $(format_bytes $size)" +elif [ -f "${DUMP_DIR}/photon-db-latest.jsonl" ]; then + size=$(stat -c%s "${DUMP_DIR}/photon-db-latest.jsonl") + log "Final JSONL database size: $(format_bytes $size)" +fi + +log "Data download process completed successfully!" \ No newline at end of file diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100755 index 000000000..c9dd5b8d1 --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1,102 @@ +#!/bin/bash +set -e + +# Default values +PHOTON_JAR="/photon/photon.jar" +DATA_DIR="${PHOTON_DATA_DIR:-/photon/data}" +DUMP_DIR="/photon/dumps" + +# Logging function +log() { + echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" +} + +log "Starting Photon Docker container..." +log "Configuration:" +log " Countries: ${PHOTON_COUNTRIES:-'all'}" +log " Languages: ${PHOTON_LANGUAGES}" +log " Data directory: ${DATA_DIR}" +log " Listen IP: ${PHOTON_LISTEN_IP}" +log " Listen port: ${PHOTON_LISTEN_PORT}" +log " Auto-download: ${PHOTON_AUTO_DOWNLOAD}" + +# Create necessary directories +mkdir -p "${DATA_DIR}" "${DUMP_DIR}" + +# Check if data needs to be downloaded/imported +if [ "${PHOTON_AUTO_DOWNLOAD}" = "true" ] && [ ! -f "${DATA_DIR}/nodes" ]; then + log "No existing data found, downloading and importing..." + + if [ -n "${PHOTON_COUNTRIES}" ]; then + log "Downloading country-specific data for: ${PHOTON_COUNTRIES}" + /photon/download-data.sh "${PHOTON_COUNTRIES}" + else + log "Downloading world-wide data..." + /photon/download-data.sh + fi + + # Import the downloaded data + log "Importing data into Photon..." + if [ -f "${DUMP_DIR}/photon-db-latest.bz2" ]; then + log "Importing from compressed dump..." + bzip2 -dc "${DUMP_DIR}/photon-db-latest.bz2" | java -jar "${PHOTON_JAR}" \ + -nominatim-import \ + -import-file - \ + -data-dir "${DATA_DIR}" \ + -languages "${PHOTON_LANGUAGES}" + elif [ -f "${DUMP_DIR}/photon-db-latest.jsonl" ]; then + log "Importing from JSONL dump..." + java -jar "${PHOTON_JAR}" \ + -nominatim-import \ + -import-file "${DUMP_DIR}/photon-db-latest.jsonl" \ + -data-dir "${DATA_DIR}" \ + -languages "${PHOTON_LANGUAGES}" + else + log "ERROR: No dump file found for import!" + exit 1 + fi + + log "Data import completed successfully!" +fi + +# Build the Photon server command +JAVA_OPTS="${JAVA_OPTS:-}" +PHOTON_ARGS="" + +# Add basic server arguments +PHOTON_ARGS="${PHOTON_ARGS} -data-dir ${DATA_DIR}" +PHOTON_ARGS="${PHOTON_ARGS} -listen-ip ${PHOTON_LISTEN_IP}" +PHOTON_ARGS="${PHOTON_ARGS} -listen-port ${PHOTON_LISTEN_PORT}" +PHOTON_ARGS="${PHOTON_ARGS} -languages ${PHOTON_LANGUAGES}" + +# Add optional arguments +if [ "${PHOTON_CORS_ANY}" = "true" ]; then + PHOTON_ARGS="${PHOTON_ARGS} -cors-any" +fi + +if [ "${PHOTON_UPDATE_API}" = "true" ]; then + PHOTON_ARGS="${PHOTON_ARGS} -enable-update-api" +fi + +if [ -n "${PHOTON_MAX_RESULTS}" ]; then + PHOTON_ARGS="${PHOTON_ARGS} -max-results ${PHOTON_MAX_RESULTS}" +fi + +if [ -n "${PHOTON_SYNONYM_FILE}" ] && [ -f "${PHOTON_SYNONYM_FILE}" ]; then + PHOTON_ARGS="${PHOTON_ARGS} -synonym-file ${PHOTON_SYNONYM_FILE}" +fi + +if [ -n "${PHOTON_EXTRA_TAGS}" ]; then + PHOTON_ARGS="${PHOTON_ARGS} -extra-tags ${PHOTON_EXTRA_TAGS}" +fi + +# Handle custom arguments +if [ -n "${PHOTON_CUSTOM_ARGS}" ]; then + PHOTON_ARGS="${PHOTON_ARGS} ${PHOTON_CUSTOM_ARGS}" +fi + +log "Starting Photon server..." +log "Command: java ${JAVA_OPTS} -jar ${PHOTON_JAR} ${PHOTON_ARGS}" + +# Start Photon server +exec java ${JAVA_OPTS} -jar "${PHOTON_JAR}" ${PHOTON_ARGS} \ No newline at end of file diff --git a/docker/nginx.conf b/docker/nginx.conf new file mode 100644 index 000000000..9caa3a35d --- /dev/null +++ b/docker/nginx.conf @@ -0,0 +1,111 @@ +events { + worker_connections 1024; +} + +http { + upstream photon_world { + server photon-world:2322; + } + + upstream photon_europe { + server photon-europe:2322; + } + + upstream photon_north_america { + server photon-north-america:2322; + } + + upstream photon_germany { + server photon-germany:2322; + } + + # Logging + access_log /var/log/nginx/access.log; + error_log /var/log/nginx/error.log; + + # Default server for world-wide queries + server { + listen 80 default_server; + server_name _; + + location / { + proxy_pass http://photon_world; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # CORS headers + add_header Access-Control-Allow-Origin *; + add_header Access-Control-Allow-Methods "GET, POST, OPTIONS"; + add_header Access-Control-Allow-Headers "Origin, X-Requested-With, Content-Type, Accept"; + + if ($request_method = 'OPTIONS') { + return 204; + } + } + + location /health { + access_log off; + return 200 "OK\n"; + add_header Content-Type text/plain; + } + } + + # European queries + server { + listen 80; + server_name europe.* eu.*; + + location / { + proxy_pass http://photon_europe; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # CORS headers + add_header Access-Control-Allow-Origin *; + add_header Access-Control-Allow-Methods "GET, POST, OPTIONS"; + add_header Access-Control-Allow-Headers "Origin, X-Requested-With, Content-Type, Accept"; + } + } + + # North American queries + server { + listen 80; + server_name na.* america.* us.* ca.*; + + location / { + proxy_pass http://photon_north_america; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # CORS headers + add_header Access-Control-Allow-Origin *; + add_header Access-Control-Allow-Methods "GET, POST, OPTIONS"; + add_header Access-Control-Allow-Headers "Origin, X-Requested-With, Content-Type, Accept"; + } + } + + # German queries + server { + listen 80; + server_name de.* germany.*; + + location / { + proxy_pass http://photon_germany; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # CORS headers + add_header Access-Control-Allow-Origin *; + add_header Access-Control-Allow-Methods "GET, POST, OPTIONS"; + add_header Access-Control-Allow-Headers "Origin, X-Requested-With, Content-Type, Accept"; + } + } +} \ No newline at end of file diff --git a/docker/test-deployment.sh b/docker/test-deployment.sh new file mode 100755 index 000000000..46b39f780 --- /dev/null +++ b/docker/test-deployment.sh @@ -0,0 +1,169 @@ +#!/bin/bash +set -e + +# Test script for Docker multi-country deployment +# This script validates the Docker setup without requiring actual deployment + +echo "=== Photon Docker Multi-Country Deployment Test ===" +echo + +# Test 1: Check Docker files exist +echo "✓ Checking Docker files..." +required_files=( + "Dockerfile" + "docker-compose.yml" + "docker-compose.simple.yml" + "docker/entrypoint.sh" + "docker/download-data.sh" + "docker/nginx.conf" + ".dockerignore" + "DOCKER.md" +) + +for file in "${required_files[@]}"; do + if [ -f "$file" ]; then + echo " ✓ $file exists" + else + echo " ✗ $file missing" + exit 1 + fi +done + +# Test 2: Basic Dockerfile validation +echo +echo "✓ Validating Dockerfile structure..." +if grep -q "FROM.*jdk" Dockerfile && grep -q "COPY.*jar" Dockerfile; then + echo " ✓ Dockerfile structure looks correct" +else + echo " ✗ Dockerfile structure appears incorrect" + exit 1 +fi + +# Test 3: Basic docker-compose validation +echo +echo "✓ Validating docker-compose files structure..." +compose_files=("docker-compose.yml" "docker-compose.simple.yml") +for compose_file in "${compose_files[@]}"; do + if grep -q "version:" "$compose_file" && grep -q "services:" "$compose_file"; then + echo " ✓ $compose_file structure looks correct" + else + echo " ✗ $compose_file structure appears incorrect" + exit 1 + fi +done + +# Test 4: Check script permissions and syntax +echo +echo "✓ Checking scripts..." +scripts=("docker/entrypoint.sh" "docker/download-data.sh") +for script in "${scripts[@]}"; do + if [ -x "$script" ]; then + echo " ✓ $script is executable" + else + echo " ℹ Making $script executable" + chmod +x "$script" + fi + + # Basic bash syntax check + if bash -n "$script"; then + echo " ✓ $script syntax is valid" + else + echo " ✗ $script has syntax errors" + exit 1 + fi +done + +# Test 5: Validate environment variables in docker-compose +echo +echo "✓ Validating environment variables..." +if grep -q "PHOTON_COUNTRIES" docker-compose.yml; then + echo " ✓ PHOTON_COUNTRIES variable found" +else + echo " ✗ PHOTON_COUNTRIES variable missing" + exit 1 +fi + +if grep -q "PHOTON_LANGUAGES" docker-compose.yml; then + echo " ✓ PHOTON_LANGUAGES variable found" +else + echo " ✗ PHOTON_LANGUAGES variable missing" + exit 1 +fi + +# Test 6: Check for health checks +echo +echo "✓ Checking health checks configuration..." +if grep -q "healthcheck:" docker-compose.yml; then + echo " ✓ Health checks configured" +else + echo " ✗ Health checks missing" + exit 1 +fi + +# Test 7: Validate volume configurations +echo +echo "✓ Checking volume configurations..." +if grep -q "volumes:" docker-compose.yml; then + echo " ✓ Volumes configured for data persistence" +else + echo " ✗ Volume configuration missing" + exit 1 +fi + +# Test 8: Check if ports are properly configured +echo +echo "✓ Validating port configurations..." +expected_ports=("2322" "2323" "2324" "2325") +for port in "${expected_ports[@]}"; do + if grep -q "$port:2322" docker-compose.yml; then + echo " ✓ Port $port is configured" + else + echo " ℹ Port $port not found (this may be expected)" + fi +done + +# Test 9: Validate country codes in examples +echo +echo "✓ Checking country code examples..." +valid_country_codes=("de" "fr" "it" "es" "us" "ca" "mx" "nl" "be") +found_valid_codes=false +for code in "${valid_country_codes[@]}"; do + if grep -q "$code" docker-compose.yml; then + found_valid_codes=true + break + fi +done + +if [ "$found_valid_codes" = true ]; then + echo " ✓ Valid country codes found in examples" +else + echo " ✗ No valid country codes found in examples" + exit 1 +fi + +# Test 10: Check CORS and security configurations +echo +echo "✓ Checking security configurations..." +if grep -q "PHOTON_CORS_ANY" docker-compose.yml; then + echo " ✓ CORS configuration available" +else + echo " ✗ CORS configuration missing" + exit 1 +fi + +echo +echo "=== All Tests Passed! ===" +echo +echo "Docker multi-country deployment is ready to use." +echo +echo "Next steps:" +echo "1. For single country: docker-compose -f docker-compose.simple.yml up -d" +echo "2. For multi-country: docker-compose up -d" +echo "3. For custom setup: See DOCKER.md for detailed configuration" +echo +echo "Services will be available at:" +echo "- Simple deployment: http://localhost:2322" +echo "- Multi-country world: http://localhost:2322" +echo "- Multi-country Europe: http://localhost:2323" +echo "- Multi-country North America: http://localhost:2324" +echo "- Multi-country Germany: http://localhost:2325" \ No newline at end of file