feat: generate config for upstreams with keepalive enabled

Dockerfile

@@ -4,7 +4,7 @@ LABEL description="Single caching container for caching game content at LAN part
 LABEL maintainer="LanCache.Net Team <team@lancache.net>"
 
 RUN	apt-get update							;\
-	apt-get install -y jq git				;
+	apt-get install -y jq git dnsutils		;
 
 ENV GENERICCACHE_VERSION=2 \
     CACHE_MODE=monolithic \

overlay/etc/nginx/conf.d/35_upstream_maps.conf

@@ -0,0 +1,4 @@
+# Default passthrough map - overwritten when ENABLE_UPSTREAM_KEEPALIVE=true
+map $http_host $upstream_name {
+    default $host;
+}

overlay/etc/nginx/sites-available/upstream.conf.d/30_primary_proxy.conf

@@ -1,7 +1,11 @@
   # Proxy all requests to upstream
   location / {
-    # Simple proxy the request
-    proxy_pass http://$host$request_uri;
+    # Enable HTTP/1.1 and keepalive for supported upstreams
+    proxy_http_version 1.1;
+    proxy_set_header Connection "";
+
+    # Proxy using upstream pools (falls back to $host if not mapped)
+    proxy_pass http://$upstream_name$request_uri;
 
     # Catch the errors to process the redirects
     proxy_intercept_errors on;

overlay/etc/supervisor/conf.d/upstream_refresh.conf

@@ -0,0 +1,6 @@
+[program:upstream_refresh]
+command=/scripts/refresh_upstreams.sh
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0

overlay/hooks/entrypoint-pre.d/16_generate_upstream_keepalive.sh

@@ -0,0 +1,159 @@
+#!/bin/bash
+set -e
+
+# Generates upstream pools with keepalive for concrete (non-wildcard) domains
+# from cache_domains.json
+
+if [[ "${ENABLE_UPSTREAM_KEEPALIVE}" != "true" ]]; then
+    echo "Upstream keepalive disabled (set ENABLE_UPSTREAM_KEEPALIVE=true to enable)"
+    exit 0
+fi
+
+# Normalize UPSTREAM_DNS (allow semicolons like lancache-dns)
+UPSTREAM_DNS="$(echo -n "${UPSTREAM_DNS}" | sed 's/[;]/ /g')"
+
+# DNS is resolved at generation time using UPSTREAM_DNS.
+# The refresh_upstreams.sh script re-runs this periodically to keep IPs current.
+#
+# Future: nginx 1.27.3+ supports dynamic DNS via 'resolve' parameter (previously commercial-only).
+# This would replace generation-time resolution and the refresh script. Example:
+#
+#   resolver $UPSTREAM_DNS ipv6=off;
+#   upstream example_com {
+#       zone lancache_upstreams 1m;
+#       server example.com resolve;
+#       keepalive 16;
+#       keepalive_timeout 5m;
+#   }
+#
+# See: https://nginx.org/en/docs/http/ngx_http_upstream_module.html#resolve
+
+IFS=' '
+cd /data/cachedomains
+
+TEMP_PATH=$(mktemp -d)
+POOLS_TMP_FILE="${TEMP_PATH}/pools.conf"
+POOLS_FILE="/etc/nginx/conf.d/40_upstream_pools.conf"
+MAPS_TMP_FILE="${TEMP_PATH}/maps.conf"
+MAPS_FILE="/etc/nginx/conf.d/35_upstream_maps.conf"
+
+echo "Generating upstream keepalive pools from cache_domains.json"
+
+# Initialize pools file
+echo "# Auto-generated upstream pools with keepalive" > "$POOLS_TMP_FILE"
+echo "# Generated from cache_domains.json at $(date)" >> "$POOLS_TMP_FILE"
+echo "" >> "$POOLS_TMP_FILE"
+
+# Initialize maps file
+echo "# Map hostnames to upstream pools for keepalive routing" > "$MAPS_TMP_FILE"
+echo "" >> "$MAPS_TMP_FILE"
+echo "map \$http_host \$upstream_name_default {" >> "$MAPS_TMP_FILE"
+echo "    hostnames;" >> "$MAPS_TMP_FILE"
+echo "    default \$host;  # Fallback to direct proxy for unmapped domains" >> "$MAPS_TMP_FILE"
+
+STEAM_UPSTREAM_NAME=""
+
+while read CACHE_ENTRY; do
+    SERVICE_NAME=$(jq -r ".cache_domains[$CACHE_ENTRY].name" cache_domains.json)
+
+    echo "Processing service: ${SERVICE_NAME}"
+
+    # Loop through domain files for this service
+    while read CACHEHOSTS_FILEID; do
+        # Get the domain file name
+        CACHEHOSTS_FILENAME=$(jq -r ".cache_domains[$CACHE_ENTRY].domain_files[$CACHEHOSTS_FILEID]" cache_domains.json)
+
+        if [ ! -f "${CACHEHOSTS_FILENAME}" ]; then
+            echo "  Warning: Domain file not found: ${CACHEHOSTS_FILENAME}"
+            continue
+        fi
+
+        while read DOMAIN; do
+            # Skip empty lines and whitespace
+            DOMAIN=$(echo "$DOMAIN" | tr -d '[:space:]')
+            if [ -z "$DOMAIN" ]; then
+                continue
+            fi
+
+            # Skip comments
+            if [[ "$DOMAIN" =~ ^# ]]; then
+                continue
+            fi
+
+            # Skip wildcards - can't create upstream for *.example.com
+            if [[ "$DOMAIN" =~ \* ]]; then
+                echo "  Skipping wildcard: $DOMAIN"
+                continue
+            fi
+
+            # Resolve domain using external DNS to bypass local lancache DNS overrides
+            # This prevents nginx from looping back to itself
+            RESOLVED_IP=$(dig +short +time=2 +tries=1 "$DOMAIN" @${UPSTREAM_DNS%% *} 2>/dev/null | grep -E '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$' | head -1)
+            if [ -z "$RESOLVED_IP" ]; then
+                echo "  Skipping unresolvable domain: $DOMAIN"
+                continue
+            fi
+
+            echo "  Adding upstream for: $DOMAIN -> $RESOLVED_IP"
+
+            # Sanitize domain name for use as upstream name
+            # Replace dots and dashes with underscores
+            UPSTREAM_NAME=$(echo "$DOMAIN" | sed 's/[.-]/_/g')
+
+            # Generate upstream block with resolved IP
+            cat >> "$POOLS_TMP_FILE" <<EOF
+upstream ${UPSTREAM_NAME} {
+    server ${RESOLVED_IP};  # $DOMAIN
+    keepalive 16;
+    keepalive_timeout 5m;
+}
+
+EOF
+
+            # Add mapping entry
+            echo "    ${DOMAIN} ${UPSTREAM_NAME};" >> "$MAPS_TMP_FILE"
+
+            # Track steam upstream for user-agent based routing
+            if [[ "$SERVICE_NAME" == "steam" && -z "$STEAM_UPSTREAM_NAME" ]]; then
+                STEAM_UPSTREAM_NAME="$UPSTREAM_NAME"
+            fi
+        done < "${CACHEHOSTS_FILENAME}"
+    done < <(jq -r ".cache_domains[$CACHE_ENTRY].domain_files | to_entries[] | .key" cache_domains.json)
+done < <(jq -r '.cache_domains | to_entries[] | .key' cache_domains.json)
+
+# Close the hostname map block
+echo "}" >> "$MAPS_TMP_FILE"
+
+# Add user-agent based map for Steam client detection
+# Steam uses a trigger domain for DNS but actual requests come from various *.steamcontent.com hosts.
+# Instead of hardcoding the wildcard (which bypasses cache-domains logic), we detect the
+# Valve/Steam user-agent and route those requests to the steam upstream pool.
+echo "" >> "$MAPS_TMP_FILE"
+echo "# Steam user-agent detection - routes Steam client traffic to steam upstream pool" >> "$MAPS_TMP_FILE"
+echo "map \$http_user_agent \$upstream_name {" >> "$MAPS_TMP_FILE"
+echo "    default \$upstream_name_default;" >> "$MAPS_TMP_FILE"
+if [ -n "$STEAM_UPSTREAM_NAME" ]; then
+    echo "    ~Valve\\/Steam\\ HTTP\\ Client\\ 1\\.0 ${STEAM_UPSTREAM_NAME};" >> "$MAPS_TMP_FILE"
+fi
+echo "}" >> "$MAPS_TMP_FILE"
+
+# Copy to final locations
+cp "$POOLS_TMP_FILE" $POOLS_FILE
+cp "$MAPS_TMP_FILE" $MAPS_FILE
+
+# Validate final configuration after copying
+echo "Validating final nginx configuration..."
+if nginx -t 2>/dev/null; then
+    echo "✓ Nginx configuration is valid"
+    echo "Output files:"
+    echo "  - $POOLS_FILE"
+    echo "  - $MAPS_FILE"
+else
+    echo "ERROR: Generated keepalive configuration caused nginx validation to fail!"
+    echo "Rolling back generated files to allow nginx to start normally..."
+    rm -f $POOLS_FILE
+    rm -f $MAPS_FILE
+fi
+
+# Cleanup
+rm -rf "$TEMP_PATH"

overlay/scripts/refresh_upstreams.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+set -e
+
+if [[ "${ENABLE_UPSTREAM_KEEPALIVE}" != "true" ]]; then
+    exit 0
+fi
+
+REFRESH_INTERVAL=${UPSTREAM_REFRESH_INTERVAL:-1h}
+
+if [[ "$REFRESH_INTERVAL" == "0" ]]; then
+    echo "Upstream refresh disabled (UPSTREAM_REFRESH_INTERVAL=0)"
+    exit 0
+fi
+
+echo "Starting upstream refresh loop (interval: $REFRESH_INTERVAL)"
+
+while true; do
+    sleep $REFRESH_INTERVAL
+
+    echo "[$(date)] Refreshing upstream pools..."
+
+    if /hooks/entrypoint-pre.d/16_generate_upstream_keepalive.sh > /dev/null 2>&1; then
+        nginx -s reload
+        echo "[$(date)] Upstream pools refreshed and nginx reloaded"
+    else
+        echo "[$(date)] Upstream generation failed, skipping reload"
+    fi
+done