feat(l1): docker compose parallel snapsync run in loop

.gitignore

@@ -18,6 +18,7 @@ tooling/ef_tests/state/runner_v2/success_report.txt
 tooling/reorgs/data
 
 tooling/sync/logs/
+tooling/sync/multisync_logs/
 
 # Repos checked out by make target
 /hive/

tooling/sync/Makefile

@@ -2,7 +2,9 @@
 flamegraph-branch flamegraph-inner flamegraph-mainnet flamegraph-sepolia flamegraph-holesky \
 flamegraph-hoodi start-lighthouse start-ethrex backup-db start-mainnet-metrics-docker \
 start-sepolia-metrics-docker start-holesky-metrics-docker start-hoodi-metrics-docker \
-start-metrics-docker tail-syncing-logs tail-metrics-logs copy_flamegraph import-with-metrics
+start-metrics-docker tail-syncing-logs tail-metrics-logs copy_flamegraph import-with-metrics \
+multisync-up multisync-down multisync-clean multisync-logs multisync-status \
+multisync-restart multisync-monitor multisync-run multisync-loop
 
 ETHREX_DIR ?= "../.."
 EVM ?= levm
@@ -220,3 +222,74 @@ server-sync:
 	sleep 0.2
 
 	tmux new-window -t sync:2 -n ethrex "cd ../../metrics && docker stop metrics-ethereum-metrics-exporter-1 || true && docker compose -f docker-compose-metrics.yaml -f docker-compose-metrics-l1.overrides.yaml up -d && cd .. && ulimit -n 1000000 && rm -rf ~/.local/share/ethrex && RUST_LOG=info,ethrex_p2p::sync=debug $(if $(DEBUG_ASSERT),RUSTFLAGS='-C debug-assertions=yes') $(if $(HEALING),SKIP_START_SNAP_SYNC=1) cargo run --release --bin ethrex --features rocksdb --  --http.addr 0.0.0.0 --metrics --metrics.port 3701 --network $(SERVER_SYNC_NETWORK) $(if $(MEMORY),--datadir memory) --authrpc.jwtsecret ~/secrets/jwt.hex $(if $(or $(FULL_SYNC),$(HEALING)),--syncmode full)  2>&1 | tee $(LOGS_FILE)"
+
+# ==============================================================================
+# Docker Compose Multi-Network Snapsync
+# ==============================================================================
+
+MULTISYNC_COMPOSE = docker compose -f docker-compose.multisync.yaml
+MULTISYNC_NETWORKS ?= hoodi,sepolia,mainnet
+MULTISYNC_SERVICES = $(shell echo $(MULTISYNC_NETWORKS) | awk -F, '{for(i=1;i<=NF;i++) if($$i) printf "setup-jwt-%s ethrex-%s consensus-%s ", $$i, $$i, $$i}' | sed 's/ $$//')
+
+multisync-up: ## Start all networks specified in MULTISYNC_NETWORKS via Docker Compose.
+	$(MULTISYNC_COMPOSE) up -d $(MULTISYNC_SERVICES)
+
+multisync-down: ## Stop and remove all snapsync containers.
+	$(MULTISYNC_COMPOSE) down
+
+multisync-clean: ## Stop, remove containers AND volumes (full reset).
+	$(MULTISYNC_COMPOSE) down -v
+
+multisync-logs: ## Tail logs from all networks.
+	$(MULTISYNC_COMPOSE) logs -f
+
+multisync-logs-%: ## Tail logs for a specific network (e.g., multisync-logs-hoodi).
+	$(MULTISYNC_COMPOSE) logs -f ethrex-$* consensus-$*
+
+multisync-logs-ethrex-%: ## Tail only ethrex logs for a network (e.g., multisync-logs-ethrex-hoodi).
+	$(MULTISYNC_COMPOSE) logs -f ethrex-$*
+
+multisync-logs-consensus-%: ## Tail only consensus logs for a network (e.g., multisync-logs-consensus-hoodi).
+	$(MULTISYNC_COMPOSE) logs -f consensus-$*
+
+multisync-restart: ## Restart the cycle (clean volumes + start fresh).
+	$(MULTISYNC_COMPOSE) down -v
+	$(MULTISYNC_COMPOSE) up -d $(MULTISYNC_SERVICES)
+
+multisync-monitor: ## Monitor all networks (one-shot, exits on completion).
+	python3 docker_monitor.py --networks $(MULTISYNC_NETWORKS) --exit-on-success
+
+multisync-run: ## Full run: start + monitor (one-shot, exits on completion).
+	$(MULTISYNC_COMPOSE) up -d $(MULTISYNC_SERVICES)
+	@echo "Waiting 10s for containers to start..."
+	@sleep 10
+	python3 docker_monitor.py --networks $(MULTISYNC_NETWORKS) --exit-on-success
+
+multisync-loop: ## Continuous loop: sync all networks, restart on success, repeat forever.
+	$(MULTISYNC_COMPOSE) up -d $(MULTISYNC_SERVICES)
+	@echo "Waiting 10s for containers to start..."
+	@sleep 10
+	python3 docker_monitor.py --networks $(MULTISYNC_NETWORKS) --compose-file docker-compose.multisync.yaml --compose-dir $(CURDIR)
+
+multisync-history: ## View the run history log.
+	@if [ -f multisync_logs/run_history.log ]; then \
+		cat multisync_logs/run_history.log; \
+	else \
+		echo "No run history found. Run 'make multisync-loop' first."; \
+	fi
+
+multisync-list-logs: ## List all saved run logs.
+	@if [ -d multisync_logs ]; then \
+		echo "=== Saved Run Logs ===" && \
+		ls -la multisync_logs/ && \
+		echo "" && \
+		for dir in multisync_logs/run_*/; do \
+			if [ -d "$$dir" ]; then \
+				echo "$$dir:"; \
+				ls "$$dir"; \
+				echo ""; \
+			fi; \
+		done; \
+	else \
+		echo "No logs directory found."; \
+	fi

tooling/sync/docker-compose.multisync.yaml

@@ -0,0 +1,225 @@
+# Multi-network parallel snapsync
+# Usage:
+#   docker compose -f docker-compose.multisync.yaml up -d
+#   docker compose -f docker-compose.multisync.yaml logs -f
+#   docker compose -f docker-compose.multisync.yaml down -v
+#
+# Each network runs in isolation with its own volumes.
+# Host ports are mapped for external RPC access:
+#   hoodi:   localhost:8545
+#   sepolia: localhost:8546
+#   mainnet: localhost:8547
+
+x-ethrex-common: &ethrex-common
+  image: "ghcr.io/lambdaclass/ethrex:main"
+  pull_policy: always
+  ulimits:
+    nofile: 1000000
+  restart: unless-stopped
+
+x-consensus-common: &consensus-common
+  image: sigp/lighthouse:v8.0.1
+  restart: unless-stopped
+
+services:
+  # =============================================================================
+  # HOODI
+  # =============================================================================
+  setup-jwt-hoodi:
+    image: alpine
+    volumes:
+      - secrets-hoodi:/secrets
+    command: sh -c 'apk add openssl && openssl rand -hex 32 | tr -d "\n" | tee /secrets/jwt.hex'
+
+  consensus-hoodi:
+    <<: *consensus-common
+    container_name: consensus-hoodi
+    volumes:
+      - secrets-hoodi:/secrets
+      - consensus-hoodi:/root/.lighthouse
+    command: >
+      lighthouse bn
+      --network hoodi
+      --http --http-address 0.0.0.0
+      --execution-endpoint http://ethrex-hoodi:8551
+      --execution-jwt /secrets/jwt.hex
+      --checkpoint-sync-url https://hoodi-checkpoint-sync.stakely.io/
+      --checkpoint-sync-url-timeout 600
+      --purge-db
+    depends_on:
+      setup-jwt-hoodi:
+        condition: service_completed_successfully
+
+  ethrex-hoodi:
+    <<: *ethrex-common
+    container_name: ethrex-hoodi
+    ports:
+      - "8545:8545"  # RPC
+    volumes:
+      - secrets-hoodi:/secrets
+      - ethrex-hoodi:/data
+    command: >
+      --http.addr 0.0.0.0
+      --network hoodi
+      --authrpc.addr 0.0.0.0
+      --authrpc.jwtsecret /secrets/jwt.hex
+      --syncmode snap
+      --datadir /data
+    depends_on:
+      setup-jwt-hoodi:
+        condition: service_completed_successfully
+
+  # =============================================================================
+  # SEPOLIA
+  # =============================================================================
+  setup-jwt-sepolia:
+    image: alpine
+    volumes:
+      - secrets-sepolia:/secrets
+    command: sh -c 'apk add openssl && openssl rand -hex 32 | tr -d "\n" | tee /secrets/jwt.hex'
+
+  consensus-sepolia:
+    <<: *consensus-common
+    container_name: consensus-sepolia
+    volumes:
+      - secrets-sepolia:/secrets
+      - consensus-sepolia:/root/.lighthouse
+    command: >
+      lighthouse bn
+      --network sepolia
+      --http --http-address 0.0.0.0
+      --execution-endpoint http://ethrex-sepolia:8551
+      --execution-jwt /secrets/jwt.hex
+      --checkpoint-sync-url https://checkpoint-sync.sepolia.ethpandaops.io
+      --checkpoint-sync-url-timeout 600
+      --purge-db
+    depends_on:
+      setup-jwt-sepolia:
+        condition: service_completed_successfully
+
+  ethrex-sepolia:
+    <<: *ethrex-common
+    container_name: ethrex-sepolia
+    ports:
+      - "8546:8545"  # RPC on different host port
+    volumes:
+      - secrets-sepolia:/secrets
+      - ethrex-sepolia:/data
+    command: >
+      --http.addr 0.0.0.0
+      --network sepolia
+      --authrpc.addr 0.0.0.0
+      --authrpc.jwtsecret /secrets/jwt.hex
+      --syncmode snap
+      --datadir /data
+    depends_on:
+      setup-jwt-sepolia:
+        condition: service_completed_successfully
+
+  # =============================================================================
+  # MAINNET
+  # =============================================================================
+  setup-jwt-mainnet:
+    image: alpine
+    volumes:
+      - secrets-mainnet:/secrets
+    command: sh -c 'apk add openssl && openssl rand -hex 32 | tr -d "\n" | tee /secrets/jwt.hex'
+
+  consensus-mainnet:
+    <<: *consensus-common
+    container_name: consensus-mainnet
+    volumes:
+      - secrets-mainnet:/secrets
+      - consensus-mainnet:/root/.lighthouse
+    command: >
+      lighthouse bn
+      --network mainnet
+      --http --http-address 0.0.0.0
+      --execution-endpoint http://ethrex-mainnet:8551
+      --execution-jwt /secrets/jwt.hex
+      --checkpoint-sync-url https://mainnet-checkpoint-sync.attestant.io
+      --checkpoint-sync-url-timeout 600
+      --purge-db
+    depends_on:
+      setup-jwt-mainnet:
+        condition: service_completed_successfully
+
+  ethrex-mainnet:
+    <<: *ethrex-common
+    container_name: ethrex-mainnet
+    ports:
+      - "8547:8545"  # RPC on different host port
+    volumes:
+      - secrets-mainnet:/secrets
+      - ethrex-mainnet:/data
+    command: >
+      --http.addr 0.0.0.0
+      --network mainnet
+      --authrpc.addr 0.0.0.0
+      --authrpc.jwtsecret /secrets/jwt.hex
+      --syncmode snap
+      --datadir /data
+    depends_on:
+      setup-jwt-mainnet:
+        condition: service_completed_successfully
+
+  # =============================================================================
+  # HOODI-2
+  # =============================================================================
+  setup-jwt-hoodi-2:
+    image: alpine
+    volumes:
+      - secrets-hoodi-2:/secrets
+    command: sh -c 'apk add openssl && openssl rand -hex 32 | tr -d "\n" | tee /secrets/jwt.hex'
+
+  consensus-hoodi-2:
+    <<: *consensus-common
+    container_name: consensus-hoodi-2
+    volumes:
+      - secrets-hoodi-2:/secrets
+      - consensus-hoodi-2:/root/.lighthouse
+    command: >
+      lighthouse bn
+      --network hoodi
+      --http --http-address 0.0.0.0
+      --execution-endpoint http://ethrex-hoodi-2:8551
+      --execution-jwt /secrets/jwt.hex
+      --checkpoint-sync-url https://hoodi-checkpoint-sync.stakely.io/
+      --checkpoint-sync-url-timeout 600
+      --purge-db
+    depends_on:
+      setup-jwt-hoodi-2:
+        condition: service_completed_successfully
+
+  ethrex-hoodi-2:
+    <<: *ethrex-common
+    container_name: ethrex-hoodi-2
+    ports:
+      - "8548:8545"  # RPC on different host port
+    volumes:
+      - secrets-hoodi-2:/secrets
+      - ethrex-hoodi-2:/data
+    command: >
+      --http.addr 0.0.0.0
+      --network hoodi
+      --authrpc.addr 0.0.0.0
+      --authrpc.jwtsecret /secrets/jwt.hex
+      --syncmode snap
+      --datadir /data
+    depends_on:
+      setup-jwt-hoodi-2:
+        condition: service_completed_successfully
+
+volumes:
+  secrets-hoodi:
+  secrets-sepolia:
+  secrets-mainnet:
+  consensus-hoodi:
+  consensus-sepolia:
+  consensus-mainnet:
+  ethrex-hoodi:
+  ethrex-sepolia:
+  ethrex-mainnet:
+  secrets-hoodi-2:
+  consensus-hoodi-2:
+  ethrex-hoodi-2: