feat(l1): docker compose parallel snapsync run in loop

.gitignore

@@ -18,6 +18,7 @@ tooling/ef_tests/state/runner_v2/success_report.txt
 tooling/reorgs/data
 
 tooling/sync/logs/
+tooling/sync/multisync_logs/
 
 # Repos checked out by make target
 /hive/

tooling/sync/Makefile

@@ -2,7 +2,9 @@
 flamegraph-branch flamegraph-inner flamegraph-mainnet flamegraph-sepolia flamegraph-holesky \
 flamegraph-hoodi start-lighthouse start-ethrex backup-db start-mainnet-metrics-docker \
 start-sepolia-metrics-docker start-holesky-metrics-docker start-hoodi-metrics-docker \
-start-metrics-docker tail-syncing-logs tail-metrics-logs copy_flamegraph import-with-metrics
+start-metrics-docker tail-syncing-logs tail-metrics-logs copy_flamegraph import-with-metrics \
+multisync-up multisync-down multisync-clean multisync-logs multisync-status \
+multisync-restart multisync-monitor multisync-run multisync-loop
 
 ETHREX_DIR ?= "../.."
 EVM ?= levm
@@ -220,3 +222,76 @@ server-sync:
 	sleep 0.2
 
 	tmux new-window -t sync:2 -n ethrex "cd ../../metrics && docker stop metrics-ethereum-metrics-exporter-1 || true && docker compose -f docker-compose-metrics.yaml -f docker-compose-metrics-l1.overrides.yaml up -d && cd .. && ulimit -n 1000000 && rm -rf ~/.local/share/ethrex && RUST_LOG=info,ethrex_p2p::sync=debug $(if $(DEBUG_ASSERT),RUSTFLAGS='-C debug-assertions=yes') $(if $(HEALING),SKIP_START_SNAP_SYNC=1) cargo run --release --bin ethrex --features rocksdb --  --http.addr 0.0.0.0 --metrics --metrics.port 3701 --network $(SERVER_SYNC_NETWORK) $(if $(MEMORY),--datadir memory) --authrpc.jwtsecret ~/secrets/jwt.hex $(if $(or $(FULL_SYNC),$(HEALING)),--syncmode full)  2>&1 | tee $(LOGS_FILE)"
+
+# ==============================================================================
+# Docker Compose Multi-Network Snapsync
+# ==============================================================================
+
+MULTISYNC_COMPOSE = docker compose -f docker-compose.multisync.yaml
+MULTISYNC_NETWORKS ?= hoodi,sepolia,mainnet
+comma := ,
+MULTISYNC_NETWORK_LIST := $(subst $(comma), ,$(MULTISYNC_NETWORKS))
+MULTISYNC_SERVICES := $(foreach n,$(MULTISYNC_NETWORK_LIST),setup-jwt-$(n) ethrex-$(n) consensus-$(n))
+
+multisync-up: ## Start all networks specified in MULTISYNC_NETWORKS via Docker Compose.
+	$(MULTISYNC_COMPOSE) up -d $(MULTISYNC_SERVICES)
+
+multisync-down: ## Stop and remove all snapsync containers.
+	$(MULTISYNC_COMPOSE) down
+
+multisync-clean: ## Stop, remove containers AND volumes (full reset).
+	$(MULTISYNC_COMPOSE) down -v
+
+multisync-logs: ## Tail logs from all networks.
+	$(MULTISYNC_COMPOSE) logs -f
+
+multisync-logs-%: ## Tail logs for a specific network (e.g., multisync-logs-hoodi).
+	$(MULTISYNC_COMPOSE) logs -f ethrex-$* consensus-$*
+
+multisync-logs-ethrex-%: ## Tail only ethrex logs for a network (e.g., multisync-logs-ethrex-hoodi).
+	$(MULTISYNC_COMPOSE) logs -f ethrex-$*
+
+multisync-logs-consensus-%: ## Tail only consensus logs for a network (e.g., multisync-logs-consensus-hoodi).
+	$(MULTISYNC_COMPOSE) logs -f consensus-$*
+
+multisync-restart: ## Restart the cycle (clean volumes + start fresh).
+	$(MULTISYNC_COMPOSE) down -v
+	$(MULTISYNC_COMPOSE) up -d $(MULTISYNC_SERVICES)
+
+multisync-monitor: ## Monitor all networks (one-shot, exits on completion).
+	python3 docker_monitor.py --networks $(MULTISYNC_NETWORKS) --exit-on-success
+
+multisync-run: ## Full run: start + monitor (one-shot, exits on completion).
+	$(MULTISYNC_COMPOSE) up -d $(MULTISYNC_SERVICES)
+	@echo "Waiting 10s for containers to start..."
+	@sleep 10
+	python3 docker_monitor.py --networks $(MULTISYNC_NETWORKS) --exit-on-success
+
+multisync-loop: ## Continuous loop: sync all networks, restart on success, repeat forever.
+	$(MULTISYNC_COMPOSE) up -d $(MULTISYNC_SERVICES)
+	@echo "Waiting 10s for containers to start..."
+	@sleep 10
+	python3 docker_monitor.py --networks $(MULTISYNC_NETWORKS) --compose-file docker-compose.multisync.yaml --compose-dir $(CURDIR)
+
+multisync-history: ## View the run history log.
+	@if [ -f multisync_logs/run_history.log ]; then \
+		cat multisync_logs/run_history.log; \
+	else \
+		echo "No run history found. Run 'make multisync-loop' first."; \
+	fi
+
+multisync-list-logs: ## List all saved run logs.
+	@if [ -d multisync_logs ]; then \
+		echo "=== Saved Run Logs ===" && \
+		ls -la multisync_logs/ && \
+		echo "" && \
+		for dir in multisync_logs/run_*/; do \
+			if [ -d "$$dir" ]; then \
+				echo "$$dir:"; \
+				ls "$$dir"; \
+				echo ""; \
+			fi; \
+		done; \
+	else \
+		echo "No logs directory found."; \
+	fi

tooling/sync/README.md

@@ -61,3 +61,126 @@ It's advisable to only run flamegraphs on blocks that have already been synced,
 - `make copy-flamegraph` can be used to quickly copy the flamegraph generated by the flamegraph commands from the `ethrex` repo folder to the `tooling/sync/flamegraphs` folder so it isn't overwritten by future flamegraph runs. `GRAPHNAME` can be provided to give the file a custom name.
 
 - `make import-with-metrics` can be used to import blocks from an RLP file with metrics enabled, specially useful for a block processing profile. The path to the rlp file can be passed with the `RLP_FILE` environment variable, while the network can be provided with the `NETWORK` variable.
+
+## Multi-Network Parallel Snapsync
+
+This feature allows running multiple Ethrex nodes in parallel (hoodi, sepolia, mainnet) via Docker Compose, with automated monitoring, Slack notifications, and a history log of runs.
+
+### Overview
+
+The parallel snapsync system:
+- Spawns multiple networks simultaneously via Docker Compose
+- Monitors snapsync progress with a 4-hour timeout
+- Verifies block processing for 22 minutes after sync completion
+- Sends Slack notifications on success/failure
+- Maintains a history log of all runs
+- On success: restarts containers and begins a new sync cycle
+- On failure: keeps containers running for debugging
+
+### Requirements
+
+- Docker and Docker Compose
+- Python 3 with the `requests` library (`pip install requests`)
+- (Optional) Slack webhook URLs for notifications
+
+### Quick Start
+
+```bash
+# Start a continuous monitoring loop (recommended for servers)
+make multisync-loop
+
+# Or run a single sync cycle
+make multisync-run
+```
+
+### Docker Compose Setup
+
+The `docker-compose.multisync.yaml` file defines services for each network with isolated volumes. Each network uses Lighthouse as the consensus client with checkpoint sync.
+
+Host port mapping:
+- **hoodi**: `localhost:8545`
+- **sepolia**: `localhost:8546`
+- **mainnet**: `localhost:8547`
+- **hoodi-2**: `localhost:8548` (for additional testing)
+
+### Environment Variables
+
+Create a `.env` file in `tooling/sync/` with:
+
+```bash
+# Slack notifications (optional)
+SLACK_WEBHOOK_URL_SUCCESS=https://hooks.slack.com/services/...
+SLACK_WEBHOOK_URL_FAILED=https://hooks.slack.com/services/...
+```
+
+The `MULTISYNC_NETWORKS` variable controls which networks to sync (default: `hoodi,sepolia,mainnet`):
+
+```bash
+# Sync only hoodi and sepolia
+make multisync-loop MULTISYNC_NETWORKS=hoodi,sepolia
+```
+
+### Monitoring Behavior
+
+The `docker_monitor.py` script manages the sync lifecycle:
+
+1. **Waiting**: Node container starting up
+2. **Syncing**: Snapsync in progress (4-hour timeout)
+3. **Block Processing**: Sync complete, verifying block processing (22 minutes)
+4. **Success**: Network synced and processing blocks
+5. **Failed**: Timeout, stall, or error detected
+
+The monitor checks for:
+- Sync timeout (default 4 hours)
+- Block processing stall (10 minutes without new blocks)
+- Node unresponsiveness
+
+### Logs and History
+
+Logs are saved to `tooling/sync/multisync_logs/`:
+
+```
+multisync_logs/
+├── run_history.log          # Append-only history of all runs
+└── run_YYYYMMDD_HHMMSS/     # Per-run folder
+    ├── summary.txt          # Run summary
+    ├── ethrex-hoodi.log     # Ethrex logs per network
+    ├── consensus-hoodi.log  # Lighthouse logs per network
+    └── ...
+```
+
+### Commands
+
+**Starting and Stopping:**
+
+- `make multisync-up` starts all networks via Docker Compose.
+- `make multisync-down` stops and removes containers (preserves volumes).
+- `make multisync-clean` stops containers and removes volumes (full reset).
+- `make multisync-restart` restarts the cycle (clean volumes + start fresh).
+
+**Monitoring:**
+
+- `make multisync-loop` runs continuous sync cycles (recommended for servers). On success, restarts and syncs again. On failure, stops for debugging.
+- `make multisync-run` runs a single sync cycle and exits on completion.
+- `make multisync-monitor` monitors already-running containers (one-shot).
+
+**Logs:**
+
+- `make multisync-logs` tails logs from all networks.
+- `make multisync-logs-hoodi` tails logs for a specific network.
+- `make multisync-logs-ethrex-hoodi` tails only ethrex logs for a network.
+- `make multisync-logs-consensus-hoodi` tails only consensus logs for a network.
+- `make multisync-history` views the run history log.
+- `make multisync-list-logs` lists all saved run logs.
+
+### Slack Notifications
+
+When configured, notifications are sent:
+- On **success**: All networks synced and processing blocks
+- On **failure**: Any network failed (timeout, stall, or error)
+
+Notifications include:
+- Run ID and count
+- Host, branch, and commit info
+- Per-network status with sync time and blocks processed
+- Link to the commit on GitHub