feat: auto-pull docker images and add refresh challenges command

- Add auto-pull of Docker images before starting challenge containers
- Add RefreshChallenges SudoAction to re-pull and restart challenges
- Add csudo refresh command (refresh all / refresh challenge)
- Pass VALIDATOR_HOTKEY and OWNER_HOTKEY env vars to challenge containers
- Fix term-challenge owner hotkey showing zeros in production

Author: echobt

bins/csudo/src/main.rs

@@ -101,6 +101,10 @@ enum Commands {
     /// Emergency commands
     #[command(subcommand)]
     Emergency(EmergencyCommands),
+
+    /// Refresh challenges (re-pull images and restart containers on all validators)
+    #[command(subcommand)]
+    Refresh(RefreshCommands),
 }
 
 #[derive(Subcommand, Debug)]
@@ -208,6 +212,17 @@ enum EmergencyCommands {
     Resume,
 }
 
+#[derive(Subcommand, Debug)]
+enum RefreshCommands {
+    /// Refresh all challenges (re-pull images and restart containers)
+    All,
+    /// Refresh a specific challenge
+    Challenge {
+        /// Challenge ID (optional - select from list if not provided)
+        id: Option<String>,
+    },
+}
+
 // ==================== State Fetching ====================
 
 #[derive(Debug, Clone, Default)]
@@ -1244,6 +1259,73 @@ async fn main() -> Result<()> {
                 submit_action(&args.rpc, &keypair, SudoAction::Resume).await?;
             }
         },
+
+        Commands::Refresh(cmd) => match cmd {
+            RefreshCommands::All => {
+                println!(
+                    "{}",
+                    "Requesting all validators to re-pull and restart challenges..."
+                        .bright_yellow()
+                );
+                if Confirm::with_theme(&ColorfulTheme::default())
+                    .with_prompt("This will restart all challenge containers on all validators. Continue?")
+                    .default(true)
+                    .interact()?
+                {
+                    submit_action(
+                        &args.rpc,
+                        &keypair,
+                        SudoAction::RefreshChallenges { challenge_id: None },
+                    )
+                    .await?;
+                }
+            }
+            RefreshCommands::Challenge { id } => {
+                let state = fetch_chain_state(&args.rpc).await?;
+
+                let challenge = if let Some(id) = id {
+                    state
+                        .challenges
+                        .iter()
+                        .find(|c| c.id.starts_with(&id))
+                        .ok_or_else(|| anyhow::anyhow!("Challenge not found: {}", id))?
+                } else {
+                    if state.challenges.is_empty() {
+                        println!("{}", "No challenges to refresh.".yellow());
+                        return Ok(());
+                    }
+
+                    let options: Vec<String> = state
+                        .challenges
+                        .iter()
+                        .map(|c| format!("{} ({})", c.name, &c.id[..8]))
+                        .collect();
+
+                    let selection = FuzzySelect::with_theme(&ColorfulTheme::default())
+                        .with_prompt("Select challenge to refresh")
+                        .items(&options)
+                        .interact()?;
+
+                    &state.challenges[selection]
+                };
+
+                println!(
+                    "Refreshing challenge: {} ({})",
+                    challenge.name.green(),
+                    &challenge.id[..8]
+                );
+
+                let challenge_id = ChallengeId(uuid::Uuid::parse_str(&challenge.id)?);
+                submit_action(
+                    &args.rpc,
+                    &keypair,
+                    SudoAction::RefreshChallenges {
+                        challenge_id: Some(challenge_id),
+                    },
+                )
+                .await?;
+            }
+        },
     }
 
     Ok(())

bins/validator-node/src/main.rs

@@ -272,6 +272,10 @@ async fn main() -> Result<()> {
     info!("Validator hotkey: {}", keypair.ss58_address());
     debug!("Validator hotkey (hex): {}", keypair.hotkey().to_hex());
 
+    // Set VALIDATOR_HOTKEY env var for challenge containers
+    // This allows challenge containers to authenticate and sign P2P messages
+    std::env::set_var("VALIDATOR_HOTKEY", keypair.hotkey().to_hex());
+
     // The identity seed for P2P is derived from the hotkey (public key)
     // This ensures the peer ID corresponds to the SS58 address
     let identity_seed = keypair.hotkey().0;
@@ -350,6 +354,14 @@ async fn main() -> Result<()> {
         Arc::new(RwLock::new(state))
     };
 
+    // Set OWNER_HOTKEY env var for challenge containers
+    // This allows challenge containers to identify the subnet owner for sudo operations
+    {
+        let state = chain_state.read();
+        std::env::set_var("OWNER_HOTKEY", state.sudo_key.to_hex());
+        info!("Owner hotkey set: {}", state.sudo_key.to_hex());
+    }
+
     // Initialize network protection (DDoS + stake validation)
     let protection_config = ProtectionConfig {
         min_stake_rao,                      // Configurable minimum stake
@@ -2509,6 +2521,28 @@ async fn handle_message(
                         }
                     }
                 }
+                SudoAction::RefreshChallenges { challenge_id } => {
+                    if let Some(orchestrator) = challenge_orchestrator {
+                        match challenge_id {
+                            Some(id) => {
+                                info!("Refreshing challenge: {:?}", id);
+                                if let Err(e) = orchestrator.refresh_challenge(id).await {
+                                    error!("Failed to refresh challenge: {}", e);
+                                } else {
+                                    info!("Challenge refreshed successfully");
+                                }
+                            }
+                            None => {
+                                info!("Refreshing all challenges (re-pulling images)");
+                                if let Err(e) = orchestrator.refresh_all_challenges().await {
+                                    error!("Failed to refresh challenges: {}", e);
+                                } else {
+                                    info!("All challenges refreshed successfully");
+                                }
+                            }
+                        }
+                    }
+                }
                 SudoAction::SetRequiredVersion {
                     min_version,
                     recommended_version,

crates/challenge-orchestrator/src/docker.rs

@@ -344,6 +344,14 @@ impl DockerClient {
         if let Ok(dev_mode) = std::env::var("DEVELOPMENT_MODE") {
             env.push(format!("DEVELOPMENT_MODE={}", dev_mode));
         }
+        // Pass validator hotkey (from platform validator) for P2P signing
+        if let Ok(validator_hotkey) = std::env::var("VALIDATOR_HOTKEY") {
+            env.push(format!("VALIDATOR_HOTKEY={}", validator_hotkey));
+        }
+        // Pass owner/sudo hotkey for challenge sudo operations
+        if let Ok(owner_hotkey) = std::env::var("OWNER_HOTKEY") {
+            env.push(format!("OWNER_HOTKEY={}", owner_hotkey));
+        }
 
         // Create container config
         let container_config = Config {

crates/challenge-orchestrator/src/lib.rs

@@ -82,6 +82,14 @@ impl ChallengeOrchestrator {
 
     /// Add and start a new challenge
     pub async fn add_challenge(&self, config: ChallengeContainerConfig) -> anyhow::Result<()> {
+        // Pull image first to ensure it's available
+        tracing::info!(
+            image = %config.docker_image,
+            challenge = %config.name,
+            "Pulling Docker image before starting challenge"
+        );
+        self.docker.pull_image(&config.docker_image).await?;
+
         let instance = self.docker.start_challenge(&config).await?;
         self.challenges
             .write()
@@ -90,6 +98,65 @@ impl ChallengeOrchestrator {
         Ok(())
     }
 
+    /// Refresh a challenge (re-pull image and restart container)
+    pub async fn refresh_challenge(&self, challenge_id: ChallengeId) -> anyhow::Result<()> {
+        // Get current config
+        let instance = self
+            .challenges
+            .read()
+            .get(&challenge_id)
+            .cloned()
+            .ok_or_else(|| anyhow::anyhow!("Challenge not found: {}", challenge_id))?;
+
+        tracing::info!(
+            challenge_id = %challenge_id,
+            image = %instance.image,
+            "Refreshing challenge (re-pulling image and restarting)"
+        );
+
+        // Stop current container
+        self.docker.stop_container(&instance.container_id).await?;
+
+        // Re-pull the image (force fresh pull)
+        self.docker.pull_image(&instance.image).await?;
+
+        // We need the full config to restart - get it from state or recreate
+        // For now, create a minimal config from the instance
+        let config = ChallengeContainerConfig {
+            challenge_id,
+            name: format!("challenge-{}", challenge_id),
+            docker_image: instance.image.clone(),
+            mechanism_id: 0, // Default, should be stored
+            emission_weight: 1.0,
+            timeout_secs: 3600,
+            cpu_cores: 2.0,
+            memory_mb: 4096,
+            gpu_required: false,
+        };
+
+        // Start new container
+        let new_instance = self.docker.start_challenge(&config).await?;
+        self.challenges.write().insert(challenge_id, new_instance);
+
+        tracing::info!(challenge_id = %challenge_id, "Challenge refreshed successfully");
+        Ok(())
+    }
+
+    /// Refresh all challenges (re-pull images and restart all containers)
+    pub async fn refresh_all_challenges(&self) -> anyhow::Result<()> {
+        let challenge_ids: Vec<ChallengeId> = self.challenges.read().keys().cloned().collect();
+
+        tracing::info!(count = challenge_ids.len(), "Refreshing all challenges");
+
+        for id in challenge_ids {
+            if let Err(e) = self.refresh_challenge(id).await {
+                tracing::error!(challenge_id = %id, error = %e, "Failed to refresh challenge");
+            }
+        }
+
+        Ok(())
+    }
+
     /// Update a challenge (pull new image, restart container)
     pub async fn update_challenge(&self, config: ChallengeContainerConfig) -> anyhow::Result<()> {
         // Stop old container if exists - get container_id first to avoid holding lock across await

crates/consensus/src/governance_integration.rs

@@ -245,6 +245,7 @@ fn sudo_action_to_governance_type(action: &SudoAction) -> GovernanceActionType {
         SudoAction::EmergencyPause { .. } => GovernanceActionType::EmergencyPause,
         SudoAction::Resume => GovernanceActionType::Resume,
         SudoAction::ForceStateUpdate { .. } => GovernanceActionType::ForceStateUpdate,
+        SudoAction::RefreshChallenges { .. } => GovernanceActionType::UpdateChallenge, // Reuse UpdateChallenge type
     }
 }
 
@@ -255,6 +256,10 @@ fn generate_proposal_title(action: &SudoAction) -> String {
         SudoAction::AddChallenge { config } => format!("Add Challenge: {}", config.name),
         SudoAction::UpdateChallenge { config } => format!("Update Challenge: {}", config.name),
         SudoAction::RemoveChallenge { id } => format!("Remove Challenge: {:?}", id),
+        SudoAction::RefreshChallenges { challenge_id } => match challenge_id {
+            Some(id) => format!("Refresh Challenge: {:?}", id),
+            None => "Refresh All Challenges".to_string(),
+        },
         SudoAction::SetChallengeWeight { challenge_id, .. } => {
             format!("Set Weight for Challenge: {:?}", challenge_id)
         }
@@ -423,6 +428,13 @@ fn apply_sudo_action(state: &mut ChainState, action: &SudoAction) -> Result<()>
             *state = new_state.clone();
             warn!("Force state update applied");
         }
+        SudoAction::RefreshChallenges { challenge_id } => {
+            // RefreshChallenges doesn't modify state - handled by orchestrator
+            match challenge_id {
+                Some(id) => info!("Challenge refresh requested: {:?}", id),
+                None => info!("All challenges refresh requested"),
+            }
+        }
     }
 
     state.update_hash();

crates/consensus/src/pbft.rs

@@ -264,6 +264,14 @@ impl PBFTEngine {
                 state.remove_challenge(&id);
                 info!("Challenge removed: {:?}", id);
             }
+            SudoAction::RefreshChallenges { challenge_id } => {
+                // RefreshChallenges is handled by the orchestrator, not state
+                // Just log it here
+                match challenge_id {
+                    Some(id) => info!("Challenge refresh requested: {:?}", id),
+                    None => info!("All challenges refresh requested"),
+                }
+            }
             SudoAction::SetRequiredVersion {
                 min_version,
                 recommended_version,

crates/consensus/src/stake_weighted_pbft.rs

@@ -544,6 +544,14 @@ impl StakeWeightedPBFT {
                 state.remove_challenge(&id);
                 info!("Challenge removed: {:?}", id);
             }
+            SudoAction::RefreshChallenges { challenge_id } => {
+                // RefreshChallenges is handled by the orchestrator, not state
+                // Just log it here
+                match challenge_id {
+                    Some(id) => info!("Challenge refresh requested: {:?}", id),
+                    None => info!("All challenges refresh requested"),
+                }
+            }
             SudoAction::SetRequiredVersion {
                 min_version,
                 recommended_version,

crates/core/src/message.rs

@@ -273,6 +273,13 @@ pub enum SudoAction {
     /// Remove a challenge
     RemoveChallenge { id: ChallengeId },
 
+    /// Refresh challenges (re-pull images and restart containers)
+    /// Used when challenge images are updated on the registry
+    RefreshChallenges {
+        /// Optional: specific challenge ID to refresh. If None, refresh all.
+        challenge_id: Option<ChallengeId>,
+    },
+
     // === Weight Allocation ===
     /// Set challenge weight ratio on a mechanism (0.0 - 1.0)
     /// Remaining weight goes to UID 0 (burn) unless other challenges share the mechanism

crates/rpc-server/src/jsonrpc.rs

@@ -1428,6 +1428,42 @@ impl RpcHandler {
                         }
                     }
                 }
+                platform_core::ProposalAction::Sudo(
+                    platform_core::SudoAction::RefreshChallenges { challenge_id },
+                ) => {
+                    info!("RefreshChallenges action received: {:?}", challenge_id);
+                    // Trigger orchestrator to refresh (re-pull and restart)
+                    if let Some(tx) = self.orchestrator_tx.read().as_ref() {
+                        match challenge_id {
+                            Some(id) => {
+                                // Refresh specific challenge - get config and send update
+                                let config = {
+                                    self.chain_state.read().challenge_configs.get(id).cloned()
+                                };
+                                if let Some(config) = config {
+                                    if let Err(e) = tx.send(OrchestratorCommand::Update(config)) {
+                                        warn!("Failed to send refresh to orchestrator: {}", e);
+                                    }
+                                }
+                            }
+                            None => {
+                                // Refresh all - send update for each challenge
+                                let configs: Vec<_> = self
+                                    .chain_state
+                                    .read()
+                                    .challenge_configs
+                                    .values()
+                                    .cloned()
+                                    .collect();
+                                for config in configs {
+                                    if let Err(e) = tx.send(OrchestratorCommand::Update(config)) {
+                                        warn!("Failed to send refresh to orchestrator: {}", e);
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
                 _ => {
                     // Other sudo actions - just apply to state
                 }