[CI] Add retries to avoid failing because of flakey network issues (#281)

* [CI] Add retries to avoid failing because of flakey network issues

* Print each retry error if all retries failed

Author: amenasria

scripts/run_examples.sh

@@ -1,5 +1,34 @@
 #!/bin/bash
 
+# Default retry count
+RETRY_COUNT=3
+
+# Parse command line arguments
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        --no-extra-dep)
+            NO_EXTRA_DEP=true
+            shift
+            ;;
+        --retry-count)
+            RETRY_COUNT="$2"
+            shift 2
+            ;;
+        --help)
+            echo "Usage: $0 [--no-extra-dep] [--retry-count N]"
+            echo "  --no-extra-dep: Exclude files that require extra dependencies"
+            echo "  --retry-count N: Number of retries for each test (default: 3)"
+            echo "  --help: Show this help message"
+            exit 0
+            ;;
+        *)
+            echo "Unknown option: $1"
+            echo "Use --help for usage information"
+            exit 1
+            ;;
+    esac
+done
+
 # List of files to exclude
 exclude_files=(
  "examples/mistral/chat/chatbot_with_streaming.py"
@@ -13,8 +42,8 @@ exclude_files=(
  "examples/mistral/agents/async_conversation_run_mcp_remote.py"
 )
 
-# Check if the first argument is "no-extra-dep" then remove all the files that require the extra dependencies
-if [ "$1" = "--no-extra-dep" ]; then
+# Check if the no-extra-dep flag is set
+if [ "$NO_EXTRA_DEP" = true ]; then
     # Add more files to the exclude list
     exclude_files+=(
       "examples/mistral/agents/async_conversation_run_mcp_remote.py"
@@ -25,19 +54,52 @@ fi
 
 failed=0
 
+# Function to run a test with retries
+run_test_with_retries() {
+    local file="$1"
+    local attempt=1
+    local error_outputs=()
+    
+    while [ $attempt -le $RETRY_COUNT ]; do
+        echo "Running $file (attempt $attempt/$RETRY_COUNT)"
+        
+        # Run the script and capture both exit status and error output
+        local current_output=$(python3 "$file" 2>&1)
+        local exit_code=$?
+        
+        if [ $exit_code -eq 0 ]; then
+            echo "Success"
+            return 0
+        else
+            # Store the error output from this attempt
+            error_outputs+=("Attempt $attempt: $current_output")
+            
+            if [ $attempt -lt $RETRY_COUNT ]; then
+                echo "Failed (attempt $attempt/$RETRY_COUNT), retrying..."
+                sleep 1  # Brief pause before retry
+            else
+                echo "Failed after $RETRY_COUNT attempts"
+                echo "Error outputs from all attempts:"
+                for error_output in "${error_outputs[@]}"; do
+                    echo "$error_output"
+                    echo "---"
+                done
+                return 1
+            fi
+        fi
+        
+        attempt=$((attempt + 1))
+    done
+}
+
 for file in examples/mistral/**/*.py; do
     # Check if the file is not in the exclude list
     if [ -f "$file" ] && [[ ! " ${exclude_files[@]} " =~ " $file " ]]; then
-        echo "Running $file"
-        # Run the script and capture the exit status
-        if python3 "$file" > /dev/null; then
-            echo "Success"
-        else
-            echo "Failed"
+        if ! run_test_with_retries "$file"; then
             failed=1
         fi
     else
-      echo "Skipped $file"
+        echo "Skipped $file"
     fi
 done