setup_docuchat.sh

#!/bin/bash

# =============================================================================
# DocuChat Agent - Complete Setup Script
# =============================================================================
# This script handles the complete installation and configuration of DocuChat Agent
# including Neo4j, Ollama, Python dependencies, and system requirements.
#
# Usage:
#   ./setup_docuchat.sh [OPTIONS]
#
# Options:
#   --auto             Run fully automated setup (no prompts)
#   --interactive      Interactive mode with confirmations (default)
#   --resume-from=N    Resume from specific phase (1-6)
#   --skip-docker      Skip Docker/Neo4j setup (use existing)
#   --skip-ollama      Skip Ollama installation (use existing)
#   --help            Show this help message
#
# Requirements:
#   - Linux, macOS, or WSL2
#   - 8GB+ RAM recommended
#   - 10GB+ free disk space
#   - Internet connection for downloads
# =============================================================================

set -euo pipefail  # Exit on error, undefined vars, pipe failures

# =============================================================================
# CONFIGURATION AND GLOBALS
# =============================================================================

# Script metadata
readonly SCRIPT_VERSION="1.0.0"
readonly SCRIPT_NAME="DocuChat Setup"
readonly LOG_FILE="setup_docuchat.log"
readonly CONFIG_FILE=".setup_progress"

# Colors for output
readonly RED='\033[0;31m'
readonly GREEN='\033[0;32m'
readonly YELLOW='\033[1;33m'
readonly BLUE='\033[0;34m'
readonly PURPLE='\033[0;35m'
readonly CYAN='\033[0;36m'
readonly WHITE='\033[1;37m'
readonly NC='\033[0m' # No Color

# Unicode symbols
readonly CHECK_MARK="✓"
readonly CROSS_MARK="✗"
readonly INFO_MARK="ℹ"
readonly WARN_MARK="⚠"
readonly ROCKET="🚀"
readonly GEAR="⚙️"

# Default settings
AUTO_MODE=false
INTERACTIVE_MODE=true
RESUME_FROM=1
SKIP_DOCKER=false
SKIP_OLLAMA=false

# System detection
OS_TYPE=""
PACKAGE_MANAGER=""
PYTHON_CMD=""
PIP_CMD=""

# Progress tracking
PHASE_STATUS=(false false false false false false)
readonly TOTAL_PHASES=6

# =============================================================================
# UTILITY FUNCTIONS
# =============================================================================

# Logging function
log() {
    local level=$1
    shift
    local message="$*"
    local timestamp=$(date '+%Y-%m-%d %H:%M:%S')

    case $level in
        "INFO")  echo -e "${BLUE}${INFO_MARK}${NC} $message" ;;
        "SUCCESS") echo -e "${GREEN}${CHECK_MARK}${NC} $message" ;;
        "WARN")  echo -e "${YELLOW}${WARN_MARK}${NC} $message" ;;
        "ERROR") echo -e "${RED}${CROSS_MARK}${NC} $message" ;;
        "HEADER") echo -e "\n${PURPLE}${GEAR}${NC} ${WHITE}$message${NC}" ;;
    esac

    echo "[$timestamp] [$level] $message" >> "$LOG_FILE"
}

# Progress bar function
show_progress() {
    local current=$1
    local total=$2
    local description=$3
    local width=50
    local percentage=$((current * 100 / total))
    local filled=$((current * width / total))
    local empty=$((width - filled))

    printf "\r${CYAN}Progress:${NC} ["
    printf "%*s" $filled | tr ' ' '='
    printf "%*s" $empty | tr ' ' '-'
    printf "] %d%% - %s" $percentage "$description"

    if [ $current -eq $total ]; then
        echo ""
    fi
}

# Check if command exists
command_exists() {
    command -v "$1" >/dev/null 2>&1
}

# Detect operating system
detect_os() {
    log "INFO" "Detecting operating system..."

    case "$(uname -s)" in
        Linux*)
            OS_TYPE="Linux"
            if command_exists apt-get; then
                PACKAGE_MANAGER="apt"
            elif command_exists yum; then
                PACKAGE_MANAGER="yum"
            elif command_exists dnf; then
                PACKAGE_MANAGER="dnf"
            elif command_exists pacman; then
                PACKAGE_MANAGER="pacman"
            else
                log "ERROR" "Unsupported Linux distribution. Please install dependencies manually."
                exit 1
            fi
            ;;
        Darwin*)
            OS_TYPE="macOS"
            PACKAGE_MANAGER="brew"
            if ! command_exists brew; then
                log "WARN" "Homebrew not found. Installing Homebrew..."
                # Download and verify Homebrew installer
                local brew_install_script="/tmp/homebrew_install.sh"
                curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh -o "$brew_install_script"
                # Verify script is not empty and contains expected content
                if [ ! -s "$brew_install_script" ] || ! grep -q "Homebrew" "$brew_install_script"; then
                    log "ERROR" "Failed to download valid Homebrew installer"
                    exit 1
                fi
                /bin/bash "$brew_install_script"
                rm -f "$brew_install_script"
            fi
            ;;
        CYGWIN*|MINGW*|MSYS*)
            log "ERROR" "Windows detected. Please use WSL2 for best compatibility."
            exit 1
            ;;
        *)
            log "ERROR" "Unsupported operating system: $(uname -s)"
            exit 1
            ;;
    esac

    log "SUCCESS" "Detected $OS_TYPE with $PACKAGE_MANAGER package manager"
}

# Detect Python installation
detect_python() {
    log "INFO" "Detecting Python installation..."

    for cmd in python3.12 python3.11 python3.10 python3 python; do
        if command_exists "$cmd"; then
            local version=$($cmd --version 2>&1 | grep -oE '[0-9]+\.[0-9]+')
            local major=$(echo $version | cut -d. -f1)
            local minor=$(echo $version | cut -d. -f2)

            if [ "$major" -eq 3 ] && [ "$minor" -ge 10 ]; then
                PYTHON_CMD="$cmd"
                PIP_CMD="$cmd -m pip"
                log "SUCCESS" "Found Python $version at $cmd"
                return 0
            fi
        fi
    done

    log "ERROR" "Python 3.10+ not found. Please install Python 3.10 or later."
    exit 1
}

# Save progress
save_progress() {
    local phase=$1
    PHASE_STATUS[$((phase-1))]=true
    echo "PHASE_$phase=true" >> "$CONFIG_FILE"
}

# Load progress
load_progress() {
    if [ -f "$CONFIG_FILE" ]; then
        source "$CONFIG_FILE"
        for i in {1..6}; do
            local var="PHASE_$i"
            if [ "${!var:-false}" = "true" ]; then
                PHASE_STATUS[$((i-1))]=true
            fi
        done
    fi
}

# =============================================================================
# PHASE IMPLEMENTATIONS
# =============================================================================

# Phase 1: System Requirements Check
phase1_system_check() {
    log "HEADER" "Phase 1/6: System Requirements Check"

    if [ "${PHASE_STATUS[0]}" = "true" ] && [ $RESUME_FROM -le 1 ]; then
        log "INFO" "Phase 1 already completed, skipping..."
        return 0
    fi

    show_progress 1 10 "Checking system requirements..."

    # Check available disk space
    local available_space=$(df . | tail -1 | awk '{print $4}')
    local required_space=10485760  # 10GB in KB

    show_progress 2 10 "Checking disk space..."
    if [ $available_space -lt $required_space ]; then
        log "WARN" "Low disk space detected. At least 10GB recommended."
    else
        log "SUCCESS" "Sufficient disk space available"
    fi

    # Check memory
    show_progress 3 10 "Checking memory..."
    local total_mem=$(grep MemTotal /proc/meminfo 2>/dev/null | awk '{print $2}' || echo "8388608")
    local required_mem=8388608  # 8GB in KB

    if [ $total_mem -lt $required_mem ]; then
        log "WARN" "Less than 8GB RAM detected. Performance may be limited."
    else
        log "SUCCESS" "Sufficient memory available"
    fi

    # Check Docker
    show_progress 4 10 "Checking Docker..."
    if ! command_exists docker; then
        log "INFO" "Docker not found. Installing Docker..."
        install_docker
    else
        log "SUCCESS" "Docker found"

        # Check if Docker daemon is running
        if ! docker info >/dev/null 2>&1; then
            log "INFO" "Starting Docker daemon..."
            case $OS_TYPE in
                "Linux")
                    sudo systemctl start docker 2>/dev/null || service docker start 2>/dev/null || true
                    ;;
                "macOS")
                    open -a Docker 2>/dev/null || true
                    ;;
            esac

            # Wait for Docker to start
            for i in {1..30}; do
                if docker info >/dev/null 2>&1; then
                    break
                fi
                sleep 2
                show_progress $((4 + i/10)) 10 "Waiting for Docker to start..."
            done
        fi
    fi

    # Check internet connectivity
    show_progress 6 10 "Checking internet connectivity..."
    if ! curl -s --connect-timeout 5 google.com >/dev/null; then
        log "ERROR" "No internet connection. Please check your network."
        exit 1
    fi
    log "SUCCESS" "Internet connectivity verified"

    # Check curl and git
    show_progress 7 10 "Checking essential tools..."
    for tool in curl git; do
        if ! command_exists $tool; then
            log "INFO" "Installing $tool..."
            install_system_tool $tool
        else
            log "SUCCESS" "$tool found"
        fi
    done

    show_progress 10 10 "System check completed"
    save_progress 1
    log "SUCCESS" "Phase 1 completed successfully"
}

# Install Docker based on OS
install_docker() {
    case $OS_TYPE in
        "Linux")
            if [ "$PACKAGE_MANAGER" = "apt" ]; then
                sudo apt-get update
                sudo apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release
                curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
                echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
                sudo apt-get update
                sudo apt-get install -y docker-ce docker-ce-cli containerd.io
                sudo systemctl enable docker
                sudo usermod -aG docker $USER
            elif [ "$PACKAGE_MANAGER" = "yum" ] || [ "$PACKAGE_MANAGER" = "dnf" ]; then
                sudo $PACKAGE_MANAGER install -y yum-utils
                sudo yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
                sudo $PACKAGE_MANAGER install -y docker-ce docker-ce-cli containerd.io
                sudo systemctl enable docker
                sudo usermod -aG docker $USER
            fi
            ;;
        "macOS")
            if ! command_exists brew; then
                log "ERROR" "Homebrew required for Docker installation on macOS"
                exit 1
            fi
            brew install --cask docker
            ;;
    esac

    log "SUCCESS" "Docker installed successfully"
    log "WARN" "You may need to log out and back in for Docker permissions to take effect"
}

# Install system tools
install_system_tool() {
    local tool=$1

    case $OS_TYPE in
        "Linux")
            case $PACKAGE_MANAGER in
                "apt") sudo apt-get install -y $tool ;;
                "yum"|"dnf") sudo $PACKAGE_MANAGER install -y $tool ;;
                "pacman") sudo pacman -S --noconfirm $tool ;;
            esac
            ;;
        "macOS")
            brew install $tool
            ;;
    esac
}

# Phase 2: Neo4j Container Setup
phase2_neo4j_setup() {
    log "HEADER" "Phase 2/6: Neo4j Database Setup"

    if [ "${PHASE_STATUS[1]}" = "true" ] && [ $RESUME_FROM -le 2 ]; then
        log "INFO" "Phase 2 already completed, skipping..."
        return 0
    fi

    if [ "$SKIP_DOCKER" = "true" ]; then
        log "INFO" "Skipping Docker/Neo4j setup as requested"
        save_progress 2
        return 0
    fi

    show_progress 1 8 "Setting up Neo4j database..."

    # Stop existing container if running
    if docker ps -a | grep -q docuchat-neo4j; then
        log "INFO" "Stopping existing Neo4j container..."
        docker stop docuchat-neo4j >/dev/null 2>&1 || true
        docker rm docuchat-neo4j >/dev/null 2>&1 || true
    fi

    show_progress 2 8 "Pulling Neo4j image..."
    docker pull neo4j:5.15-community

    show_progress 3 8 "Creating data volumes..."
    # Create directories for persistent data
    mkdir -p ./data/neo4j/data
    mkdir -p ./data/neo4j/logs
    mkdir -p ./data/neo4j/import
    mkdir -p ./data/neo4j/plugins

    show_progress 4 8 "Starting Neo4j container..."
    docker run -d \
        --name docuchat-neo4j \
        --restart unless-stopped \
        -p 7474:7474 \
        -p 7687:7687 \
        -v "$(pwd)/data/neo4j/data:/data" \
        -v "$(pwd)/data/neo4j/logs:/logs" \
        -v "$(pwd)/data/neo4j/import:/var/lib/neo4j/import" \
        -v "$(pwd)/data/neo4j/plugins:/plugins" \
        -e NEO4J_AUTH=neo4j/docuchat123 \
        -e NEO4J_PLUGINS='["apoc"]' \
        -e NEO4J_dbms_security_procedures_unrestricted=apoc.* \
        -e NEO4J_dbms_memory_heap_initial__size=1G \
        -e NEO4J_dbms_memory_heap_max__size=2G \
        neo4j:5.15-community

    show_progress 5 8 "Waiting for Neo4j to start..."

    # Wait for Neo4j to be ready
    local max_attempts=60
    local attempt=1

    while [ $attempt -le $max_attempts ]; do
        if curl -s http://localhost:7474 >/dev/null 2>&1; then
            log "SUCCESS" "Neo4j is ready!"
            break
        fi

        if [ $attempt -eq $max_attempts ]; then
            log "ERROR" "Neo4j failed to start within 2 minutes"
            docker logs docuchat-neo4j --tail 20
            exit 1
        fi

        sleep 2
        show_progress $((5 + attempt/12)) 8 "Waiting for Neo4j (${attempt}s)..."
        ((attempt++))
    done

    show_progress 7 8 "Testing Neo4j connection..."

    # Test connection with Python
    if command_exists $PYTHON_CMD; then
        $PYTHON_CMD -c "
import sys
try:
    from neo4j import GraphDatabase
    driver = GraphDatabase.driver('bolt://localhost:7687', auth=('neo4j', 'docuchat123'))
    with driver.session() as session:
        result = session.run('RETURN 1 as test')
        print('Neo4j connection test: SUCCESS')
    driver.close()
except Exception as e:
    print(f'Neo4j connection test failed: {e}')
    sys.exit(1)
" 2>/dev/null || log "WARN" "Could not test Neo4j connection (Python neo4j driver not installed yet)"
    fi

    show_progress 8 8 "Neo4j setup completed"
    save_progress 2
    log "SUCCESS" "Phase 2 completed successfully"
    log "INFO" "Neo4j Web Interface: http://localhost:7474"
    log "INFO" "Username: neo4j, Password: docuchat123"
}

# Phase 3: System Tools Installation
phase3_system_tools() {
    log "HEADER" "Phase 3/6: System Tools Installation"

    if [ "${PHASE_STATUS[2]}" = "true" ] && [ $RESUME_FROM -le 3 ]; then
        log "INFO" "Phase 3 already completed, skipping..."
        return 0
    fi

    show_progress 1 6 "Installing system tools..."

    # Install Tesseract OCR
    show_progress 2 6 "Installing Tesseract OCR..."
    if ! command_exists tesseract; then
        case $OS_TYPE in
            "Linux")
                case $PACKAGE_MANAGER in
                    "apt")
                        sudo apt-get update
                        sudo apt-get install -y tesseract-ocr tesseract-ocr-eng
                        ;;
                    "yum"|"dnf")
                        sudo $PACKAGE_MANAGER install -y tesseract tesseract-langpack-eng
                        ;;
                    "pacman")
                        sudo pacman -S --noconfirm tesseract tesseract-data-eng
                        ;;
                esac
                ;;
            "macOS")
                brew install tesseract
                ;;
        esac
        log "SUCCESS" "Tesseract OCR installed"
    else
        log "SUCCESS" "Tesseract OCR already installed"
    fi

    # Install Ollama
    show_progress 3 6 "Installing Ollama..."
    if [ "$SKIP_OLLAMA" = "false" ]; then
        if ! command_exists ollama; then
            log "INFO" "Downloading and installing Ollama..."
            # Download Ollama installer to a temporary file first
            local ollama_installer="/tmp/ollama_install.sh"
            if ! curl -fsSL https://ollama.ai/install.sh -o "$ollama_installer"; then
                log "ERROR" "Failed to download Ollama installer"
                exit 1
            fi
            # Verify the script is not empty and contains expected Ollama content
            if [ ! -s "$ollama_installer" ] || ! grep -q "ollama" "$ollama_installer"; then
                log "ERROR" "Downloaded Ollama installer appears invalid"
                rm -f "$ollama_installer"
                exit 1
            fi
            # Execute the installer
            sh "$ollama_installer"
            rm -f "$ollama_installer"

            # Start Ollama service
            case $OS_TYPE in
                "Linux")
                    if command_exists systemctl; then
                        sudo systemctl enable ollama
                        sudo systemctl start ollama
                    fi
                    ;;
                "macOS")
                    # Ollama will start automatically on macOS
                    ;;
            esac

            log "SUCCESS" "Ollama installed"
        else
            log "SUCCESS" "Ollama already installed"
        fi

        # Wait for Ollama to be ready
        show_progress 4 6 "Starting Ollama service..."
        local max_attempts=30
        local attempt=1

        while [ $attempt -le $max_attempts ]; do
            if curl -s http://localhost:11434/api/version >/dev/null 2>&1; then
                log "SUCCESS" "Ollama service is ready"
                break
            fi

            if [ $attempt -eq $max_attempts ]; then
                log "WARN" "Ollama service may not be ready. Continuing anyway..."
                break
            fi

            sleep 2
            ((attempt++))
        done

        # Download default model
        show_progress 5 6 "Downloading Ollama model (this may take a while)..."
        if ! ollama list | grep -q "gemma3:1b"; then
            log "INFO" "Downloading gemma3:1b model (default model for DocuChat)..."
            ollama pull gemma3:1b
            log "SUCCESS" "Model downloaded successfully"
        else
            log "SUCCESS" "Ollama model already available"
        fi
    else
        log "INFO" "Skipping Ollama installation as requested"
    fi

    show_progress 6 6 "System tools installation completed"
    save_progress 3
    log "SUCCESS" "Phase 3 completed successfully"
}

# Phase 4: Python Environment Setup
phase4_python_setup() {
    log "HEADER" "Phase 4/6: Python Environment Setup"

    if [ "${PHASE_STATUS[3]}" = "true" ] && [ $RESUME_FROM -le 4 ]; then
        log "INFO" "Phase 4 already completed, skipping..."
        return 0
    fi

    show_progress 1 8 "Setting up Python environment..."

    # Create virtual environment
    show_progress 2 8 "Creating virtual environment..."
    if [ ! -d "venv" ]; then
        $PYTHON_CMD -m venv venv
        log "SUCCESS" "Virtual environment created"
    else
        log "SUCCESS" "Virtual environment already exists"
    fi

    # Activate virtual environment
    source venv/bin/activate

    # Upgrade pip
    show_progress 3 8 "Upgrading pip..."
    python -m pip install --upgrade pip setuptools wheel

    # Install requirements (use pyproject.toml as single source of truth)
    show_progress 4 8 "Installing Python dependencies (this may take 10-15 minutes)..."
    if [ -f "pyproject.toml" ]; then
        pip install -e .
        log "SUCCESS" "Dependencies installed from pyproject.toml"
    elif [ -f "requirements.txt" ]; then
        pip install -r requirements.txt
        log "SUCCESS" "Requirements installed from requirements.txt"
    else
        log "ERROR" "No requirements.txt or pyproject.toml found"
        exit 1
    fi

    # Install spaCy model
    show_progress 5 8 "Installing spaCy language model..."
    python -m spacy download en_core_web_sm

    # Install Playwright browsers
    show_progress 6 8 "Installing Playwright browsers..."
    if pip list | grep -q playwright; then
        playwright install
        log "SUCCESS" "Playwright browsers installed"
    else
        log "WARN" "Playwright not found in dependencies"
    fi

    # DocuChat package already installed above, skip redundant installation
    show_progress 7 8 "Verifying DocuChat installation..."
    if python -c "import docuchat" 2>/dev/null; then
        log "SUCCESS" "DocuChat package verified"
    else
        log "WARN" "DocuChat import failed, attempting reinstall..."
        pip install -e .
    fi

    show_progress 8 8 "Python environment setup completed"
    save_progress 4
    log "SUCCESS" "Phase 4 completed successfully"
}

# Phase 5: Service Configuration
phase5_configuration() {
    log "HEADER" "Phase 5/6: Service Configuration"

    if [ "${PHASE_STATUS[4]}" = "true" ] && [ $RESUME_FROM -le 5 ]; then
        log "INFO" "Phase 5 already completed, skipping..."
        return 0
    fi

    show_progress 1 6 "Configuring services..."

    # Ensure virtual environment is active
    if [ ! -f "venv/bin/activate" ]; then
        log "ERROR" "Virtual environment not found. Please run Phase 4 first."
        exit 1
    fi
    source venv/bin/activate

    # Create data directories
    show_progress 2 6 "Creating data directories..."
    mkdir -p data/{documents,feedback,graph_db,model_cache,vector_db}
    log "SUCCESS" "Data directories created"

    # Initialize ChromaDB
    show_progress 3 6 "Initializing ChromaDB..."
    python -c "
import chromadb
try:
    client = chromadb.PersistentClient(path='./chroma')
    print('ChromaDB initialized successfully')
except Exception as e:
    print(f'ChromaDB initialization failed: {e}')
    exit(1)
" || log "WARN" "ChromaDB initialization failed"

    # Test Neo4j connection
    show_progress 4 6 "Testing Neo4j connection..."
    python -c "
try:
    from neo4j import GraphDatabase
    driver = GraphDatabase.driver('bolt://localhost:7687', auth=('neo4j', 'docuchat123'))
    with driver.session() as session:
        result = session.run('RETURN 1 as test')
        print('Neo4j connection: SUCCESS')
    driver.close()
except Exception as e:
    print(f'Neo4j connection failed: {e}')
    exit(1)
" || log "WARN" "Neo4j connection test failed"

    # Generate hardware profile
    show_progress 5 6 "Generating hardware profile..."
    python -c "
import json
import psutil
import platform
from pathlib import Path

profile = {
    'cpu_count': psutil.cpu_count(),
    'memory_gb': psutil.virtual_memory().total / (1024**3),
    'platform': platform.platform(),
    'python_version': platform.python_version(),
    'setup_completed': True
}

with open('data/hardware_profile.json', 'w') as f:
    json.dump(profile, f, indent=2)

print('Hardware profile generated')
" || log "WARN" "Hardware profile generation failed"

    show_progress 6 6 "Configuration completed"
    save_progress 5
    log "SUCCESS" "Phase 5 completed successfully"
}

# Phase 6: Verification and Testing
phase6_verification() {
    log "HEADER" "Phase 6/6: Verification and Testing"

    if [ "${PHASE_STATUS[5]}" = "true" ] && [ $RESUME_FROM -le 6 ]; then
        log "INFO" "Phase 6 already completed, skipping..."
        return 0
    fi

    show_progress 1 8 "Running verification tests..."

    # Ensure virtual environment is active
    source venv/bin/activate

    # Test DocuChat CLI
    show_progress 2 8 "Testing DocuChat CLI..."
    if python -m docuchat --version >/dev/null 2>&1; then
        local version=$(python -m docuchat --version 2>&1)
        log "SUCCESS" "DocuChat CLI working: $version"
    else
        log "ERROR" "DocuChat CLI test failed"
        exit 1
    fi

    # Test system status
    show_progress 3 8 "Testing system status..."
    python -m docuchat status --check-dependencies >/dev/null 2>&1 || log "WARN" "Some dependencies may be missing"

    # Test Docker containers
    show_progress 4 8 "Checking Docker containers..."
    if docker ps | grep -q docuchat-neo4j; then
        log "SUCCESS" "Neo4j container is running"
    else
        log "WARN" "Neo4j container not running"
    fi

    # Test Ollama
    show_progress 5 8 "Testing Ollama..."
    if [ "$SKIP_OLLAMA" = "false" ]; then
        if curl -s http://localhost:11434/api/version >/dev/null 2>&1; then
            log "SUCCESS" "Ollama service is running"
        else
            log "WARN" "Ollama service not responding"
        fi
    fi

    # Create sample test
    show_progress 6 8 "Creating sample test files..."
    mkdir -p test_documents
    cat > test_documents/sample.txt << 'EOF'
# DocuChat Test Document

This is a sample document for testing DocuChat Agent.

## Key Features
- Document ingestion and processing
- Knowledge graph creation
- Natural language querying
- Local processing for privacy

## Technology Stack
- Python 3.10+
- Neo4j for graph database
- ChromaDB for vector storage
- Ollama for local LLM

This document can be used to test the ingestion pipeline.
EOF

    show_progress 7 8 "Running sample ingestion test..."
    if python -m docuchat ingest  self_tests/training_data --max-files 1 >/dev/null 2>&1; then
        log "SUCCESS" "Sample document ingestion test passed"
    else
        log "WARN" "Sample document ingestion test failed (this is non-critical)"
    fi

    show_progress 8 8 "Verification completed"
    save_progress 6
    log "SUCCESS" "Phase 6 completed successfully"
}

# =============================================================================
# MAIN EXECUTION FUNCTIONS
# =============================================================================

# Show help
show_help() {
    cat << EOF
${WHITE}$SCRIPT_NAME v$SCRIPT_VERSION${NC}

${CYAN}USAGE:${NC}
    $0 [OPTIONS]

${CYAN}OPTIONS:${NC}
    --auto             Run fully automated setup (no prompts)
    --interactive      Interactive mode with confirmations (default)
    --resume-from=N    Resume from specific phase (1-6)
    --skip-docker      Skip Docker/Neo4j setup (use existing)
    --skip-ollama      Skip Ollama installation (use existing)
    --help            Show this help message

${CYAN}PHASES:${NC}
    1. System Requirements Check
    2. Neo4j Database Setup
    3. System Tools Installation
    4. Python Environment Setup
    5. Service Configuration
    6. Verification and Testing

${CYAN}REQUIREMENTS:${NC}
    - Linux, macOS, or WSL2
    - 8GB+ RAM recommended
    - 10GB+ free disk space
    - Internet connection

${CYAN}EXAMPLES:${NC}
    $0                              # Interactive setup
    $0 --auto                       # Fully automated
    $0 --resume-from=3              # Resume from phase 3
    $0 --skip-docker --skip-ollama  # Skip external services

For more information, see: https://github.com/your-org/docuchat-agent
EOF
}

# Parse command line arguments
parse_args() {
    while [[ $# -gt 0 ]]; do
        case $1 in
            --auto)
                AUTO_MODE=true
                INTERACTIVE_MODE=false
                shift
                ;;
            --interactive)
                INTERACTIVE_MODE=true
                AUTO_MODE=false
                shift
                ;;
            --resume-from=*)
                RESUME_FROM="${1#*=}"
                if ! [[ "$RESUME_FROM" =~ ^[1-6]$ ]]; then
                    log "ERROR" "Invalid phase number. Must be 1-6."
                    exit 1
                fi
                shift
                ;;
            --skip-docker)
                SKIP_DOCKER=true
                shift
                ;;
            --skip-ollama)
                SKIP_OLLAMA=true
                shift
                ;;
            --help)
                show_help
                exit 0
                ;;
            *)
                log "ERROR" "Unknown option: $1"
                show_help
                exit 1
                ;;
        esac
    done
}

# Ask for confirmation in interactive mode
confirm() {
    local message=$1
    local default=${2:-"y"}

    if [ "$AUTO_MODE" = "true" ]; then
        return 0
    fi

    if [ "$INTERACTIVE_MODE" = "true" ]; then
        if [ "$default" = "y" ]; then
            echo -ne "${YELLOW}$message [Y/n]: ${NC}"
        else
            echo -ne "${YELLOW}$message [y/N]: ${NC}"
        fi

        read -r response
        case $response in
            [Yy]|[Yy][Ee][Ss]|"")
                if [ "$default" = "y" ]; then return 0; else return 1; fi
                ;;
            [Nn]|[Nn][Oo])
                if [ "$default" = "y" ]; then return 1; else return 0; fi
                ;;
            *)
                log "ERROR" "Please answer yes or no."
                confirm "$message" "$default"
                ;;
        esac
    fi

    return 0
}

# Show welcome message
show_welcome() {
    clear
    cat << 'EOF'
╔══════════════════════════════════════════════════════════════════════════════╗
║                                                                              ║
║  ██████╗  ██████╗  ██████╗██╗   ██╗ ██████╗██╗  ██╗ █████╗ ████████╗        ║
║  ██╔══██╗██╔═══██╗██╔════╝██║   ██║██╔════╝██║  ██║██╔══██╗╚══██╔══╝        ║
║  ██║  ██║██║   ██║██║     ██║   ██║██║     ███████║███████║   ██║           ║
║  ██║  ██║██║   ██║██║     ██║   ██║██║     ██╔══██║██╔══██║   ██║           ║
║  ██████╔╝╚██████╔╝╚██████╗╚██████╔╝╚██████╗██║  ██║██║  ██║   ██║           ║
║  ╚═════╝  ╚═════╝  ╚═════╝ ╚═════╝  ╚═════╝╚═╝  ╚═╝╚═╝  ╚═╝   ╚═╝           ║
║                       AGENT SETUP - Complete Installation                   ║
╚══════════════════════════════════════════════════════════════════════════════╝
EOF

    echo -e "\n${WHITE}Welcome to DocuChat Agent Setup!${NC}"
    echo -e "${CYAN}This script will install and configure everything needed to run DocuChat.${NC}"
    echo ""
    echo -e "${YELLOW}What will be installed:${NC}"
    echo -e "  ${CHECK_MARK} Docker & Neo4j database container"
    echo -e "  ${CHECK_MARK} Ollama local LLM server with models"
    echo -e "  ${CHECK_MARK} Python dependencies & virtual environment"
    echo -e "  ${CHECK_MARK} System tools (Tesseract OCR, etc.)"
    echo -e "  ${CHECK_MARK} DocuChat Agent package & configuration"
    echo ""
    echo -e "${YELLOW}Requirements:${NC}"
    echo -e "  • 8GB+ RAM recommended"
    echo -e "  • 10GB+ free disk space"
    echo -e "  • Internet connection"
    echo -e "  • Linux, macOS, or WSL2"
    echo ""

    if [ "$AUTO_MODE" = "false" ]; then
        if ! confirm "Do you want to continue with the installation?"; then
            log "INFO" "Installation cancelled by user."
            exit 0
        fi
    fi

    echo ""
}

# Show completion message
show_completion() {
    echo ""
    log "SUCCESS" "🎉 DocuChat Agent setup completed successfully!"
    echo ""
    echo -e "${GREEN}${ROCKET} Quick Start Guide:${NC}"
    echo ""
    echo -e "${WHITE}1. Activate the Python environment:${NC}"
    echo -e "   ${CYAN}source venv/bin/activate${NC}"
    echo ""
    echo -e "${WHITE}2. Test the installation:${NC}"
    echo -e "   ${CYAN}python -m docuchat --version${NC}"
    echo -e "   ${CYAN}python -m docuchat status${NC}"
    echo ""
    echo -e "${WHITE}3. Ingest some documents:${NC}"
    echo -e "   ${CYAN}python -m docuchat ingest ./test_documents${NC}"
    echo ""
    echo -e "${WHITE}4. Start chatting:${NC}"
    echo -e "   ${CYAN}python -m docuchat chat${NC}"
    echo ""
    echo -e "${YELLOW}Services Status:${NC}"
    echo -e "  • Neo4j Web UI: ${CYAN}http://localhost:7474${NC} (neo4j/docuchat123)"
    echo -e "  • Ollama API: ${CYAN}http://localhost:11434${NC}"
    echo ""
    echo -e "${YELLOW}Logs and Data:${NC}"
    echo -e "  • Setup log: ${CYAN}$LOG_FILE${NC}"
    echo -e "  • Data directory: ${CYAN}./data/${NC}"
    echo -e "  • Neo4j data: ${CYAN}./data/neo4j/${NC}"
    echo ""
    echo -e "${WHITE}For help and documentation, visit:${NC}"
    echo -e "  ${CYAN}https://github.com/your-org/docuchat-agent${NC}"
    echo ""

    # Cleanup
    rm -f "$CONFIG_FILE"
}

# Main execution function
main() {
    # Initialize
    echo "" > "$LOG_FILE"
    load_progress

    # Parse arguments
    parse_args "$@"

    # Show welcome
    show_welcome

    # Detect system
    detect_os
    detect_python

    # Execute phases
    local start_time=$(date +%s)

    if [ $RESUME_FROM -le 1 ]; then phase1_system_check; fi
    if [ $RESUME_FROM -le 2 ]; then phase2_neo4j_setup; fi
    if [ $RESUME_FROM -le 3 ]; then phase3_system_tools; fi
    if [ $RESUME_FROM -le 4 ]; then phase4_python_setup; fi
    if [ $RESUME_FROM -le 5 ]; then phase5_configuration; fi
    if [ $RESUME_FROM -le 6 ]; then phase6_verification; fi

    local end_time=$(date +%s)
    local duration=$((end_time - start_time))

    log "SUCCESS" "Total setup time: ${duration}s"

    # Show completion
    show_completion
}

# =============================================================================
# SCRIPT ENTRY POINT
# =============================================================================

# Handle script interruption
trap 'log "WARN" "Setup interrupted by user"; exit 130' INT TERM

# Check if running as root (not recommended)
if [ "$EUID" -eq 0 ]; then
    log "WARN" "Running as root is not recommended. Consider running as a regular user."
    if ! confirm "Continue anyway?" "n"; then
        exit 1
    fi
fi

# Run main function
main "$@"

# End of script