DevOps Engineer

Approach: Build a comprehensive monitoring and response system

Key Concepts: System monitoring, automated response, alerting, logging

Implementation:

#!/bin/bash

# incident_response.sh - Automated CPU monitoring and incident response

set -euo pipefail

# Configuration
readonly CPU_THRESHOLD=80
readonly CHECK_INTERVAL=30
readonly CONSECUTIVE_CHECKS=2
readonly LOG_FILE="/var/log/incident_response.log"
readonly ALERT_WEBHOOK="https://hooks.slack.com/services/YOUR/WEBHOOK/URL"

# Global variables
consecutive_high_cpu=0
incident_active=false
incident_id=""

# Logging function
log() {
    local level="$1"
    local message="$2"
    local timestamp=$(date '+%Y-%m-%d %H:%M:%S')
    echo "[$timestamp] [$level] $message" | tee -a "$LOG_FILE"
}

# Get current CPU usage
get_cpu_usage() {
    # Get CPU usage as percentage (100 - idle)
    top -bn1 | grep "Cpu(s)" | awk '{print $2}' | awk -F'%' '{print $1}'
}

# Send alert notification
send_alert() {
    local severity="$1"
    local message="$2"
    local color=""
    
    case "$severity" in
        "critical") color="danger" ;;
        "warning") color="warning" ;;
        "resolved") color="good" ;;
    esac
    
    # Send to Slack webhook
    curl -X POST -H 'Content-type: application/json' \
        --data "{
            \"attachments\": [{
                \"color\": \"$color\",
                \"title\": \"[$severity] CPU Incident\",
                \"text\": \"$message\",
                \"fields\": [{
                    \"title\": \"Host\",
                    \"value\": \"$(hostname)\",
                    \"short\": true
                }, {
                    \"title\": \"Time\",
                    \"value\": \"$(date)\",
                    \"short\": true
                }]
            }]
        }" "$ALERT_WEBHOOK" 2>/dev/null || log "ERROR" "Failed to send alert"
}

# Automated remediation attempts
attempt_remediation() {
    local incident_id="$1"
    log "INFO" "Starting automated remediation for incident $incident_id"
    
    # 1. Identify top CPU consuming processes
    local top_processes=$(ps aux --sort=-%cpu | head -10)
    log "INFO" "Top CPU processes: $top_processes"
    
    # 2. Check for known problematic processes and restart services
    if pgrep -f "runaway_process" > /dev/null; then
        log "WARNING" "Detected runaway_process, attempting restart"
        sudo systemctl restart runaway_service || log "ERROR" "Failed to restart runaway_service"
    fi
    
    # 3. Clear temporary files and caches
    log "INFO" "Clearing system caches"
    sudo sync && echo 3 | sudo tee /proc/sys/vm/drop_caches > /dev/null
    
    # 4. Check disk space (high disk usage can cause CPU spikes)
    local disk_usage=$(df / | awk 'NR==2 {print $5}' | sed 's/%//')
    if [ "$disk_usage" -gt 90 ]; then
        log "WARNING" "High disk usage detected: ${disk_usage}%"
        # Clean old logs
        find /var/log -name "*.log" -mtime +7 -delete 2>/dev/null || true
    fi
    
    # 5. Restart monitoring agents if they're consuming too much CPU
    if pgrep -f "monitoring_agent" > /dev/null; then
        local agent_cpu=$(ps -C monitoring_agent -o %cpu --no-headers | awk '{sum+=$1} END {print sum}')
        if (( $(echo "$agent_cpu > 20" | bc -l) )); then
            log "WARNING" "Monitoring agent using high CPU, restarting"
            sudo systemctl restart monitoring_agent
        fi
    fi
    
    log "INFO" "Remediation attempts completed for incident $incident_id"
}

# Generate incident ID
generate_incident_id() {
    echo "INC-$(date +%Y%m%d%H%M%S)-$(hostname -s)"
}

# Main monitoring loop
monitor_cpu() {
    log "INFO" "Starting CPU monitoring (threshold: ${CPU_THRESHOLD}%, checks: ${CONSECUTIVE_CHECKS})"
    
    while true; do
        local current_cpu
        current_cpu=$(get_cpu_usage)
        
        # Remove any decimal points for comparison
        current_cpu=${current_cpu%.*}
        
        log "DEBUG" "Current CPU usage: ${current_cpu}%"
        
        if [ "$current_cpu" -gt "$CPU_THRESHOLD" ]; then
            ((consecutive_high_cpu++))
            log "WARNING" "High CPU detected: ${current_cpu}% (${consecutive_high_cpu}/${CONSECUTIVE_CHECKS})"
            
            if [ "$consecutive_high_cpu" -ge "$CONSECUTIVE_CHECKS" ] && [ "$incident_active" = false ]; then
                # Start new incident
                incident_id=$(generate_incident_id)
                incident_active=true
                
                log "CRITICAL" "CPU incident triggered: $incident_id (CPU: ${current_cpu}%)"
                send_alert "critical" "High CPU usage detected: ${current_cpu}% for ${CONSECUTIVE_CHECKS} consecutive checks. Incident ID: $incident_id"
                
                # Attempt automated remediation
                attempt_remediation "$incident_id"
            fi
        else
            if [ "$incident_active" = true ]; then
                # Incident resolved
                log "INFO" "CPU incident resolved: $incident_id (CPU: ${current_cpu}%)"
                send_alert "resolved" "CPU usage normalized: ${current_cpu}%. Incident $incident_id resolved."
                incident_active=false
                incident_id=""
            fi
            consecutive_high_cpu=0
        fi
        
        sleep "$CHECK_INTERVAL"
    done
}

# Signal handlers for graceful shutdown
cleanup() {
    log "INFO" "Shutting down CPU monitoring"
    exit 0
}

trap cleanup SIGINT SIGTERM

# Ensure log directory exists
mkdir -p "$(dirname "$LOG_FILE")"

# Check dependencies
command -v bc >/dev/null 2>&1 || { log "ERROR" "bc is required but not installed"; exit 1; }
command -v curl >/dev/null 2>&1 || { log "ERROR" "curl is required but not installed"; exit 1; }

# Start monitoring
log "INFO" "=== CPU Incident Response System Starting ==="
monitor_cpu

Key Points to Explain:

• Comprehensive monitoring with configurable thresholds and timeouts
• Automated remediation attempts before escalating to human operators
• Structured logging for post-incident analysis and debugging
• External alerting integration (Slack, PagerDuty, etc.)
• Graceful error handling and dependency checking
• Incident tracking with unique IDs for correlation across systems