AI Newsletter Digest improvements: fixed QP soft line break decoding, URL extraction, and content cleaning
This commit is contained in:
Krilly
+170
@@ -0,0 +1,170 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
# Emergency Recovery Monitor - Discord 알림
|
||||
# emergency-recovery 로그에서 실패 케이스 감지 → Discord 알림
|
||||
|
||||
# ============================================
|
||||
# Configuration (Override via environment)
|
||||
# ============================================
|
||||
LOG_DIR="${OPENCLAW_MEMORY_DIR:-$HOME/openclaw/memory}"
|
||||
ALERT_SENT_FILE="$LOG_DIR/.emergency-alert-sent"
|
||||
ALERT_WINDOW_MINUTES="${EMERGENCY_ALERT_WINDOW:-30}"
|
||||
|
||||
# Create log directory if not exists
|
||||
mkdir -p "$LOG_DIR"
|
||||
|
||||
# Load environment variables
|
||||
if [ -f "$HOME/openclaw/.env" ]; then
|
||||
# shellcheck source=/dev/null
|
||||
source "$HOME/openclaw/.env"
|
||||
elif [ -f "$HOME/.openclaw/.env" ]; then
|
||||
# shellcheck source=/dev/null
|
||||
source "$HOME/.openclaw/.env"
|
||||
fi
|
||||
|
||||
# Discord webhook from environment variable (optional)
|
||||
DISCORD_WEBHOOK="${DISCORD_WEBHOOK_URL:-}"
|
||||
|
||||
# Cleanup on exit
|
||||
trap 'rm -f /tmp/emergency-alert.txt' EXIT
|
||||
|
||||
# ============================================
|
||||
# Functions
|
||||
# ============================================
|
||||
|
||||
log() {
|
||||
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1"
|
||||
}
|
||||
|
||||
get_latest_recovery_log() {
|
||||
# Use find instead of ls (ShellCheck SC2012)
|
||||
find "$LOG_DIR" -name "emergency-recovery-*.log" -type f -print0 2>/dev/null | \
|
||||
xargs -0 ls -t 2>/dev/null | head -1
|
||||
}
|
||||
|
||||
is_alert_already_sent() {
|
||||
local latest_log="$1"
|
||||
|
||||
if [ ! -f "$ALERT_SENT_FILE" ]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
local sent_log
|
||||
sent_log=$(cat "$ALERT_SENT_FILE" 2>/dev/null || echo "")
|
||||
|
||||
[ "$sent_log" = "$latest_log" ]
|
||||
}
|
||||
|
||||
mark_alert_sent() {
|
||||
local latest_log="$1"
|
||||
|
||||
# Atomic write
|
||||
echo "$latest_log" > "$ALERT_SENT_FILE.tmp"
|
||||
mv "$ALERT_SENT_FILE.tmp" "$ALERT_SENT_FILE"
|
||||
}
|
||||
|
||||
send_alert() {
|
||||
local latest_log="$1"
|
||||
local timestamp
|
||||
timestamp=$(basename "$latest_log" | sed 's/emergency-recovery-//;s/.log//')
|
||||
|
||||
# Discord 알림 메시지 생성
|
||||
cat > /tmp/emergency-alert.txt << EOF
|
||||
🚨 **긴급: OpenClaw 자가복구 실패**
|
||||
|
||||
**시간:** $timestamp
|
||||
**상태:**
|
||||
- Level 1 (Watchdog) ❌
|
||||
- Level 2 (Health Check) ❌
|
||||
- Level 3 (Claude Recovery) ❌
|
||||
|
||||
**수동 개입 필요합니다.**
|
||||
|
||||
**로그:**
|
||||
- \`$latest_log\`
|
||||
- \`$LOG_DIR/claude-session-$timestamp.log\`
|
||||
- \`$LOG_DIR/emergency-recovery-report-$timestamp.md\` (Claude가 생성했을 경우)
|
||||
|
||||
**복구 시도:**
|
||||
1. \`openclaw status\` 확인
|
||||
2. \`~/.openclaw/logs/*.log\` 에러 확인
|
||||
3. \`openclaw gateway restart\` 시도
|
||||
4. 필요 시 \`openclaw gateway stop && sleep 5 && openclaw gateway start\`
|
||||
EOF
|
||||
|
||||
local alert_msg
|
||||
alert_msg=$(cat /tmp/emergency-alert.txt)
|
||||
|
||||
# Discord 직접 호출 (webhook 있을 경우)
|
||||
if [ -n "$DISCORD_WEBHOOK" ]; then
|
||||
local response_code
|
||||
response_code=$(curl -s -o /dev/null -w "%{http_code}" \
|
||||
-X POST "$DISCORD_WEBHOOK" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"content\": \"$alert_msg\"}" \
|
||||
2>&1 || echo "000")
|
||||
|
||||
if [ "$response_code" = "200" ] || [ "$response_code" = "204" ]; then
|
||||
log "✅ Discord notification sent (HTTP $response_code)"
|
||||
else
|
||||
log "⚠️ Discord notification failed (HTTP $response_code), falling back to stdout"
|
||||
cat /tmp/emergency-alert.txt
|
||||
fi
|
||||
else
|
||||
# Webhook 없으면 stdout 출력 (크론이 message tool로 전달)
|
||||
log "INFO: DISCORD_WEBHOOK_URL not set, printing to stdout"
|
||||
cat /tmp/emergency-alert.txt
|
||||
fi
|
||||
}
|
||||
|
||||
# ============================================
|
||||
# Main Logic
|
||||
# ============================================
|
||||
|
||||
main() {
|
||||
# 최근 N분 내 emergency-recovery 로그 찾기
|
||||
local recent_logs
|
||||
recent_logs=$(find "$LOG_DIR" -name "emergency-recovery-*.log" -type f -mmin -"$ALERT_WINDOW_MINUTES" 2>/dev/null)
|
||||
|
||||
if [ -z "$recent_logs" ]; then
|
||||
# 최근 emergency recovery 없음
|
||||
log "No recent emergency recovery logs found (last ${ALERT_WINDOW_MINUTES} minutes)"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# 가장 최근 로그 확인
|
||||
local latest_log
|
||||
latest_log=$(get_latest_recovery_log)
|
||||
|
||||
if [ -z "$latest_log" ] || [ ! -f "$latest_log" ]; then
|
||||
log "No valid emergency recovery logs found"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# 이미 알림 보낸 로그인지 체크
|
||||
if is_alert_already_sent "$latest_log"; then
|
||||
log "Alert already sent for: $latest_log"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# "MANUAL INTERVENTION REQUIRED" 패턴 검색
|
||||
if grep -q "MANUAL INTERVENTION REQUIRED" "$latest_log"; then
|
||||
log "Found failed recovery in: $latest_log"
|
||||
|
||||
# 알림 전송
|
||||
send_alert "$latest_log"
|
||||
|
||||
# 알림 보냄 기록
|
||||
mark_alert_sent "$latest_log"
|
||||
|
||||
exit 0
|
||||
else
|
||||
log "No manual intervention required in: $latest_log"
|
||||
fi
|
||||
|
||||
exit 0
|
||||
}
|
||||
|
||||
# Run main function
|
||||
main
|
||||
@@ -0,0 +1,341 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
# OpenClaw Emergency Recovery (Level 3 Self-Healing)
|
||||
# Claude Code PTY 세션으로 자동 진단 및 복구 시도
|
||||
#
|
||||
# NOTE: Claude is started with --permission-mode dontAsk to skip the
|
||||
# workspace trust prompt. Without this flag, Claude hangs waiting for
|
||||
# user input in the tmux session, causing recovery to fail.
|
||||
|
||||
# ============================================
|
||||
# Configuration (Override via environment)
|
||||
# ============================================
|
||||
RECOVERY_TIMEOUT="${EMERGENCY_RECOVERY_TIMEOUT:-1800}" # 30분
|
||||
GATEWAY_URL="${OPENCLAW_GATEWAY_URL:-http://localhost:18789/}"
|
||||
LOG_DIR="${OPENCLAW_MEMORY_DIR:-$HOME/openclaw/memory}"
|
||||
CLAUDE_WORKSPACE_TRUST_TIMEOUT="${CLAUDE_WORKSPACE_TRUST_TIMEOUT:-10}"
|
||||
CLAUDE_STARTUP_WAIT="${CLAUDE_STARTUP_WAIT:-5}"
|
||||
WORKSPACE_TRUST_CONFIRM_WAIT="${WORKSPACE_TRUST_CONFIRM_WAIT:-3}"
|
||||
|
||||
TIMESTAMP=$(date +%Y-%m-%d-%H%M)
|
||||
LOG_FILE="$LOG_DIR/emergency-recovery-$TIMESTAMP.log"
|
||||
REPORT_FILE="$LOG_DIR/emergency-recovery-report-$TIMESTAMP.md"
|
||||
SESSION_LOG="$LOG_DIR/claude-session-$TIMESTAMP.log"
|
||||
TMUX_SESSION="emergency_recovery_$TIMESTAMP"
|
||||
|
||||
# Performance metrics
|
||||
METRICS_FILE="$LOG_DIR/.emergency-recovery-metrics.json"
|
||||
|
||||
# Create log directory if not exists
|
||||
mkdir -p "$LOG_DIR"
|
||||
|
||||
# Load environment variables
|
||||
if [ -f "$HOME/openclaw/.env" ]; then
|
||||
# shellcheck source=/dev/null
|
||||
source "$HOME/openclaw/.env"
|
||||
elif [ -f "$HOME/.openclaw/.env" ]; then
|
||||
# shellcheck source=/dev/null
|
||||
source "$HOME/.openclaw/.env"
|
||||
fi
|
||||
|
||||
# Discord webhook from environment variable (optional)
|
||||
DISCORD_WEBHOOK="${DISCORD_WEBHOOK_URL:-}"
|
||||
|
||||
if [ -z "$DISCORD_WEBHOOK" ]; then
|
||||
echo "INFO: DISCORD_WEBHOOK_URL not set. Notifications disabled."
|
||||
fi
|
||||
|
||||
# ============================================
|
||||
# Functions
|
||||
# ============================================
|
||||
|
||||
log() {
|
||||
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG_FILE"
|
||||
}
|
||||
|
||||
send_discord_notification() {
|
||||
local message="$1"
|
||||
if [ -n "$DISCORD_WEBHOOK" ]; then
|
||||
local response_code
|
||||
response_code=$(curl -s -o /dev/null -w "%{http_code}" \
|
||||
-X POST "$DISCORD_WEBHOOK" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"content\": \"$message\"}" \
|
||||
2>&1 || echo "000")
|
||||
|
||||
if [ "$response_code" = "200" ] || [ "$response_code" = "204" ]; then
|
||||
log "✅ Discord notification sent (HTTP $response_code)"
|
||||
else
|
||||
log "⚠️ Discord notification failed (HTTP $response_code)"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
check_dependencies() {
|
||||
local missing_deps=()
|
||||
|
||||
if ! command -v tmux &> /dev/null; then
|
||||
missing_deps+=("tmux")
|
||||
fi
|
||||
|
||||
if ! command -v claude &> /dev/null; then
|
||||
missing_deps+=("claude")
|
||||
fi
|
||||
|
||||
if [ ${#missing_deps[@]} -gt 0 ]; then
|
||||
log "❌ Missing dependencies: ${missing_deps[*]}"
|
||||
send_discord_notification "🚨 **Level 3 Emergency Recovery 실패**\n\n필수 의존성이 설치되지 않았습니다:\n- ${missing_deps[*]}\n\n설치 방법:\n\`\`\`bash\nbrew install ${missing_deps[*]}\n\`\`\`"
|
||||
return 1
|
||||
fi
|
||||
|
||||
log "✅ Dependencies check passed"
|
||||
return 0
|
||||
}
|
||||
|
||||
wait_for_claude_prompt() {
|
||||
local session="$1"
|
||||
local timeout="$2"
|
||||
|
||||
log "Waiting for Claude workspace trust prompt (timeout: ${timeout}s)..."
|
||||
|
||||
for _ in $(seq 1 "$timeout"); do
|
||||
local output
|
||||
output=$(tmux capture-pane -t "$session" -p 2>/dev/null || echo "")
|
||||
|
||||
if echo "$output" | grep -q "trust this workspace"; then
|
||||
log "✅ Claude workspace trust prompt detected"
|
||||
return 0
|
||||
fi
|
||||
|
||||
sleep 1
|
||||
done
|
||||
|
||||
log "⚠️ Claude workspace trust prompt not detected after ${timeout}s"
|
||||
return 1
|
||||
}
|
||||
|
||||
capture_tmux_session() {
|
||||
local session="$1"
|
||||
local output_file="$2"
|
||||
|
||||
if tmux capture-pane -t "$session" -p > "$output_file" 2>/dev/null; then
|
||||
log "✅ tmux session captured: $output_file"
|
||||
return 0
|
||||
else
|
||||
log "⚠️ Failed to capture tmux session"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
check_claude_quota() {
|
||||
local session_log="$1"
|
||||
|
||||
if grep -qE "rate limit|quota exceeded|429|too many requests" "$session_log"; then
|
||||
log "⚠️ Claude API rate limited or quota exceeded"
|
||||
return 1
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
rotate_old_logs() {
|
||||
local deleted_count
|
||||
deleted_count=$(find "$LOG_DIR" -name "emergency-recovery-*.log" -mtime +14 -delete -print 2>/dev/null | wc -l)
|
||||
deleted_count=$((deleted_count + $(find "$LOG_DIR" -name "claude-session-*.log" -mtime +14 -delete -print 2>/dev/null | wc -l)))
|
||||
|
||||
if [ "$deleted_count" -gt 0 ]; then
|
||||
log "Rotated $deleted_count old log files"
|
||||
fi
|
||||
}
|
||||
|
||||
record_metric() {
|
||||
local metric_name="$1"
|
||||
local result="$2"
|
||||
local duration="$3"
|
||||
local timestamp
|
||||
timestamp=$(date +%s)
|
||||
|
||||
# Append to metrics file (JSON Lines format)
|
||||
echo "{\"timestamp\":$timestamp,\"metric\":\"$metric_name\",\"result\":\"$result\",\"duration\":$duration}" >> "$METRICS_FILE"
|
||||
}
|
||||
|
||||
cleanup_tmux_session() {
|
||||
local session="$1"
|
||||
|
||||
if tmux has-session -t "$session" 2>/dev/null; then
|
||||
log "Terminating tmux session: $session"
|
||||
tmux kill-session -t "$session" 2>/dev/null || true
|
||||
fi
|
||||
}
|
||||
|
||||
# ============================================
|
||||
# Main Recovery Logic
|
||||
# ============================================
|
||||
|
||||
main() {
|
||||
local start_time
|
||||
start_time=$(date +%s)
|
||||
|
||||
log "=== Emergency Recovery Started (PID: $$) ==="
|
||||
|
||||
# 0. Log rotation
|
||||
rotate_old_logs
|
||||
|
||||
# 1. Check dependencies
|
||||
if ! check_dependencies; then
|
||||
log "🚨 Cannot proceed without required dependencies"
|
||||
record_metric "emergency_recovery" "dependency_failed" 0
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 2. Claude Code PTY 세션 시작
|
||||
log "Starting Claude Code session in tmux..."
|
||||
|
||||
if ! tmux new-session -d -s "$TMUX_SESSION" "claude --permission-mode dontAsk" 2>> "$LOG_FILE"; then
|
||||
log "❌ Failed to start tmux session"
|
||||
send_discord_notification "🚨 **Level 3 실패**\n\ntmux 세션 시작 실패.\n\n수동 개입 필요:\n\`$LOG_FILE\`"
|
||||
record_metric "emergency_recovery" "tmux_failed" 0
|
||||
exit 1
|
||||
fi
|
||||
|
||||
sleep "$CLAUDE_STARTUP_WAIT"
|
||||
|
||||
# 3. 워크스페이스 신뢰 (프롬프트 감지)
|
||||
if wait_for_claude_prompt "$TMUX_SESSION" "$CLAUDE_WORKSPACE_TRUST_TIMEOUT"; then
|
||||
log "Trusting workspace..."
|
||||
tmux send-keys -t "$TMUX_SESSION" "" C-m
|
||||
sleep "$WORKSPACE_TRUST_CONFIRM_WAIT"
|
||||
else
|
||||
log "⚠️ Proceeding without workspace trust confirmation"
|
||||
fi
|
||||
|
||||
# 4. 긴급 복구 명령 전송
|
||||
log "Sending emergency recovery command to Claude..."
|
||||
|
||||
local recovery_command
|
||||
recovery_command="OpenClaw 게이트웨이가 5분간 재시작했으나 복구되지 않았습니다. 긴급 진단 및 복구를 시작하세요.
|
||||
|
||||
작업 순서:
|
||||
1. \`openclaw status\` 체크
|
||||
2. 로그 분석 (~/.openclaw/logs/*.log)
|
||||
3. 설정 검증 (~/.openclaw/openclaw.json)
|
||||
4. 포트 충돌 체크 (\`lsof -i :18789\`)
|
||||
5. 의존성 체크 (\`npm list\`, \`node --version\`)
|
||||
6. 복구 시도 (설정 수정, 프로세스 재시작)
|
||||
7. 결과를 $REPORT_FILE 에 기록
|
||||
|
||||
작업 제한시간: ${RECOVERY_TIMEOUT}초 이내
|
||||
목표: Gateway가 $GATEWAY_URL 에서 HTTP 200 응답하도록 복구"
|
||||
|
||||
if ! tmux send-keys -t "$TMUX_SESSION" "$recovery_command" C-m 2>> "$LOG_FILE"; then
|
||||
log "❌ Failed to send command to Claude"
|
||||
cleanup_tmux_session "$TMUX_SESSION"
|
||||
send_discord_notification "🚨 **Level 3 실패**\n\nClaude 명령 전송 실패.\n\n수동 개입 필요:\n\`$LOG_FILE\`"
|
||||
record_metric "emergency_recovery" "command_failed" 0
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 5. Claude 작업 대기
|
||||
log "Waiting ${RECOVERY_TIMEOUT}s for Claude to complete recovery..."
|
||||
sleep "$RECOVERY_TIMEOUT"
|
||||
|
||||
# 6. tmux 세션 캡처
|
||||
log "Capturing Claude session output..."
|
||||
capture_tmux_session "$TMUX_SESSION" "$SESSION_LOG"
|
||||
|
||||
# 7. Claude 할당량 체크
|
||||
local SUCCESS="unknown"
|
||||
|
||||
if ! check_claude_quota "$SESSION_LOG"; then
|
||||
send_discord_notification "⚠️ **Level 3 Emergency Recovery 실패**\n\nClaude API 할당량 소진 또는 속도 제한.\n\n다음 단계:\n1. Claude 할당량 확인: \`claude\` → \`/usage\`\n2. 수동 복구 시도\n\n세션 로그: \`$SESSION_LOG\`"
|
||||
SUCCESS="false"
|
||||
fi
|
||||
|
||||
# 8. 결과 확인
|
||||
log "Checking recovery result..."
|
||||
|
||||
local http_code
|
||||
http_code=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 "$GATEWAY_URL" 2>/dev/null || echo "000")
|
||||
|
||||
if [ "$http_code" = "200" ] && [ "$SUCCESS" != "false" ]; then
|
||||
log "✅ Claude successfully recovered the gateway! (HTTP $http_code)"
|
||||
SUCCESS="true"
|
||||
else
|
||||
log "❌ Gateway still unhealthy after Claude recovery (HTTP $http_code)"
|
||||
SUCCESS="false"
|
||||
fi
|
||||
|
||||
# 9. tmux 세션 종료
|
||||
cleanup_tmux_session "$TMUX_SESSION"
|
||||
|
||||
# 10. Performance metrics
|
||||
local end_time
|
||||
end_time=$(date +%s)
|
||||
local total_time=$((end_time - start_time))
|
||||
record_metric "emergency_recovery" "$SUCCESS" "$total_time"
|
||||
|
||||
# 11. Discord 알림 및 종료
|
||||
log "=== Emergency Recovery Completed (${total_time}s) ==="
|
||||
|
||||
if [ "$SUCCESS" = "true" ]; then
|
||||
log "✅ Sending success notification to Discord..."
|
||||
send_discord_notification "✅ **Level 3 Emergency Recovery 성공!**\n\nGateway가 Claude에 의해 복구되었습니다.\n- 복구 시간: ${total_time}초\n- HTTP 상태: $http_code\n- 로그: \`$LOG_FILE\`\n- Claude 세션: \`$SESSION_LOG\`"
|
||||
exit 0
|
||||
else
|
||||
log "🚨 Sending failure notification to Discord..."
|
||||
|
||||
local failure_msg
|
||||
failure_msg="🚨 **Level 3 Emergency Recovery 실패!**\n\n**모든 자동 복구 시스템이 실패했습니다:**\n- Level 1 (Watchdog): ❌\n- Level 2 (Health Check): ❌\n- Level 3 (Claude Recovery): ❌\n\n**수동 개입 필요**\n- HTTP 상태: $http_code\n- 복구 시간: ${total_time}초\n- 로그: \`$LOG_FILE\`\n- Claude 세션: \`$SESSION_LOG\`\n- 복구 리포트: \`$REPORT_FILE\` (Claude가 생성했을 경우)"
|
||||
|
||||
send_discord_notification "$failure_msg"
|
||||
|
||||
# 로그에도 기록
|
||||
cat >> "$LOG_FILE" << EOF
|
||||
|
||||
=== MANUAL INTERVENTION REQUIRED ===
|
||||
Level 1 (Watchdog) ❌
|
||||
Level 2 (Health Check) ❌
|
||||
Level 3 (Claude Recovery) ❌
|
||||
|
||||
수동 개입 필요합니다.
|
||||
복구 시간: ${total_time}초
|
||||
로그: $LOG_FILE
|
||||
Claude 세션: $SESSION_LOG
|
||||
복구 리포트: $REPORT_FILE (Claude가 생성했을 경우)
|
||||
EOF
|
||||
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Run main function
|
||||
main
|
||||
|
||||
# ============================================
|
||||
# v1.1.0: Incident Documentation (ContextVault feedback)
|
||||
# ============================================
|
||||
log_incident() {
|
||||
local incident_file="${OPENCLAW_DIR}/memory/incidents/$(date +%Y-%m-%d_%H%M%S).md"
|
||||
mkdir -p "$(dirname "$incident_file")"
|
||||
|
||||
cat > "$incident_file" << EOF
|
||||
# Incident Report - $(date '+%Y-%m-%d %H:%M:%S')
|
||||
|
||||
## Trigger
|
||||
- Health check failed 3 times
|
||||
- Last error: $(tail -5 "${OPENCLAW_DIR}/logs/gateway.log" 2>/dev/null | head -3)
|
||||
|
||||
## Claude Diagnosis
|
||||
$(tmux capture-pane -t emergency-recovery -p 2>/dev/null | tail -50)
|
||||
|
||||
## Resolution
|
||||
- Status: $1
|
||||
- Duration: $2 seconds
|
||||
|
||||
## Prevention
|
||||
- TODO: Add prevention steps based on root cause
|
||||
EOF
|
||||
|
||||
echo "📝 Incident logged: $incident_file"
|
||||
}
|
||||
@@ -0,0 +1,219 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
# OpenClaw Gateway Health Check (Level 2 Self-Healing)
|
||||
# HTTP 응답 검증 → 실패 시 재시작 → 5분 후 재검증 → 실패 시 Level 3 escalation
|
||||
|
||||
# ============================================
|
||||
# Configuration (Override via environment)
|
||||
# ============================================
|
||||
GATEWAY_URL="${OPENCLAW_GATEWAY_URL:-http://localhost:18789/}"
|
||||
MAX_RETRIES="${HEALTH_CHECK_MAX_RETRIES:-3}"
|
||||
RETRY_DELAY="${HEALTH_CHECK_RETRY_DELAY:-30}"
|
||||
ESCALATION_WAIT="${HEALTH_CHECK_ESCALATION_WAIT:-300}"
|
||||
LOG_DIR="${OPENCLAW_MEMORY_DIR:-$HOME/openclaw/memory}"
|
||||
LOG_FILE="$LOG_DIR/healthcheck-$(date +%Y-%m-%d).log"
|
||||
HTTP_TIMEOUT="${HEALTH_CHECK_HTTP_TIMEOUT:-10}"
|
||||
|
||||
# Performance metrics
|
||||
METRICS_FILE="$LOG_DIR/.healthcheck-metrics.json"
|
||||
|
||||
# Lock file로 중복 실행 방지
|
||||
LOCKFILE=/tmp/openclaw-healthcheck.lock
|
||||
if [ -f "$LOCKFILE" ]; then
|
||||
echo "[$(date '+%Y-%m-%d %H:%M:%S')] Previous health check still running, skipping..."
|
||||
exit 0
|
||||
fi
|
||||
touch "$LOCKFILE"
|
||||
trap 'rm -f "$LOCKFILE"' EXIT
|
||||
|
||||
# Create log directory if not exists
|
||||
mkdir -p "$LOG_DIR"
|
||||
|
||||
# Load environment variables
|
||||
if [ -f "$HOME/openclaw/.env" ]; then
|
||||
# shellcheck source=/dev/null
|
||||
source "$HOME/openclaw/.env"
|
||||
elif [ -f "$HOME/.openclaw/.env" ]; then
|
||||
# shellcheck source=/dev/null
|
||||
source "$HOME/.openclaw/.env"
|
||||
fi
|
||||
|
||||
# Discord webhook from environment variable (optional)
|
||||
DISCORD_WEBHOOK="${DISCORD_WEBHOOK_URL:-}"
|
||||
|
||||
# Validate webhook URL (optional, warning only)
|
||||
if [ -z "$DISCORD_WEBHOOK" ]; then
|
||||
echo "[$(date '+%Y-%m-%d %H:%M:%S')] INFO: DISCORD_WEBHOOK_URL not set. Notifications disabled." | tee -a "$LOG_FILE"
|
||||
fi
|
||||
|
||||
# ============================================
|
||||
# Functions
|
||||
# ============================================
|
||||
|
||||
log() {
|
||||
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG_FILE"
|
||||
}
|
||||
|
||||
send_discord_notification() {
|
||||
local message="$1"
|
||||
if [ -n "$DISCORD_WEBHOOK" ]; then
|
||||
local response_code
|
||||
response_code=$(curl -s -o /dev/null -w "%{http_code}" \
|
||||
-X POST "$DISCORD_WEBHOOK" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"content\": \"$message\"}" \
|
||||
2>&1)
|
||||
|
||||
if [ "$response_code" = "200" ] || [ "$response_code" = "204" ]; then
|
||||
log "✅ Discord notification sent (HTTP $response_code)"
|
||||
else
|
||||
log "⚠️ Discord notification failed (HTTP $response_code)"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
check_http() {
|
||||
local start_time
|
||||
start_time=$(date +%s)
|
||||
|
||||
local http_code
|
||||
http_code=$(curl -s -o /dev/null -w "%{http_code}" \
|
||||
--max-time "$HTTP_TIMEOUT" \
|
||||
"$GATEWAY_URL" 2>/dev/null || echo "000")
|
||||
|
||||
local end_time
|
||||
end_time=$(date +%s)
|
||||
local response_time=$((end_time - start_time))
|
||||
|
||||
# Record metric
|
||||
record_metric "http_check" "$http_code" "$response_time"
|
||||
|
||||
if [ "$http_code" = "200" ]; then
|
||||
log "HTTP check passed (${response_time}s)"
|
||||
return 0
|
||||
else
|
||||
log "HTTP check failed: HTTP $http_code (${response_time}s)"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
restart_gateway() {
|
||||
log "Restarting OpenClaw Gateway..."
|
||||
|
||||
local start_time
|
||||
start_time=$(date +%s)
|
||||
|
||||
if openclaw gateway restart >> "$LOG_FILE" 2>&1; then
|
||||
local end_time
|
||||
end_time=$(date +%s)
|
||||
local restart_time=$((end_time - start_time))
|
||||
|
||||
log "Gateway restart completed (${restart_time}s)"
|
||||
record_metric "gateway_restart" "success" "$restart_time"
|
||||
|
||||
sleep "$RETRY_DELAY"
|
||||
return 0
|
||||
else
|
||||
log "⚠️ Gateway restart command failed"
|
||||
record_metric "gateway_restart" "failed" 0
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
rotate_old_logs() {
|
||||
# Delete logs older than 14 days
|
||||
local deleted_count
|
||||
deleted_count=$(find "$LOG_DIR" -name "healthcheck-*.log" -mtime +14 -delete -print 2>/dev/null | wc -l)
|
||||
|
||||
if [ "$deleted_count" -gt 0 ]; then
|
||||
log "Rotated $deleted_count old log files"
|
||||
fi
|
||||
}
|
||||
|
||||
record_metric() {
|
||||
local metric_name="$1"
|
||||
local result="$2"
|
||||
local duration="$3"
|
||||
local timestamp
|
||||
timestamp=$(date +%s)
|
||||
|
||||
# Append to metrics file (JSON Lines format)
|
||||
echo "{\"timestamp\":$timestamp,\"metric\":\"$metric_name\",\"result\":\"$result\",\"duration\":$duration}" >> "$METRICS_FILE"
|
||||
}
|
||||
|
||||
escalate_to_level3() {
|
||||
log "🚨 Still unhealthy after ${ESCALATION_WAIT}s, triggering emergency recovery..."
|
||||
|
||||
# Discord 알림 (Level 3 시작)
|
||||
send_discord_notification "🚨 **Level 3 Emergency Recovery 시작**\n\n${ESCALATION_WAIT}초 대기 후에도 Gateway 복구 안 됨.\nClaude가 자동으로 진단 및 복구를 시도합니다.\n\n예상 소요 시간: 30분\n현재 시각: $(date '+%Y-%m-%d %H:%M:%S')"
|
||||
|
||||
local emergency_script="$HOME/openclaw/scripts/emergency-recovery.sh"
|
||||
|
||||
if [ -f "$emergency_script" ]; then
|
||||
bash "$emergency_script"
|
||||
else
|
||||
log "❌ Emergency recovery script not found: $emergency_script"
|
||||
send_discord_notification "🚨 **Level 3 실행 실패**\n\nEmergency recovery script not found:\n\`$emergency_script\`\n\n수동 개입 필요."
|
||||
fi
|
||||
}
|
||||
|
||||
# ============================================
|
||||
# Main Logic
|
||||
# ============================================
|
||||
|
||||
main() {
|
||||
log "=== Health Check Started (PID: $$) ==="
|
||||
|
||||
# Log rotation (cleanup old logs)
|
||||
rotate_old_logs
|
||||
|
||||
# HTTP 응답 체크
|
||||
if ! check_http; then
|
||||
log "⚠️ Gateway unhealthy (HTTP failed)"
|
||||
|
||||
# 3번 재시도
|
||||
for i in $(seq 1 "$MAX_RETRIES"); do
|
||||
log "Retry $i/$MAX_RETRIES..."
|
||||
|
||||
if restart_gateway && check_http; then
|
||||
log "✅ Recovery successful on retry $i"
|
||||
|
||||
# Discord 알림 (복구 성공)
|
||||
send_discord_notification "✅ **Gateway 복구 성공**\n\nLevel 2 Health Check가 Gateway를 재시작하여 복구했습니다.\n- 재시도 횟수: $i/$MAX_RETRIES\n- 현재 시각: $(date '+%Y-%m-%d %H:%M:%S')"
|
||||
|
||||
record_metric "recovery" "success" "$i"
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
|
||||
log "❌ Recovery failed after $MAX_RETRIES retries"
|
||||
log "🚨 Escalating to Level 3 (Claude Emergency Recovery)..."
|
||||
record_metric "recovery" "failed" "$MAX_RETRIES"
|
||||
|
||||
# Discord 알림 (Level 3로 escalation)
|
||||
send_discord_notification "⚠️ **Level 2 Health Check 실패**\n\nGateway를 ${MAX_RETRIES}회 재시작했으나 복구 실패.\n${ESCALATION_WAIT}초 후 Level 3 (Claude Emergency Recovery)로 escalation합니다.\n\n현재 시각: $(date '+%Y-%m-%d %H:%M:%S')"
|
||||
|
||||
# 5분 대기 후 최종 검증
|
||||
sleep "$ESCALATION_WAIT"
|
||||
|
||||
if ! check_http; then
|
||||
escalate_to_level3
|
||||
else
|
||||
log "✅ Gateway recovered during waiting period"
|
||||
|
||||
# Discord 알림 (대기 중 복구됨)
|
||||
send_discord_notification "✅ **Gateway 자동 복구됨**\n\n${ESCALATION_WAIT}초 대기 중 Gateway가 스스로 복구되었습니다.\nLevel 3 Emergency Recovery는 실행하지 않습니다."
|
||||
|
||||
record_metric "recovery" "self_healed" 0
|
||||
fi
|
||||
else
|
||||
log "✅ Gateway healthy"
|
||||
record_metric "health_check" "healthy" 0
|
||||
fi
|
||||
|
||||
log "=== Health Check Completed ==="
|
||||
}
|
||||
|
||||
# Run main function
|
||||
main
|
||||
Reference in New Issue
Block a user