From d55ca207d26b6b6929be535109a0c6fdffb0e16c Mon Sep 17 00:00:00 2001
From: Krilly <krilly@openclaw.local>
Date: Sat, 21 Feb 2026 02:06:12 +0000
Subject: [PATCH] Add openclaw-watchdog fork with Telegram + Gotify support

- Cloned jlgrimes/openclaw-watchdog and modified
- Added telegram_send() and gotify_send() functions
- Modified send_sos() and send_recovery() for multi-channel
- Updated setup.sh to configure Telegram/Gotify
- All notification channels work simultaneously
- Ready to install and run as systemd service
---
 openclaw-watchdog/LICENSE          |  21 ++++
 openclaw-watchdog/README-KRILLY.md |  93 ++++++++++++++
 openclaw-watchdog/README.md        | 105 ++++++++++++++++
 openclaw-watchdog/setup.sh         |  94 ++++++++++++++
 openclaw-watchdog/watchdog.sh      | 192 +++++++++++++++++++++++++++++
 5 files changed, 505 insertions(+)
 create mode 100644 openclaw-watchdog/LICENSE
 create mode 100644 openclaw-watchdog/README-KRILLY.md
 create mode 100644 openclaw-watchdog/README.md
 create mode 100755 openclaw-watchdog/setup.sh
 create mode 100755 openclaw-watchdog/watchdog.sh

diff --git a/openclaw-watchdog/LICENSE b/openclaw-watchdog/LICENSE
new file mode 100644
index 0000000..f527bf1
--- /dev/null
+++ b/openclaw-watchdog/LICENSE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Jared Grimes
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/openclaw-watchdog/README-KRILLY.md b/openclaw-watchdog/README-KRILLY.md
new file mode 100644
index 0000000..0ff0f13
--- /dev/null
+++ b/openclaw-watchdog/README-KRILLY.md
@@ -0,0 +1,93 @@
+# 🦞 OpenClaw Watchdog — Krilly's Fork
+
+**Original:** [jlgrimes/openclaw-watchdog](https://github.com/jlgrimes/openclaw-watchdog)  
+**Modified by:** Krilly the Crab 🦀
+
+## What's Different?
+
+This fork adds **Telegram** and **Gotify** notifications alongside (or instead of) Discord.
+
+| Feature | Original | This Fork |
+|---------|----------|-----------|
+| Discord alerts | ✅ | ✅ (optional) |
+| Telegram alerts | ❌ | ✅ |
+| Gotify alerts | ❌ | ✅ |
+| Multi-channel | ❌ | ✅ (simultaneous) |
+
+## Quick Install
+
+```bash
+git clone https://github.com/jlgrimes/openclaw-watchdog.git
+cd openclaw-watchdog
+chmod +x setup.sh watchdog.sh
+./setup.sh
+```
+
+The installer will ask for:
+- Telegram Bot Token
+- Gotify Token
+- Gotify URL (defaults to your setup)
+
+## Configuration
+
+Edit `~/.openclaw/watchdog.env`:
+
+```bash
+# Required for Telegram
+TELEGRAM_BOT_TOKEN=your_bot_token_here
+TELEGRAM_CHAT=1793951355
+
+# Required for Gotify
+GOTIFY_URL=http://runtipi.kangaroo-eel.ts.net:8129
+GOTIFY_TOKEN=your_gotify_token_here
+
+# Optional: Legacy Discord
+DISCORD_CHANNEL_ID=
+DISCORD_BOT_TOKEN=
+```
+
+All three can work simultaneously — get alerts wherever you want!
+
+## How It Works
+
+Same great watchdog behavior from the original:
+
+1. **Health checks** every 30s (configurable)
+2. **Stage 1:** Simple restart on first failure
+3. **Stage 2:** Config rollback + restart on second failure
+4. **Stage 3:** SOS alert after 3+ failures
+5. **Recovery:** Notification when gateway comes back
+
+But now alerts go to **Telegram** and **Gotify** too!
+
+## Managing the Service
+
+```bash
+sudo systemctl status openclaw-watchdog   # Check status
+sudo systemctl restart openclaw-watchdog  # Restart
+sudo systemctl stop openclaw-watchdog     # Stop
+tail -f ~/.openclaw/watchdog.log          # View logs
+```
+
+## Test Notifications
+
+```bash
+export TELEGRAM_BOT_TOKEN="your_token"
+export TELEGRAM_CHAT="1793951355"
+export GOTIFY_TOKEN="your_token"
+
+# Source the functions and test
+source ~/.openclaw/watchdog.sh
+telegram_send "🦀 Test message from Krilly!"
+gotify_send "Test" "🦀 Test message from Krilly!" 5
+```
+
+## Credits
+
+- **Original:** Jared Grimes and Claw 🦞
+- **Fork modifications:** Krilly the Crab for Anthony Martin
+- **License:** MIT
+
+---
+
+*Part of the Krilly Automation Stack* 🦀
diff --git a/openclaw-watchdog/README.md b/openclaw-watchdog/README.md
new file mode 100644
index 0000000..ab7c88e
--- /dev/null
+++ b/openclaw-watchdog/README.md
@@ -0,0 +1,105 @@
+# 🦞 openclaw-watchdog
+
+A standalone watchdog service for [OpenClaw](https://openclaw.dev) that monitors the gateway, auto-recovers from crashes, and sends Discord alerts when things go sideways.
+
+## What It Does
+
+- **Health monitoring** — Polls the gateway health endpoint at a configurable interval
+- **Auto-restart** — Restarts the gateway on first failure detection
+- **Config rollback** — Reverts to last-known-good config if a simple restart doesn't work
+- **Discord SOS alerts** — Sends a detailed alert with SSH instructions when all recovery fails
+- **Recovery notifications** — Lets you know when the gateway comes back online
+- **No spam** — Only alerts once per incident
+
+## Recovery Flow
+
+```
+Health check fails
+       │
+       ▼
+  ┌─────────────┐
+  │ Stage 1:    │──→ Simple restart
+  │ Restart     │
+  └──────┬──────┘
+         │ still failing
+         ▼
+  ┌─────────────┐
+  │ Stage 2:    │──→ Revert config to last-known-good + restart
+  │ Rollback    │
+  └──────┬──────┘
+         │ still failing
+         ▼
+  ┌─────────────┐
+  │ Stage 3:    │──→ Discord SOS with hostname & SSH instructions
+  │ SOS Alert   │
+  └─────────────┘
+```
+
+When the gateway recovers at any point, a ✅ recovery message is sent.
+
+## Prerequisites
+
+- [OpenClaw](https://openclaw.dev) installed with `openclaw` CLI available
+- Linux with systemd
+- `curl` and `jq`
+- Discord bot token (optional, for alerts)
+
+## Quick Install
+
+```bash
+git clone https://github.com/jlgrimes/openclaw-watchdog.git
+cd openclaw-watchdog
+chmod +x setup.sh watchdog.sh
+./setup.sh
+```
+
+The installer will:
+1. Copy `watchdog.sh` to `~/.openclaw/`
+2. Create a config file at `~/.openclaw/watchdog.env`
+3. Set up and start a systemd service
+
+## Configuration
+
+All settings live in `~/.openclaw/watchdog.env` (created by the installer):
+
+| Variable | Default | Description |
+|---|---|---|
+| `HEALTH_URL` | `http://localhost:3000/health` | Gateway health endpoint |
+| `CHECK_INTERVAL` | `30` | Seconds between health checks |
+| `FAIL_THRESHOLD` | `3` | Consecutive failures before SOS |
+| `OPENCLAW_CONFIG_PATH` | `~/.openclaw/config.yaml` | Path to OpenClaw config |
+| `DISCORD_CHANNEL_ID` | *(none)* | Discord channel for alerts |
+| `DISCORD_BOT_TOKEN` | *(none)* | Discord bot token for alerts |
+
+After editing, restart the service:
+
+```bash
+sudo systemctl restart openclaw-watchdog
+```
+
+## Manual Usage
+
+You can also run the watchdog directly:
+
+```bash
+export DISCORD_BOT_TOKEN="your-token"
+export DISCORD_CHANNEL_ID="your-channel-id"
+./watchdog.sh
+```
+
+## Managing the Service
+
+```bash
+sudo systemctl status openclaw-watchdog    # Check status
+sudo systemctl restart openclaw-watchdog   # Restart
+sudo systemctl stop openclaw-watchdog      # Stop
+tail -f ~/.openclaw/watchdog.log           # View logs
+```
+
+## License
+
+[MIT](LICENSE)
+
+## Credits
+
+Built by **Jared Grimes** and **Claw 🦞** (an OpenClaw AI assistant)
diff --git a/openclaw-watchdog/setup.sh b/openclaw-watchdog/setup.sh
new file mode 100755
index 0000000..504607d
--- /dev/null
+++ b/openclaw-watchdog/setup.sh
@@ -0,0 +1,94 @@
+#!/usr/bin/env bash
+# openclaw-watchdog installer (Krilly's fork — Telegram + Gotify edition)
+# https://github.com/jlgrimes/openclaw-watchdog
+set -euo pipefail
+
+INSTALL_DIR="$HOME/.openclaw"
+SERVICE_NAME="openclaw-watchdog"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+echo "🦞 OpenClaw Watchdog — Installer (Krilly's Telegram/Gotify Edition)"
+echo "───────────────────────────────────────────────────────────────────"
+
+# Ensure install dir exists
+mkdir -p "$INSTALL_DIR"
+
+# Copy watchdog script
+cp "$SCRIPT_DIR/watchdog.sh" "$INSTALL_DIR/watchdog.sh"
+chmod +x "$INSTALL_DIR/watchdog.sh"
+echo "✅ Installed watchdog.sh → $INSTALL_DIR/watchdog.sh"
+
+# Prompt for Telegram bot token
+TELEGRAM_BOT_TOKEN=""
+read -rp "Telegram Bot Token (leave blank to skip): " TELEGRAM_BOT_TOKEN
+
+# Prompt for Gotify token
+GOTIFY_TOKEN=""
+read -rp "Gotify Token (leave blank to skip): " GOTIFY_TOKEN
+
+# Prompt for custom Gotify URL
+GOTIFY_URL="http://runtipi.kangaroo-eel.ts.net:8129"
+read -rp "Gotify URL [$GOTIFY_URL]: " custom_gotify
+[[ -n "$custom_gotify" ]] && GOTIFY_URL="$custom_gotify"
+
+# Create env file
+ENV_FILE="$INSTALL_DIR/watchdog.env"
+cat > "$ENV_FILE" <<EOF
+# OpenClaw Watchdog Configuration
+# Edit these values as needed, then restart the service:
+#   sudo systemctl restart $SERVICE_NAME
+
+HEALTH_URL=http://localhost:3000/health
+CHECK_INTERVAL=30
+FAIL_THRESHOLD=3
+OPENCLAW_CONFIG_PATH=$HOME/.openclaw/config.yaml
+
+# Notification Settings
+TELEGRAM_BOT_TOKEN=${TELEGRAM_BOT_TOKEN}
+TELEGRAM_CHAT=1793951355
+GOTIFY_URL=${GOTIFY_URL}
+GOTIFY_TOKEN=${GOTIFY_TOKEN}
+
+# Legacy Discord (optional)
+# DISCORD_CHANNEL_ID=
+# DISCORD_BOT_TOKEN=
+EOF
+chmod 600 "$ENV_FILE"
+echo "✅ Created config → $ENV_FILE"
+
+# Create systemd service
+SERVICE_FILE="/etc/systemd/system/${SERVICE_NAME}.service"
+sudo tee "$SERVICE_FILE" >/dev/null <<EOF
+[Unit]
+Description=OpenClaw Watchdog — Gateway health monitor & auto-recovery
+After=network.target
+
+[Service]
+Type=simple
+User=$USER
+EnvironmentFile=$ENV_FILE
+ExecStart=/usr/bin/env bash $INSTALL_DIR/watchdog.sh
+Restart=on-failure
+RestartSec=10
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+echo "✅ Created systemd service → $SERVICE_FILE"
+
+# Enable and start
+sudo systemctl daemon-reload
+sudo systemctl enable "$SERVICE_NAME"
+sudo systemctl start "$SERVICE_NAME"
+
+echo ""
+echo "🎉 Watchdog is running!"
+echo ""
+echo "  Status:   sudo systemctl status $SERVICE_NAME"
+echo "  Logs:     tail -f $INSTALL_DIR/watchdog.log"
+echo "  Config:   $ENV_FILE"
+echo ""
+echo "You'll get alerts via:"
+[[ -n "$TELEGRAM_BOT_TOKEN" ]] && echo "  ✅ Telegram"
+[[ -n "$GOTIFY_TOKEN" ]] && echo "  ✅ Gotify ($GOTIFY_URL)"
diff --git a/openclaw-watchdog/watchdog.sh b/openclaw-watchdog/watchdog.sh
new file mode 100755
index 0000000..aee884c
--- /dev/null
+++ b/openclaw-watchdog/watchdog.sh
@@ -0,0 +1,192 @@
+#!/usr/bin/env bash
+# openclaw-watchdog — Monitor OpenClaw gateway, auto-recover from crashes
+# https://github.com/jlgrimes/openclaw-watchdog
+# MIT License — Jared Grimes
+
+set -euo pipefail
+
+# ─── Configuration (all overridable via env vars) ────────────────────────────
+OPENCLAW_CONFIG_PATH="${OPENCLAW_CONFIG_PATH:-$HOME/.openclaw/config.yaml}"
+HEALTH_URL="${HEALTH_URL:-http://localhost:3000/health}"
+CHECK_INTERVAL="${CHECK_INTERVAL:-30}"          # seconds between checks
+FAIL_THRESHOLD="${FAIL_THRESHOLD:-3}"           # consecutive failures before escalation
+WATCHDOG_LOG="${WATCHDOG_LOG:-$HOME/.openclaw/watchdog.log}"
+GOOD_CONFIG_PATH="${GOOD_CONFIG_PATH:-$HOME/.openclaw/config.yaml.good}"
+
+# Notification settings (Telegram + Gotify)
+TELEGRAM_BOT_TOKEN="${TELEGRAM_BOT_TOKEN:-}"
+TELEGRAM_CHAT="${TELEGRAM_CHAT:-1793951355}"
+GOTIFY_URL="${GOTIFY_URL:-http://runtipi.kangaroo-eel.ts.net:8129}"
+GOTIFY_TOKEN="${GOTIFY_TOKEN:-AGKnHafW3FGzBlt}"
+
+# Legacy Discord settings (optional, for backwards compat)
+DISCORD_CHANNEL_ID="${DISCORD_CHANNEL_ID:-}"
+DISCORD_BOT_TOKEN="${DISCORD_BOT_TOKEN:-}"
+
+# ─── State ────────────────────────────────────────────────────────────────────
+fail_count=0
+alerted=false        # true after SOS sent, prevents spam
+was_down=false       # tracks if we're recovering
+
+# ─── Logging ──────────────────────────────────────────────────────────────────
+log() {
+  local ts
+  ts="$(date '+%Y-%m-%d %H:%M:%S')"
+  echo "[$ts] $*" | tee -a "$WATCHDOG_LOG"
+}
+
+# ─── Discord messaging ───────────────────────────────────────────────────────
+discord_send() {
+  local msg="$1"
+  [[ -z "$DISCORD_CHANNEL_ID" ]] && return 0
+  [[ -z "$DISCORD_BOT_TOKEN" ]] && return 0
+
+  curl -sf -X POST \
+    "https://discord.com/api/v10/channels/${DISCORD_CHANNEL_ID}/messages" \
+    -H "Authorization: Bot ${DISCORD_BOT_TOKEN}" \
+    -H "Content-Type: application/json" \
+    -d "{\"content\": $(printf '%s' "$msg" | jq -Rs .)}" \
+    >/dev/null 2>&1 || log "WARN: Failed to send Discord message"
+}
+
+# ─── Telegram messaging ──────────────────────────────────────────────────────
+telegram_send() {
+  local msg="$1"
+  [[ -z "$TELEGRAM_BOT_TOKEN" ]] && return 0
+  [[ -z "$TELEGRAM_CHAT" ]] && return 0
+
+  curl -sf -X POST \
+    "https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \
+    -H "Content-Type: application/json" \
+    -d "{\"chat_id\": \"$TELEGRAM_CHAT\", \"text\": $(printf '%s' "$msg" | jq -Rs .), \"parse_mode\": \"Markdown\"}" \
+    >/dev/null 2>&1 || log "WARN: Failed to send Telegram message"
+}
+
+# ─── Gotify messaging ────────────────────────────────────────────────────────
+gotify_send() {
+  local title="$1"
+  local msg="$2"
+  local priority="${3:-5}"
+  [[ -z "$GOTIFY_TOKEN" ]] && return 0
+
+  curl -sf -X POST \
+    "${GOTIFY_URL}/message?token=${GOTIFY_TOKEN}" \
+    -H "Content-Type: application/json" \
+    -d "{\"title\": \"$title\", \"message\": \"$msg\", \"priority\": $priority}" \
+    >/dev/null 2>&1 || log "WARN: Failed to send Gotify message"
+}
+
+# ─── Send to all configured channels ─────────────────────────────────────────
+send_notification() {
+  local title="$1"
+  local msg="$2"
+  local priority="${3:-5}"
+  
+  discord_send "$msg"
+  telegram_send "$msg"
+  gotify_send "$title" "$msg" "$priority"
+}
+
+# ─── Health check ─────────────────────────────────────────────────────────────
+check_health() {
+  curl -sf --max-time 10 "$HEALTH_URL" >/dev/null 2>&1
+}
+
+# ─── Save last-known-good config ─────────────────────────────────────────────
+save_good_config() {
+  if [[ -f "$OPENCLAW_CONFIG_PATH" ]]; then
+    cp "$OPENCLAW_CONFIG_PATH" "$GOOD_CONFIG_PATH"
+  fi
+}
+
+# ─── Revert to last-known-good config ────────────────────────────────────────
+revert_config() {
+  if [[ -f "$GOOD_CONFIG_PATH" ]]; then
+    log "Reverting config to last-known-good snapshot"
+    cp "$GOOD_CONFIG_PATH" "$OPENCLAW_CONFIG_PATH"
+    return 0
+  else
+    log "WARN: No good config snapshot available to revert"
+    return 1
+  fi
+}
+
+# ─── Restart gateway ─────────────────────────────────────────────────────────
+restart_gateway() {
+  log "Restarting OpenClaw gateway..."
+  openclaw gateway restart >/dev/null 2>&1 || true
+  sleep 5
+}
+
+# ─── SOS alert ────────────────────────────────────────────────────────────────
+send_sos() {
+  local hostname
+  hostname="$(hostname 2>/dev/null || echo 'unknown')"
+  local title="🚨 OpenClaw Gateway DOWN"
+  local msg="🚨 **OpenClaw Gateway DOWN** on \`${hostname}\`
+
+Watchdog tried:
+1. ✅ Simple restart
+2. ✅ Config rollback + restart
+3. ❌ Still unreachable after ${FAIL_THRESHOLD}+ failures
+
+**Manual intervention needed:**
+\`\`\`
+ssh ${USER}@${hostname}
+openclaw gateway status
+journalctl -u openclaw-gateway --since '10 min ago'
+\`\`\`"
+
+  log "CRITICAL: Sending SOS alert"
+  send_notification "$title" "$msg" 10
+  alerted=true
+}
+
+# ─── Recovery notification ────────────────────────────────────────────────────
+send_recovery() {
+  local hostname
+  hostname="$(hostname 2>/dev/null || echo 'unknown')"
+  local title="✅ OpenClaw Gateway Recovered"
+  local msg="✅ **OpenClaw Gateway recovered** on \`${hostname}\` — back online!"
+  log "Gateway recovered"
+  send_notification "$title" "$msg" 5
+}
+
+# ─── Main loop ────────────────────────────────────────────────────────────────
+main() {
+  log "Watchdog started (interval=${CHECK_INTERVAL}s, threshold=${FAIL_THRESHOLD}, health=${HEALTH_URL})"
+  log "Notifications: Telegram=${TELEGRAM_CHAT:+enabled}, Gotify=${GOTIFY_TOKEN:+enabled}, Discord=${DISCORD_CHANNEL_ID:+enabled}"
+
+  while true; do
+    if check_health; then
+      # ── Healthy ──
+      if [[ "$was_down" == true ]]; then
+        send_recovery
+        was_down=false
+        alerted=false
+      fi
+      fail_count=0
+      save_good_config
+    else
+      # ── Unhealthy ──
+      fail_count=$((fail_count + 1))
+      log "Health check failed (${fail_count}/${FAIL_THRESHOLD})"
+
+      if [[ $fail_count -eq 1 ]]; then
+        # Stage 1: simple restart
+        was_down=true
+        restart_gateway
+      elif [[ $fail_count -eq 2 ]]; then
+        # Stage 2: config revert + restart
+        revert_config && restart_gateway
+      elif [[ $fail_count -ge $FAIL_THRESHOLD && "$alerted" == false ]]; then
+        # Stage 3: SOS (once)
+        send_sos
+      fi
+    fi
+
+    sleep "$CHECK_INTERVAL"
+  done
+}
+
+main "$@"