Compare commits
2 Commits
2d85d3873d
...
d55ca207d2
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d55ca207d2 | ||
|
|
180532d1e3 |
@@ -11,6 +11,8 @@ LOG_FILE="$SCRIPT_DIR/reminder-log.json"
|
||||
source "$SCRIPT_DIR/../../.env" 2>/dev/null || true
|
||||
|
||||
TELEGRAM_CHAT="${TELEGRAM_CHAT:-1793951355}"
|
||||
GOTIFY_URL="${GOTIFY_URL:-http://runtipi.kangaroo-eel.ts.net:8129}"
|
||||
GOTIFY_TOKEN="${GOTIFY_TOKEN:-AGKnHafW3FGzBlt}"
|
||||
|
||||
log() {
|
||||
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1"
|
||||
@@ -165,6 +167,24 @@ send_telegram() {
|
||||
}
|
||||
}
|
||||
|
||||
# Send Gotify message
|
||||
send_gotify() {
|
||||
local title="$1"
|
||||
local message="$2"
|
||||
local priority="${3:-5}"
|
||||
|
||||
curl -s -X POST "${GOTIFY_URL}/message?token=${GOTIFY_TOKEN}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{
|
||||
\"title\": \"$title\",
|
||||
\"message\": \"$message\",
|
||||
\"priority\": $priority
|
||||
}" > /dev/null || {
|
||||
log "ERROR: Failed to send Gotify message"
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
# Generate gift suggestions text
|
||||
gift_suggestions() {
|
||||
local ideas=$(echo "$1" | jq -r '.gift_ideas | join(", ")')
|
||||
@@ -248,6 +268,14 @@ check_birthdays() {
|
||||
if [[ "$should_remind" == true ]]; then
|
||||
log "Sending $reminder_type reminder for $name"
|
||||
send_telegram "$message"
|
||||
|
||||
# Also send to Gotify (strip markdown for cleaner display)
|
||||
local plain_message
|
||||
plain_message=$(echo -e "$message" | sed 's/\*//g')
|
||||
local priority=5
|
||||
[[ "$reminder_type" == "today" ]] && priority=8
|
||||
send_gotify "Birthday: $name" "$plain_message" "$priority"
|
||||
|
||||
log_reminder "$name" "$reminder_type"
|
||||
((reminders_sent++))
|
||||
fi
|
||||
|
||||
@@ -35,14 +35,6 @@
|
||||
"notes": "8 years old (born July 2016). Loves games, books, LEGO, sports.",
|
||||
"gift_ideas": ["LEGO set", "Books", "Board games", "Sports equipment", "Science kit"],
|
||||
"past_gifts": []
|
||||
},
|
||||
{
|
||||
"name": "Mia Martin",
|
||||
"relationship": "Doggo 🐕",
|
||||
"birthday": "XX-XX",
|
||||
"notes": "14 years old, beloved geriatric girl. Treats, toys, comfy beds.",
|
||||
"gift_ideas": ["Premium dog treats", "Comfy bed", "New toy", "Grooming session"],
|
||||
"past_gifts": []
|
||||
}
|
||||
],
|
||||
"settings": {
|
||||
|
||||
@@ -12,6 +12,8 @@ source "$SCRIPT_DIR/../../.env" 2>/dev/null || true
|
||||
FRESHRSS_URL="${FRESHRSS_URL:-http://freshrss.kangaroo-eel.ts.net}"
|
||||
FRESHRSS_USER="${FRESHRSS_USER:-anthony}"
|
||||
TELEGRAM_CHAT="${TELEGRAM_CHAT:-1793951355}"
|
||||
GOTIFY_URL="${GOTIFY_URL:-http://runtipi.kangaroo-eel.ts.net:8129}"
|
||||
GOTIFY_TOKEN="${GOTIFY_TOKEN:-AGKnHafW3FGzBlt}"
|
||||
|
||||
# Your interest keywords for relevance ranking
|
||||
INTERESTS=(
|
||||
@@ -187,6 +189,27 @@ send_telegram() {
|
||||
log "Sent to Telegram successfully"
|
||||
}
|
||||
|
||||
send_gotify() {
|
||||
local title="$1"
|
||||
local message="$2"
|
||||
local priority="${3:-5}"
|
||||
|
||||
log "Sending to Gotify..."
|
||||
|
||||
curl -s -X POST "${GOTIFY_URL}/message?token=${GOTIFY_TOKEN}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{
|
||||
\"title\": \"$title\",
|
||||
\"message\": \"$message\",
|
||||
\"priority\": $priority
|
||||
}" > /dev/null || {
|
||||
log "ERROR: Failed to send Gotify message"
|
||||
return 1
|
||||
}
|
||||
|
||||
log "Sent to Gotify successfully"
|
||||
}
|
||||
|
||||
main() {
|
||||
log "Starting FreshRSS digest..."
|
||||
|
||||
@@ -198,9 +221,14 @@ main() {
|
||||
local digest
|
||||
digest=$(build_digest "$raw_articles")
|
||||
|
||||
# Send
|
||||
# Send to both channels
|
||||
send_telegram "$digest"
|
||||
|
||||
# Plain text version for Gotify (strip markdown)
|
||||
local plain_digest
|
||||
plain_digest=$(echo -e "$digest" | sed 's/\*//g' | sed 's/\[\([^]]*\)\]([^)]*)/\1/g')
|
||||
send_gotify "FreshRSS Digest" "$plain_digest" 5
|
||||
|
||||
log "Digest complete!"
|
||||
}
|
||||
|
||||
|
||||
21
openclaw-watchdog/LICENSE
Normal file
21
openclaw-watchdog/LICENSE
Normal file
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2026 Jared Grimes
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
93
openclaw-watchdog/README-KRILLY.md
Normal file
93
openclaw-watchdog/README-KRILLY.md
Normal file
@@ -0,0 +1,93 @@
|
||||
# 🦞 OpenClaw Watchdog — Krilly's Fork
|
||||
|
||||
**Original:** [jlgrimes/openclaw-watchdog](https://github.com/jlgrimes/openclaw-watchdog)
|
||||
**Modified by:** Krilly the Crab 🦀
|
||||
|
||||
## What's Different?
|
||||
|
||||
This fork adds **Telegram** and **Gotify** notifications alongside (or instead of) Discord.
|
||||
|
||||
| Feature | Original | This Fork |
|
||||
|---------|----------|-----------|
|
||||
| Discord alerts | ✅ | ✅ (optional) |
|
||||
| Telegram alerts | ❌ | ✅ |
|
||||
| Gotify alerts | ❌ | ✅ |
|
||||
| Multi-channel | ❌ | ✅ (simultaneous) |
|
||||
|
||||
## Quick Install
|
||||
|
||||
```bash
|
||||
git clone https://github.com/jlgrimes/openclaw-watchdog.git
|
||||
cd openclaw-watchdog
|
||||
chmod +x setup.sh watchdog.sh
|
||||
./setup.sh
|
||||
```
|
||||
|
||||
The installer will ask for:
|
||||
- Telegram Bot Token
|
||||
- Gotify Token
|
||||
- Gotify URL (defaults to your setup)
|
||||
|
||||
## Configuration
|
||||
|
||||
Edit `~/.openclaw/watchdog.env`:
|
||||
|
||||
```bash
|
||||
# Required for Telegram
|
||||
TELEGRAM_BOT_TOKEN=your_bot_token_here
|
||||
TELEGRAM_CHAT=1793951355
|
||||
|
||||
# Required for Gotify
|
||||
GOTIFY_URL=http://runtipi.kangaroo-eel.ts.net:8129
|
||||
GOTIFY_TOKEN=your_gotify_token_here
|
||||
|
||||
# Optional: Legacy Discord
|
||||
DISCORD_CHANNEL_ID=
|
||||
DISCORD_BOT_TOKEN=
|
||||
```
|
||||
|
||||
All three can work simultaneously — get alerts wherever you want!
|
||||
|
||||
## How It Works
|
||||
|
||||
Same great watchdog behavior from the original:
|
||||
|
||||
1. **Health checks** every 30s (configurable)
|
||||
2. **Stage 1:** Simple restart on first failure
|
||||
3. **Stage 2:** Config rollback + restart on second failure
|
||||
4. **Stage 3:** SOS alert after 3+ failures
|
||||
5. **Recovery:** Notification when gateway comes back
|
||||
|
||||
But now alerts go to **Telegram** and **Gotify** too!
|
||||
|
||||
## Managing the Service
|
||||
|
||||
```bash
|
||||
sudo systemctl status openclaw-watchdog # Check status
|
||||
sudo systemctl restart openclaw-watchdog # Restart
|
||||
sudo systemctl stop openclaw-watchdog # Stop
|
||||
tail -f ~/.openclaw/watchdog.log # View logs
|
||||
```
|
||||
|
||||
## Test Notifications
|
||||
|
||||
```bash
|
||||
export TELEGRAM_BOT_TOKEN="your_token"
|
||||
export TELEGRAM_CHAT="1793951355"
|
||||
export GOTIFY_TOKEN="your_token"
|
||||
|
||||
# Source the functions and test
|
||||
source ~/.openclaw/watchdog.sh
|
||||
telegram_send "🦀 Test message from Krilly!"
|
||||
gotify_send "Test" "🦀 Test message from Krilly!" 5
|
||||
```
|
||||
|
||||
## Credits
|
||||
|
||||
- **Original:** Jared Grimes and Claw 🦞
|
||||
- **Fork modifications:** Krilly the Crab for Anthony Martin
|
||||
- **License:** MIT
|
||||
|
||||
---
|
||||
|
||||
*Part of the Krilly Automation Stack* 🦀
|
||||
105
openclaw-watchdog/README.md
Normal file
105
openclaw-watchdog/README.md
Normal file
@@ -0,0 +1,105 @@
|
||||
# 🦞 openclaw-watchdog
|
||||
|
||||
A standalone watchdog service for [OpenClaw](https://openclaw.dev) that monitors the gateway, auto-recovers from crashes, and sends Discord alerts when things go sideways.
|
||||
|
||||
## What It Does
|
||||
|
||||
- **Health monitoring** — Polls the gateway health endpoint at a configurable interval
|
||||
- **Auto-restart** — Restarts the gateway on first failure detection
|
||||
- **Config rollback** — Reverts to last-known-good config if a simple restart doesn't work
|
||||
- **Discord SOS alerts** — Sends a detailed alert with SSH instructions when all recovery fails
|
||||
- **Recovery notifications** — Lets you know when the gateway comes back online
|
||||
- **No spam** — Only alerts once per incident
|
||||
|
||||
## Recovery Flow
|
||||
|
||||
```
|
||||
Health check fails
|
||||
│
|
||||
▼
|
||||
┌─────────────┐
|
||||
│ Stage 1: │──→ Simple restart
|
||||
│ Restart │
|
||||
└──────┬──────┘
|
||||
│ still failing
|
||||
▼
|
||||
┌─────────────┐
|
||||
│ Stage 2: │──→ Revert config to last-known-good + restart
|
||||
│ Rollback │
|
||||
└──────┬──────┘
|
||||
│ still failing
|
||||
▼
|
||||
┌─────────────┐
|
||||
│ Stage 3: │──→ Discord SOS with hostname & SSH instructions
|
||||
│ SOS Alert │
|
||||
└─────────────┘
|
||||
```
|
||||
|
||||
When the gateway recovers at any point, a ✅ recovery message is sent.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- [OpenClaw](https://openclaw.dev) installed with `openclaw` CLI available
|
||||
- Linux with systemd
|
||||
- `curl` and `jq`
|
||||
- Discord bot token (optional, for alerts)
|
||||
|
||||
## Quick Install
|
||||
|
||||
```bash
|
||||
git clone https://github.com/jlgrimes/openclaw-watchdog.git
|
||||
cd openclaw-watchdog
|
||||
chmod +x setup.sh watchdog.sh
|
||||
./setup.sh
|
||||
```
|
||||
|
||||
The installer will:
|
||||
1. Copy `watchdog.sh` to `~/.openclaw/`
|
||||
2. Create a config file at `~/.openclaw/watchdog.env`
|
||||
3. Set up and start a systemd service
|
||||
|
||||
## Configuration
|
||||
|
||||
All settings live in `~/.openclaw/watchdog.env` (created by the installer):
|
||||
|
||||
| Variable | Default | Description |
|
||||
|---|---|---|
|
||||
| `HEALTH_URL` | `http://localhost:3000/health` | Gateway health endpoint |
|
||||
| `CHECK_INTERVAL` | `30` | Seconds between health checks |
|
||||
| `FAIL_THRESHOLD` | `3` | Consecutive failures before SOS |
|
||||
| `OPENCLAW_CONFIG_PATH` | `~/.openclaw/config.yaml` | Path to OpenClaw config |
|
||||
| `DISCORD_CHANNEL_ID` | *(none)* | Discord channel for alerts |
|
||||
| `DISCORD_BOT_TOKEN` | *(none)* | Discord bot token for alerts |
|
||||
|
||||
After editing, restart the service:
|
||||
|
||||
```bash
|
||||
sudo systemctl restart openclaw-watchdog
|
||||
```
|
||||
|
||||
## Manual Usage
|
||||
|
||||
You can also run the watchdog directly:
|
||||
|
||||
```bash
|
||||
export DISCORD_BOT_TOKEN="your-token"
|
||||
export DISCORD_CHANNEL_ID="your-channel-id"
|
||||
./watchdog.sh
|
||||
```
|
||||
|
||||
## Managing the Service
|
||||
|
||||
```bash
|
||||
sudo systemctl status openclaw-watchdog # Check status
|
||||
sudo systemctl restart openclaw-watchdog # Restart
|
||||
sudo systemctl stop openclaw-watchdog # Stop
|
||||
tail -f ~/.openclaw/watchdog.log # View logs
|
||||
```
|
||||
|
||||
## License
|
||||
|
||||
[MIT](LICENSE)
|
||||
|
||||
## Credits
|
||||
|
||||
Built by **Jared Grimes** and **Claw 🦞** (an OpenClaw AI assistant)
|
||||
94
openclaw-watchdog/setup.sh
Executable file
94
openclaw-watchdog/setup.sh
Executable file
@@ -0,0 +1,94 @@
|
||||
#!/usr/bin/env bash
|
||||
# openclaw-watchdog installer (Krilly's fork — Telegram + Gotify edition)
|
||||
# https://github.com/jlgrimes/openclaw-watchdog
|
||||
set -euo pipefail
|
||||
|
||||
INSTALL_DIR="$HOME/.openclaw"
|
||||
SERVICE_NAME="openclaw-watchdog"
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
echo "🦞 OpenClaw Watchdog — Installer (Krilly's Telegram/Gotify Edition)"
|
||||
echo "───────────────────────────────────────────────────────────────────"
|
||||
|
||||
# Ensure install dir exists
|
||||
mkdir -p "$INSTALL_DIR"
|
||||
|
||||
# Copy watchdog script
|
||||
cp "$SCRIPT_DIR/watchdog.sh" "$INSTALL_DIR/watchdog.sh"
|
||||
chmod +x "$INSTALL_DIR/watchdog.sh"
|
||||
echo "✅ Installed watchdog.sh → $INSTALL_DIR/watchdog.sh"
|
||||
|
||||
# Prompt for Telegram bot token
|
||||
TELEGRAM_BOT_TOKEN=""
|
||||
read -rp "Telegram Bot Token (leave blank to skip): " TELEGRAM_BOT_TOKEN
|
||||
|
||||
# Prompt for Gotify token
|
||||
GOTIFY_TOKEN=""
|
||||
read -rp "Gotify Token (leave blank to skip): " GOTIFY_TOKEN
|
||||
|
||||
# Prompt for custom Gotify URL
|
||||
GOTIFY_URL="http://runtipi.kangaroo-eel.ts.net:8129"
|
||||
read -rp "Gotify URL [$GOTIFY_URL]: " custom_gotify
|
||||
[[ -n "$custom_gotify" ]] && GOTIFY_URL="$custom_gotify"
|
||||
|
||||
# Create env file
|
||||
ENV_FILE="$INSTALL_DIR/watchdog.env"
|
||||
cat > "$ENV_FILE" <<EOF
|
||||
# OpenClaw Watchdog Configuration
|
||||
# Edit these values as needed, then restart the service:
|
||||
# sudo systemctl restart $SERVICE_NAME
|
||||
|
||||
HEALTH_URL=http://localhost:3000/health
|
||||
CHECK_INTERVAL=30
|
||||
FAIL_THRESHOLD=3
|
||||
OPENCLAW_CONFIG_PATH=$HOME/.openclaw/config.yaml
|
||||
|
||||
# Notification Settings
|
||||
TELEGRAM_BOT_TOKEN=${TELEGRAM_BOT_TOKEN}
|
||||
TELEGRAM_CHAT=1793951355
|
||||
GOTIFY_URL=${GOTIFY_URL}
|
||||
GOTIFY_TOKEN=${GOTIFY_TOKEN}
|
||||
|
||||
# Legacy Discord (optional)
|
||||
# DISCORD_CHANNEL_ID=
|
||||
# DISCORD_BOT_TOKEN=
|
||||
EOF
|
||||
chmod 600 "$ENV_FILE"
|
||||
echo "✅ Created config → $ENV_FILE"
|
||||
|
||||
# Create systemd service
|
||||
SERVICE_FILE="/etc/systemd/system/${SERVICE_NAME}.service"
|
||||
sudo tee "$SERVICE_FILE" >/dev/null <<EOF
|
||||
[Unit]
|
||||
Description=OpenClaw Watchdog — Gateway health monitor & auto-recovery
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=$USER
|
||||
EnvironmentFile=$ENV_FILE
|
||||
ExecStart=/usr/bin/env bash $INSTALL_DIR/watchdog.sh
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
EOF
|
||||
|
||||
echo "✅ Created systemd service → $SERVICE_FILE"
|
||||
|
||||
# Enable and start
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl enable "$SERVICE_NAME"
|
||||
sudo systemctl start "$SERVICE_NAME"
|
||||
|
||||
echo ""
|
||||
echo "🎉 Watchdog is running!"
|
||||
echo ""
|
||||
echo " Status: sudo systemctl status $SERVICE_NAME"
|
||||
echo " Logs: tail -f $INSTALL_DIR/watchdog.log"
|
||||
echo " Config: $ENV_FILE"
|
||||
echo ""
|
||||
echo "You'll get alerts via:"
|
||||
[[ -n "$TELEGRAM_BOT_TOKEN" ]] && echo " ✅ Telegram"
|
||||
[[ -n "$GOTIFY_TOKEN" ]] && echo " ✅ Gotify ($GOTIFY_URL)"
|
||||
192
openclaw-watchdog/watchdog.sh
Executable file
192
openclaw-watchdog/watchdog.sh
Executable file
@@ -0,0 +1,192 @@
|
||||
#!/usr/bin/env bash
|
||||
# openclaw-watchdog — Monitor OpenClaw gateway, auto-recover from crashes
|
||||
# https://github.com/jlgrimes/openclaw-watchdog
|
||||
# MIT License — Jared Grimes
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# ─── Configuration (all overridable via env vars) ────────────────────────────
|
||||
OPENCLAW_CONFIG_PATH="${OPENCLAW_CONFIG_PATH:-$HOME/.openclaw/config.yaml}"
|
||||
HEALTH_URL="${HEALTH_URL:-http://localhost:3000/health}"
|
||||
CHECK_INTERVAL="${CHECK_INTERVAL:-30}" # seconds between checks
|
||||
FAIL_THRESHOLD="${FAIL_THRESHOLD:-3}" # consecutive failures before escalation
|
||||
WATCHDOG_LOG="${WATCHDOG_LOG:-$HOME/.openclaw/watchdog.log}"
|
||||
GOOD_CONFIG_PATH="${GOOD_CONFIG_PATH:-$HOME/.openclaw/config.yaml.good}"
|
||||
|
||||
# Notification settings (Telegram + Gotify)
|
||||
TELEGRAM_BOT_TOKEN="${TELEGRAM_BOT_TOKEN:-}"
|
||||
TELEGRAM_CHAT="${TELEGRAM_CHAT:-1793951355}"
|
||||
GOTIFY_URL="${GOTIFY_URL:-http://runtipi.kangaroo-eel.ts.net:8129}"
|
||||
GOTIFY_TOKEN="${GOTIFY_TOKEN:-AGKnHafW3FGzBlt}"
|
||||
|
||||
# Legacy Discord settings (optional, for backwards compat)
|
||||
DISCORD_CHANNEL_ID="${DISCORD_CHANNEL_ID:-}"
|
||||
DISCORD_BOT_TOKEN="${DISCORD_BOT_TOKEN:-}"
|
||||
|
||||
# ─── State ────────────────────────────────────────────────────────────────────
|
||||
fail_count=0
|
||||
alerted=false # true after SOS sent, prevents spam
|
||||
was_down=false # tracks if we're recovering
|
||||
|
||||
# ─── Logging ──────────────────────────────────────────────────────────────────
|
||||
log() {
|
||||
local ts
|
||||
ts="$(date '+%Y-%m-%d %H:%M:%S')"
|
||||
echo "[$ts] $*" | tee -a "$WATCHDOG_LOG"
|
||||
}
|
||||
|
||||
# ─── Discord messaging ───────────────────────────────────────────────────────
|
||||
discord_send() {
|
||||
local msg="$1"
|
||||
[[ -z "$DISCORD_CHANNEL_ID" ]] && return 0
|
||||
[[ -z "$DISCORD_BOT_TOKEN" ]] && return 0
|
||||
|
||||
curl -sf -X POST \
|
||||
"https://discord.com/api/v10/channels/${DISCORD_CHANNEL_ID}/messages" \
|
||||
-H "Authorization: Bot ${DISCORD_BOT_TOKEN}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"content\": $(printf '%s' "$msg" | jq -Rs .)}" \
|
||||
>/dev/null 2>&1 || log "WARN: Failed to send Discord message"
|
||||
}
|
||||
|
||||
# ─── Telegram messaging ──────────────────────────────────────────────────────
|
||||
telegram_send() {
|
||||
local msg="$1"
|
||||
[[ -z "$TELEGRAM_BOT_TOKEN" ]] && return 0
|
||||
[[ -z "$TELEGRAM_CHAT" ]] && return 0
|
||||
|
||||
curl -sf -X POST \
|
||||
"https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"chat_id\": \"$TELEGRAM_CHAT\", \"text\": $(printf '%s' "$msg" | jq -Rs .), \"parse_mode\": \"Markdown\"}" \
|
||||
>/dev/null 2>&1 || log "WARN: Failed to send Telegram message"
|
||||
}
|
||||
|
||||
# ─── Gotify messaging ────────────────────────────────────────────────────────
|
||||
gotify_send() {
|
||||
local title="$1"
|
||||
local msg="$2"
|
||||
local priority="${3:-5}"
|
||||
[[ -z "$GOTIFY_TOKEN" ]] && return 0
|
||||
|
||||
curl -sf -X POST \
|
||||
"${GOTIFY_URL}/message?token=${GOTIFY_TOKEN}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"title\": \"$title\", \"message\": \"$msg\", \"priority\": $priority}" \
|
||||
>/dev/null 2>&1 || log "WARN: Failed to send Gotify message"
|
||||
}
|
||||
|
||||
# ─── Send to all configured channels ─────────────────────────────────────────
|
||||
send_notification() {
|
||||
local title="$1"
|
||||
local msg="$2"
|
||||
local priority="${3:-5}"
|
||||
|
||||
discord_send "$msg"
|
||||
telegram_send "$msg"
|
||||
gotify_send "$title" "$msg" "$priority"
|
||||
}
|
||||
|
||||
# ─── Health check ─────────────────────────────────────────────────────────────
|
||||
check_health() {
|
||||
curl -sf --max-time 10 "$HEALTH_URL" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
# ─── Save last-known-good config ─────────────────────────────────────────────
|
||||
save_good_config() {
|
||||
if [[ -f "$OPENCLAW_CONFIG_PATH" ]]; then
|
||||
cp "$OPENCLAW_CONFIG_PATH" "$GOOD_CONFIG_PATH"
|
||||
fi
|
||||
}
|
||||
|
||||
# ─── Revert to last-known-good config ────────────────────────────────────────
|
||||
revert_config() {
|
||||
if [[ -f "$GOOD_CONFIG_PATH" ]]; then
|
||||
log "Reverting config to last-known-good snapshot"
|
||||
cp "$GOOD_CONFIG_PATH" "$OPENCLAW_CONFIG_PATH"
|
||||
return 0
|
||||
else
|
||||
log "WARN: No good config snapshot available to revert"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# ─── Restart gateway ─────────────────────────────────────────────────────────
|
||||
restart_gateway() {
|
||||
log "Restarting OpenClaw gateway..."
|
||||
openclaw gateway restart >/dev/null 2>&1 || true
|
||||
sleep 5
|
||||
}
|
||||
|
||||
# ─── SOS alert ────────────────────────────────────────────────────────────────
|
||||
send_sos() {
|
||||
local hostname
|
||||
hostname="$(hostname 2>/dev/null || echo 'unknown')"
|
||||
local title="🚨 OpenClaw Gateway DOWN"
|
||||
local msg="🚨 **OpenClaw Gateway DOWN** on \`${hostname}\`
|
||||
|
||||
Watchdog tried:
|
||||
1. ✅ Simple restart
|
||||
2. ✅ Config rollback + restart
|
||||
3. ❌ Still unreachable after ${FAIL_THRESHOLD}+ failures
|
||||
|
||||
**Manual intervention needed:**
|
||||
\`\`\`
|
||||
ssh ${USER}@${hostname}
|
||||
openclaw gateway status
|
||||
journalctl -u openclaw-gateway --since '10 min ago'
|
||||
\`\`\`"
|
||||
|
||||
log "CRITICAL: Sending SOS alert"
|
||||
send_notification "$title" "$msg" 10
|
||||
alerted=true
|
||||
}
|
||||
|
||||
# ─── Recovery notification ────────────────────────────────────────────────────
|
||||
send_recovery() {
|
||||
local hostname
|
||||
hostname="$(hostname 2>/dev/null || echo 'unknown')"
|
||||
local title="✅ OpenClaw Gateway Recovered"
|
||||
local msg="✅ **OpenClaw Gateway recovered** on \`${hostname}\` — back online!"
|
||||
log "Gateway recovered"
|
||||
send_notification "$title" "$msg" 5
|
||||
}
|
||||
|
||||
# ─── Main loop ────────────────────────────────────────────────────────────────
|
||||
main() {
|
||||
log "Watchdog started (interval=${CHECK_INTERVAL}s, threshold=${FAIL_THRESHOLD}, health=${HEALTH_URL})"
|
||||
log "Notifications: Telegram=${TELEGRAM_CHAT:+enabled}, Gotify=${GOTIFY_TOKEN:+enabled}, Discord=${DISCORD_CHANNEL_ID:+enabled}"
|
||||
|
||||
while true; do
|
||||
if check_health; then
|
||||
# ── Healthy ──
|
||||
if [[ "$was_down" == true ]]; then
|
||||
send_recovery
|
||||
was_down=false
|
||||
alerted=false
|
||||
fi
|
||||
fail_count=0
|
||||
save_good_config
|
||||
else
|
||||
# ── Unhealthy ──
|
||||
fail_count=$((fail_count + 1))
|
||||
log "Health check failed (${fail_count}/${FAIL_THRESHOLD})"
|
||||
|
||||
if [[ $fail_count -eq 1 ]]; then
|
||||
# Stage 1: simple restart
|
||||
was_down=true
|
||||
restart_gateway
|
||||
elif [[ $fail_count -eq 2 ]]; then
|
||||
# Stage 2: config revert + restart
|
||||
revert_config && restart_gateway
|
||||
elif [[ $fail_count -ge $FAIL_THRESHOLD && "$alerted" == false ]]; then
|
||||
# Stage 3: SOS (once)
|
||||
send_sos
|
||||
fi
|
||||
fi
|
||||
|
||||
sleep "$CHECK_INTERVAL"
|
||||
done
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Reference in New Issue
Block a user