AI Newsletter Digest improvements: fixed QP soft line break decoding, URL extraction, and content cleaning

scripts/verify-backup.sh

@@ -0,0 +1,90 @@
+#!/bin/bash
+# Backup verification script - checks that backups are actually happening
+# Run this as a cron job or heartbeat check
+
+set -e
+
+GITEA_TOKEN="ba94c160b97c3a0fa5cf528ecc107eb2c8cddaa7"
+REPO_URL="http://git:${GITEA_TOKEN}@gitea.kangaroo-eel.ts.net:3000/Anthony/openclaw-backup.git"
+LOG_FILE="$HOME/.openclaw/workspace/logs/backup-verify.log"
+GOTIFY_URL="http://runtipi.kangaroo-eel.ts.net:8129"
+GOTIFY_TOKEN="AGoV3cAUyUMDbyt"
+
+# Max age in hours before we consider backup stale
+MAX_AGE_HOURS=26  # Slightly more than 24 to account for cron jitter
+
+log() {
+    echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG_FILE"
+}
+
+notify() {
+    local title="$1"
+    local message="$2"
+    local priority="${3:-5}"
+    curl -s -X POST \
+        -H 'Content-Type: application/json' \
+        "${GOTIFY_URL}/message?token=${GOTIFY_TOKEN}" \
+        -d "{\"title\": \"${title}\", \"message\": \"${message}\", \"priority\": ${priority}}" > /dev/null || true
+}
+
+# Ensure log directory exists
+mkdir -p "$(dirname "$LOG_FILE")"
+
+log "Starting backup verification..."
+
+# Clone minimal repo to check the last commit
+REPO_DIR=$(mktemp -d)
+if ! git clone --depth 1 "$REPO_URL" "$REPO_DIR" 2>/dev/null; then
+    log "ERROR: Could not clone repo"
+    notify "🚨 BACKUP VERIFICATION FAILED" "Cannot clone Gitea repo" 8
+    rm -rf "$REPO_DIR"
+    exit 1
+fi
+
+cd "$REPO_DIR"
+
+# Get last commit timestamp
+LAST_COMMIT_DATE=$(git log -1 --format="%ci" 2>/dev/null || echo "")
+LAST_COMMIT_MSG=$(git log -1 --format="%s" 2>/dev/null || echo "")
+
+if [ -z "$LAST_COMMIT_DATE" ]; then
+    log "ERROR: Could not get last commit date"
+    notify "🚨 BACKUP VERIFICATION FAILED" "Repo has no commits" 8
+    rm -rf "$REPO_DIR"
+    exit 1
+fi
+
+# Parse the date (handle both GNU and BSD date)
+LAST_COMMIT_EPOCH=$(date -d "$LAST_COMMIT_DATE" +%s 2>/dev/null || date -j -f "%Y-%m-%d %H:%M:%S %z" "$LAST_COMMIT_DATE" +%s 2>/dev/null)
+CURRENT_EPOCH=$(date +%s)
+AGE_HOURS=$(( (CURRENT_EPOCH - LAST_COMMIT_EPOCH) / 3600 ))
+
+log "Last backup: $LAST_COMMIT_DATE ($AGE_HOURS hours ago)"
+log "Commit message: $LAST_COMMIT_MSG"
+
+if [ "$AGE_HOURS" -gt "$MAX_AGE_HOURS" ]; then
+    log "ALERT: Backup is stale! Last commit was $AGE_HOURS hours ago"
+    notify "🚨 BACKUP STALE" "Last backup was $AGE_HOURS hours ago (expected within $MAX_AGE_HOURS hours)\n\nLast commit: $LAST_COMMIT_DATE\nMessage: $LAST_COMMIT_MSG" 8
+    rm -rf "$REPO_DIR"
+    exit 1
+fi
+
+# Verify critical files exist in the latest commit
+log "Verifying critical files in backup..."
+MISSING_FILES=()
+[ -f "$REPO_DIR/openclaw-state/openclaw.json" ] || MISSING_FILES+=("openclaw.json")
+[ -f "$REPO_DIR/MEMORY.md" ] || MISSING_FILES+=("MEMORY.md")
+[ -d "$REPO_DIR/openclaw-state/cron" ] || MISSING_FILES+=("cron/")
+
+if [ ${#MISSING_FILES[@]} -gt 0 ]; then
+    log "ERROR: Critical files missing: ${MISSING_FILES[*]}"
+    notify "🚨 BACKUP INCOMPLETE" "Missing critical files: ${MISSING_FILES[*]}" 8
+    rm -rf "$REPO_DIR"
+    exit 1
+else
+    log "✅ All critical files present"
+fi
+
+log "✅ Backup verification PASSED"
+rm -rf "$REPO_DIR"
+exit 0