AI Newsletter Digest improvements: fixed QP soft line break decoding, URL extraction, and content cleaning

scripts/anthonymau-email-check.sh

@@ -0,0 +1,106 @@
+#!/bin/bash
+# Check for new AI-related emails in anthonymau@gmail.com
+# Filters for AI newsletters: AI Valley, The Rundown AI, AI Secret, Byte-Sized AI, etc.
+# Only alerts on emails with UID higher than last seen
+
+WORKSPACE_DIR="/home/openclaw/.openclaw/workspace"
+STATE_FILE="$WORKSPACE_DIR/memory/.anthonymau-email-state.json"
+SCRIPT_DIR="/home/openclaw/.openclaw/workspace/skills/imap-smtp-email"
+CHECK_SCRIPT="$SCRIPT_DIR/scripts/check-anthonymau-email.js"
+
+# Ensure state directory exists
+mkdir -p "$(dirname "$STATE_FILE")"
+
+# Initialize state if needed
+if [ ! -f "$STATE_FILE" ]; then
+  echo '{"last_uid": 0, "last_check": 0}' > "$STATE_FILE"
+fi
+
+# Get last seen UID
+LAST_UID=$(jq -r '.last_uid // 0' "$STATE_FILE")
+
+# Run the Node.js IMAP check (30s timeout)
+RESULT=$(cd "$SCRIPT_DIR" && NODE_TLS_REJECT_UNAUTHORIZED=0 timeout 30 node "$CHECK_SCRIPT" 2>&1)
+
+# Parse result
+STATUS=$(echo "$RESULT" | grep "^STATUS:" | cut -d: -f2)
+TOTAL=$(echo "$RESULT" | grep "^TOTAL:" | cut -d: -f2)
+LAST_UID_REMOTE=$(echo "$RESULT" | grep "^LAST_UID:" | cut -d: -f2)
+RECENT_UIDS=$(echo "$RESULT" | grep "^RECENT:" | cut -d: -f2)
+
+if [ "$STATUS" != "connected" ]; then
+  echo "Error connecting to IMAP: $RESULT" >&2
+  curl -s -X POST "http://127.0.0.1:18789/api/message/send" \
+    -H "Content-Type: application/json" \
+    -d '{"channel": "telegram", "to": "telegram:1793951355", "message": "⚠️ **anthonymau@gmail.com email check failed**\n\nIMAP connection failed. Check credentials."}' \
+    > /dev/null 2>&1
+  exit 1
+fi
+
+echo "✅ Connected! Total: $TOTAL, Last UID: $LAST_UID_REMOTE"
+
+# Get list of UIDs that are NEW (higher than last_uid)
+NEW_UIDS=""
+if [ -n "$RECENT_UIDS" ]; then
+  IFS=',' read -ra UID_ARRAY <<< "$RECENT_UIDS"
+  for MSG_UID in "${UID_ARRAY[@]}"; do
+    if [ "$MSG_UID" -gt "$LAST_UID" ]; then
+      if [ -n "$NEW_UIDS" ]; then
+        NEW_UIDS="$NEW_UIDS,$MSG_UID"
+      else
+        NEW_UIDS="$MSG_UID"
+      fi
+    fi
+  done
+fi
+
+if [ -z "$NEW_UIDS" ]; then
+  echo "No new emails since last check"
+  # Still update the check time
+  jq ".last_check = $(date +%s)" "$STATE_FILE" > "$STATE_FILE.tmp" && mv "$STATE_FILE.tmp" "$STATE_FILE"
+  exit 0
+fi
+
+echo "New UIDs to check: $NEW_UIDS"
+
+# Parse AI emails and filter for new ones only
+AI_EMAILS=$(echo "$RESULT" | grep "^AI_EMAIL:" | cut -d: -f2-)
+NEW_AI_COUNT=0
+ALERT_LINES=()
+
+# For simplicity, we'll fetch details for new UIDs only
+# The Node script already filtered to AI newsletters, so we count how many are in our new UID range
+if [ -n "$AI_EMAILS" ] && [ -n "$NEW_UIDS" ]; then
+  # Count AI emails (simplified - assumes AI emails in recent batch are new if any new UIDs exist)
+  while IFS= read -r line; do
+    if [ -n "$line" ]; then
+      FROM=$(echo "$line" | cut -d'|' -f1 | xargs)
+      SUBJECT=$(echo "$line" | cut -d'|' -f2- | xargs)
+      NEW_AI_COUNT=$((NEW_AI_COUNT + 1))
+      ALERT_LINES+=("📧 **$SUBJECT**\n   From: $FROM")
+    fi
+  done <<< "$AI_EMAILS"
+fi
+
+# Update state with latest UID
+if [ -n "$LAST_UID_REMOTE" ] && [ "$LAST_UID_REMOTE" != "0" ]; then
+  jq ".last_uid = $LAST_UID_REMOTE | .last_check = $(date +%s)" "$STATE_FILE" > "$STATE_FILE.tmp" && mv "$STATE_FILE.tmp" "$STATE_FILE"
+fi
+
+# Send alert if new AI emails found
+if [ "$NEW_AI_COUNT" -gt 0 ]; then
+  ALERT_MSG="🤖 **$NEW_AI_COUNT new AI newsletter(s)**:\n\n"
+  for LINE in "${ALERT_LINES[@]}"; do
+    ALERT_MSG+="$LINE\n\n"
+  done
+  ALERT_MSG+="— Krilly 🦀"
+  
+  curl -s -X POST "http://127.0.0.1:18789/api/message/send" \
+    -H "Content-Type: application/json" \
+    -d "{\"channel\": \"telegram\", \"to\": \"telegram:1793951355\", \"message\": $(echo "$ALERT_MSG" | jq -Rs .)}" \
+    > /dev/null 2>&1
+  
+  echo "🤖 Alert sent for $NEW_AI_COUNT AI newsletters"
+else
+  echo "No new AI newsletters since last check"
+fi