Add smart newsletter digest with feedback learning
- Runs daily at 8 PM Australia/Perth time - Fetches newsletters from past 24 hours via IMAP - Scores based on learned preferences (topics, sources) - Summarizes key points and highlights - Sends digest to Telegram + Gotify - Learns from user feedback (👍/👎/topic preferences) - Preferences stored in newsletter-preferences.json
This commit is contained in:
Krilly
265
automations/newsletter-digest/smart-digest.sh
Executable file
265
automations/newsletter-digest/smart-digest.sh
Executable file
@@ -0,0 +1,265 @@
|
||||
#!/bin/bash
|
||||
# Smart Newsletter Digest with Feedback Learning
|
||||
# Runs daily at 8 PM, summarizes newsletters, asks for feedback, learns preferences
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "$SCRIPT_DIR/../../.env" 2>/dev/null || true
|
||||
|
||||
TELEGRAM_CHAT="${TELEGRAM_CHAT:-1793951355}"
|
||||
GOTIFY_URL="${GOTIFY_URL:-http://runtipi.kangaroo-eel.ts.net:8129}"
|
||||
GOTIFY_TOKEN="${GOTIFY_TOKEN:-AGKnHafW3FGzBlt}"
|
||||
PREFERENCES_FILE="$SCRIPT_DIR/newsletter-preferences.json"
|
||||
DIGEST_HISTORY="$SCRIPT_DIR/digest-history.json"
|
||||
|
||||
# Newsletter sources to track
|
||||
NEWSLETTER_DOMAINS=(
|
||||
"aivalley.ai"
|
||||
"thedeepview.com"
|
||||
"ai-secret"
|
||||
"therundown.ai"
|
||||
"tldr.tech"
|
||||
"benedict.substack.com"
|
||||
"theinformation.com"
|
||||
"platformer"
|
||||
"notion.so"
|
||||
)
|
||||
|
||||
log() {
|
||||
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1"
|
||||
}
|
||||
|
||||
# Initialize preferences if not exists
|
||||
init_preferences() {
|
||||
if [[ ! -f "$PREFERENCES_FILE" ]]; then
|
||||
cat > "$PREFERENCES_FILE" << 'EOF'
|
||||
{
|
||||
"liked_topics": [],
|
||||
"disliked_topics": [],
|
||||
"preferred_sources": [],
|
||||
"avoided_sources": [],
|
||||
"preferred_length": "medium",
|
||||
"feedback_count": 0,
|
||||
"last_feedback": null
|
||||
}
|
||||
EOF
|
||||
log "Created preferences file"
|
||||
fi
|
||||
}
|
||||
|
||||
# Fetch newsletters from past 24 hours using IMAP
|
||||
fetch_newsletters() {
|
||||
log "Fetching newsletters from past 24 hours..."
|
||||
|
||||
local yesterday=$(date -d "24 hours ago" +%Y-%m-%d)
|
||||
|
||||
# Use the imap-smtp-email skill to fetch emails
|
||||
local emails
|
||||
emails=$(/home/openclaw/.openclaw/workspace/skills/imap-smtp-email/imap-py.py \
|
||||
--search "SINCE $yesterday" \
|
||||
--folder "INBOX" 2>/dev/null) || {
|
||||
log "WARNING: Failed to fetch emails"
|
||||
echo "[]"
|
||||
return
|
||||
}
|
||||
|
||||
# Filter for newsletter domains
|
||||
echo "$emails" | python3 -c "
|
||||
import sys
|
||||
import json
|
||||
|
||||
try:
|
||||
emails = json.load(sys.stdin)
|
||||
except:
|
||||
emails = []
|
||||
|
||||
newsletters = []
|
||||
domains = $(printf '%s\n' "${NEWSLETTER_DOMAINS[@]}" | jq -R . | jq -s .)
|
||||
|
||||
for email in emails:
|
||||
sender = email.get('from', '').lower()
|
||||
subject = email.get('subject', '')
|
||||
|
||||
for domain in domains:
|
||||
if domain.lower() in sender:
|
||||
newsletters.append({
|
||||
'from': email.get('from', ''),
|
||||
'subject': subject,
|
||||
'date': email.get('date', ''),
|
||||
'body': email.get('body', '')[:2000],
|
||||
'id': email.get('id', '')
|
||||
})
|
||||
break
|
||||
|
||||
print(json.dumps(newsletters))
|
||||
"
|
||||
}
|
||||
|
||||
# Build digest with preferences
|
||||
build_digest() {
|
||||
local newsletters="$1"
|
||||
local preferences=$(cat "$PREFERENCES_FILE")
|
||||
|
||||
python3 << PYEOF
|
||||
import json
|
||||
import sys
|
||||
|
||||
newsletters = json.loads('''$newsletters''')
|
||||
prefs = json.loads('''$preferences''')
|
||||
|
||||
liked = prefs.get('liked_topics', [])
|
||||
disliked = prefs.get('disliked_topics', [])
|
||||
preferred_sources = prefs.get('preferred_sources', [])
|
||||
|
||||
# Score each newsletter
|
||||
scored = []
|
||||
for nl in newsletters:
|
||||
score = 0
|
||||
sender = nl.get('from', '').lower()
|
||||
subject = nl.get('subject', '').lower()
|
||||
body = nl.get('body', '').lower()
|
||||
|
||||
# Boost preferred sources
|
||||
for src in preferred_sources:
|
||||
if src.lower() in sender:
|
||||
score += 3
|
||||
|
||||
# Check for liked topics
|
||||
for topic in liked:
|
||||
if topic.lower() in subject or topic.lower() in body:
|
||||
score += 2
|
||||
|
||||
# Check for disliked topics
|
||||
for topic in disliked:
|
||||
if topic.lower() in subject or topic.lower() in body:
|
||||
score -= 3
|
||||
|
||||
# Extract key bits
|
||||
key_bits = []
|
||||
body_text = nl.get('body', '')
|
||||
|
||||
# Find bullet points and important sentences
|
||||
lines = body_text.split('\n')
|
||||
for line in lines:
|
||||
line = line.strip()
|
||||
if line.startswith('•') or line.startswith('-') or line.startswith('*'):
|
||||
if len(line) > 20 and len(line) < 200:
|
||||
key_bits.append(line)
|
||||
elif any(kw in line.lower() for kw in ['new:', 'update:', 'launch', 'announce', 'breakthrough', 'important', 'key', 'top']):
|
||||
if len(line) > 30 and len(line) < 200:
|
||||
key_bits.append(line)
|
||||
|
||||
nl['score'] = score
|
||||
nl['key_bits'] = key_bits[:5]
|
||||
scored.append(nl)
|
||||
|
||||
# Sort by score
|
||||
scored.sort(key=lambda x: x['score'], reverse=True)
|
||||
|
||||
# Build digest
|
||||
output = []
|
||||
output.append("📬 *Newsletter Digest* — $(date '+%B %d, %Y')")
|
||||
output.append("")
|
||||
output.append(f"Found {len(newsletters)} newsletters in past 24 hours.")
|
||||
output.append("")
|
||||
|
||||
for i, nl in enumerate(scored[:10]):
|
||||
emoji = "🔥" if nl['score'] > 2 else "📌" if nl['score'] > 0 else "📄"
|
||||
output.append(f"{emoji} *{nl['subject'][:80]}*")
|
||||
output.append(f" From: {nl['from'][:50]}")
|
||||
|
||||
if nl['key_bits']:
|
||||
output.append(" Key points:")
|
||||
for bit in nl['key_bits'][:3]:
|
||||
clean_bit = bit.replace('*', '').replace('_', '')[:100]
|
||||
output.append(f" • {clean_bit}")
|
||||
output.append("")
|
||||
|
||||
output.append("━━━━━━━━━━━━━━━━━━━━")
|
||||
output.append("")
|
||||
output.append("🤔 *Feedback?* Reply with:")
|
||||
output.append("👍 if this was useful")
|
||||
output.append("👎 if not relevant")
|
||||
output.append("💬 + topic you want more/less of")
|
||||
|
||||
print('\n'.join(output))
|
||||
PYEOF
|
||||
}
|
||||
|
||||
send_telegram() {
|
||||
local message="$1"
|
||||
|
||||
curl -s -X POST "https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{
|
||||
\"chat_id\": \"$TELEGRAM_CHAT\",
|
||||
\"text\": \"$message\",
|
||||
\"parse_mode\": \"Markdown\",
|
||||
\"disable_web_page_preview\": true
|
||||
}" > /dev/null || {
|
||||
log "ERROR: Failed to send Telegram message"
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
send_gotify() {
|
||||
local title="$1"
|
||||
local message="$2"
|
||||
|
||||
curl -s -X POST "${GOTIFY_URL}/message?token=${GOTIFY_TOKEN}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{
|
||||
\"title\": \"$title\",
|
||||
\"message\": \"$message\",
|
||||
\"priority\": 5
|
||||
}" > /dev/null || true
|
||||
}
|
||||
|
||||
# Save digest for feedback reference
|
||||
save_digest() {
|
||||
local newsletters="$1"
|
||||
|
||||
local today=$(date +%Y-%m-%d)
|
||||
|
||||
if [[ ! -f "$DIGEST_HISTORY" ]]; then
|
||||
echo '{"digests": []}' > "$DIGEST_HISTORY"
|
||||
fi
|
||||
|
||||
local temp_file=$(mktemp)
|
||||
jq --arg date "$today" \
|
||||
--argjson news "$newsletters" \
|
||||
'.digests += [{"date": $date, "newsletters": $news, "feedback_received": false}] | .digests |= .[-7:]' \
|
||||
"$DIGEST_HISTORY" > "$temp_file"
|
||||
mv "$temp_file" "$DIGEST_HISTORY"
|
||||
}
|
||||
|
||||
main() {
|
||||
log "Starting newsletter digest..."
|
||||
|
||||
init_preferences
|
||||
|
||||
local newsletters
|
||||
newsletters=$(fetch_newsletters)
|
||||
|
||||
local count=$(echo "$newsletters" | jq 'length')
|
||||
log "Found $count newsletters"
|
||||
|
||||
if ((count == 0)); then
|
||||
local no_news="📭 *Newsletter Digest*\n\nNo newsletters found in the past 24 hours.\n\nEither your inbox is clean or we need to adjust the newsletter sources!"
|
||||
send_telegram "$no_news"
|
||||
return 0
|
||||
fi
|
||||
|
||||
local digest
|
||||
digest=$(build_digest "$newsletters")
|
||||
|
||||
send_telegram "$digest"
|
||||
send_gotify "Newsletter Digest" "$count newsletters summarized - check Telegram"
|
||||
|
||||
save_digest "$newsletters"
|
||||
|
||||
log "Digest sent successfully!"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Reference in New Issue
Block a user