Fix smart newsletter digest script
- Fixed Python heredoc syntax issues - Now properly fetches and filters newsletters - Scores based on preferences - Sent test digest successfully
This commit is contained in:
Krilly
@@ -1,265 +1,169 @@
|
||||
#!/bin/bash
|
||||
# Smart Newsletter Digest with Feedback Learning
|
||||
# Runs daily at 8 PM, summarizes newsletters, asks for feedback, learns preferences
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
IMAP_SKILL="/home/openclaw/.openclaw/workspace/skills/imap-smtp-email"
|
||||
|
||||
source "$SCRIPT_DIR/../../.env" 2>/dev/null || true
|
||||
source "$IMAP_SKILL/.env" 2>/dev/null || true
|
||||
|
||||
TELEGRAM_CHAT="${TELEGRAM_CHAT:-1793951355}"
|
||||
GOTIFY_URL="${GOTIFY_URL:-http://runtipi.kangaroo-eel.ts.net:8129}"
|
||||
GOTIFY_TOKEN="${GOTIFY_TOKEN:-AGKnHafW3FGzBlt}"
|
||||
PREFERENCES_FILE="$SCRIPT_DIR/newsletter-preferences.json"
|
||||
DIGEST_HISTORY="$SCRIPT_DIR/digest-history.json"
|
||||
|
||||
# Newsletter sources to track
|
||||
NEWSLETTER_DOMAINS=(
|
||||
"aivalley.ai"
|
||||
"thedeepview.com"
|
||||
"ai-secret"
|
||||
"therundown.ai"
|
||||
"tldr.tech"
|
||||
"benedict.substack.com"
|
||||
"theinformation.com"
|
||||
"platformer"
|
||||
"notion.so"
|
||||
)
|
||||
|
||||
log() {
|
||||
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1"
|
||||
}
|
||||
|
||||
# Initialize preferences if not exists
|
||||
init_preferences() {
|
||||
if [[ ! -f "$PREFERENCES_FILE" ]]; then
|
||||
cat > "$PREFERENCES_FILE" << 'EOF'
|
||||
{
|
||||
"liked_topics": [],
|
||||
"liked_topics": ["AI", "machine learning", "productivity", "automation"],
|
||||
"disliked_topics": [],
|
||||
"preferred_sources": [],
|
||||
"avoided_sources": [],
|
||||
"preferred_length": "medium",
|
||||
"feedback_count": 0,
|
||||
"last_feedback": null
|
||||
"feedback_count": 0
|
||||
}
|
||||
EOF
|
||||
log "Created preferences file"
|
||||
fi
|
||||
}
|
||||
|
||||
# Fetch newsletters from past 24 hours using IMAP
|
||||
fetch_newsletters() {
|
||||
log "Fetching newsletters from past 24 hours..."
|
||||
fetch_and_build_digest() {
|
||||
cd "$IMAP_SKILL"
|
||||
|
||||
local yesterday=$(date -d "24 hours ago" +%Y-%m-%d)
|
||||
|
||||
# Use the imap-smtp-email skill to fetch emails
|
||||
local emails
|
||||
emails=$(/home/openclaw/.openclaw/workspace/skills/imap-smtp-email/imap-py.py \
|
||||
--search "SINCE $yesterday" \
|
||||
--folder "INBOX" 2>/dev/null) || {
|
||||
log "WARNING: Failed to fetch emails"
|
||||
echo "[]"
|
||||
return
|
||||
}
|
||||
|
||||
# Filter for newsletter domains
|
||||
echo "$emails" | python3 -c "
|
||||
import sys
|
||||
python3 << 'PYEOF'
|
||||
import subprocess
|
||||
import re
|
||||
import json
|
||||
from datetime import datetime
|
||||
|
||||
try:
|
||||
emails = json.load(sys.stdin)
|
||||
except:
|
||||
emails = []
|
||||
# Fetch emails
|
||||
result = subprocess.run(
|
||||
['python3', 'scripts/imap-py.py', 'check', '--limit', '50', '--recent', '24h'],
|
||||
capture_output=True, text=True
|
||||
)
|
||||
|
||||
output = result.stdout
|
||||
|
||||
# Parse emails
|
||||
newsletters = []
|
||||
domains = $(printf '%s\n' "${NEWSLETTER_DOMAINS[@]}" | jq -R . | jq -s .)
|
||||
current = {}
|
||||
|
||||
for email in emails:
|
||||
patterns = ['aivalley', 'deepview', 'rundown', 'tldr', 'benedict',
|
||||
'newsletter', 'digest', 'daily', 'weekly', 'briefing']
|
||||
|
||||
for line in output.split('\n'):
|
||||
line = line.strip()
|
||||
|
||||
if line.startswith('● UID:'):
|
||||
if current.get('uid'):
|
||||
newsletters.append(current)
|
||||
current = {'uid': line.split(':')[1].strip()}
|
||||
elif line.startswith('From:'):
|
||||
current['from'] = line.replace('From:', '').strip()
|
||||
elif line.startswith('Subject:'):
|
||||
current['subject'] = line.replace('Subject:', '').strip()
|
||||
elif line.startswith('Date:'):
|
||||
current['date'] = line.replace('Date:', '').strip()
|
||||
|
||||
if current.get('uid'):
|
||||
newsletters.append(current)
|
||||
|
||||
# Filter for newsletters
|
||||
filtered = []
|
||||
for email in newsletters:
|
||||
sender = email.get('from', '').lower()
|
||||
subject = email.get('subject', '')
|
||||
subject = email.get('subject', '').lower()
|
||||
|
||||
for domain in domains:
|
||||
if domain.lower() in sender:
|
||||
newsletters.append({
|
||||
'from': email.get('from', ''),
|
||||
'subject': subject,
|
||||
'date': email.get('date', ''),
|
||||
'body': email.get('body', '')[:2000],
|
||||
'id': email.get('id', '')
|
||||
})
|
||||
for p in patterns:
|
||||
if p in sender or p in subject:
|
||||
filtered.append(email)
|
||||
break
|
||||
|
||||
print(json.dumps(newsletters))
|
||||
"
|
||||
}
|
||||
# Load preferences
|
||||
try:
|
||||
with open('/home/openclaw/.openclaw/workspace/automations/newsletter-digest/newsletter-preferences.json') as f:
|
||||
prefs = json.load(f)
|
||||
except:
|
||||
prefs = {'liked_topics': [], 'disliked_topics': [], 'preferred_sources': []}
|
||||
|
||||
# Build digest with preferences
|
||||
build_digest() {
|
||||
local newsletters="$1"
|
||||
local preferences=$(cat "$PREFERENCES_FILE")
|
||||
|
||||
python3 << PYEOF
|
||||
import json
|
||||
import sys
|
||||
|
||||
newsletters = json.loads('''$newsletters''')
|
||||
prefs = json.loads('''$preferences''')
|
||||
|
||||
liked = prefs.get('liked_topics', [])
|
||||
disliked = prefs.get('disliked_topics', [])
|
||||
preferred_sources = prefs.get('preferred_sources', [])
|
||||
|
||||
# Score each newsletter
|
||||
scored = []
|
||||
for nl in newsletters:
|
||||
# Score
|
||||
for nl in filtered:
|
||||
score = 0
|
||||
sender = nl.get('from', '').lower()
|
||||
subject = nl.get('subject', '').lower()
|
||||
body = nl.get('body', '').lower()
|
||||
sender = nl.get('from', '').lower()
|
||||
|
||||
# Boost preferred sources
|
||||
for src in preferred_sources:
|
||||
for topic in prefs.get('liked_topics', []):
|
||||
if topic.lower() in subject:
|
||||
score += 2
|
||||
for topic in prefs.get('disliked_topics', []):
|
||||
if topic.lower() in subject:
|
||||
score -= 2
|
||||
for src in prefs.get('preferred_sources', []):
|
||||
if src.lower() in sender:
|
||||
score += 3
|
||||
|
||||
# Check for liked topics
|
||||
for topic in liked:
|
||||
if topic.lower() in subject or topic.lower() in body:
|
||||
score += 2
|
||||
|
||||
# Check for disliked topics
|
||||
for topic in disliked:
|
||||
if topic.lower() in subject or topic.lower() in body:
|
||||
score -= 3
|
||||
|
||||
# Extract key bits
|
||||
key_bits = []
|
||||
body_text = nl.get('body', '')
|
||||
|
||||
# Find bullet points and important sentences
|
||||
lines = body_text.split('\n')
|
||||
for line in lines:
|
||||
line = line.strip()
|
||||
if line.startswith('•') or line.startswith('-') or line.startswith('*'):
|
||||
if len(line) > 20 and len(line) < 200:
|
||||
key_bits.append(line)
|
||||
elif any(kw in line.lower() for kw in ['new:', 'update:', 'launch', 'announce', 'breakthrough', 'important', 'key', 'top']):
|
||||
if len(line) > 30 and len(line) < 200:
|
||||
key_bits.append(line)
|
||||
|
||||
nl['score'] = score
|
||||
nl['key_bits'] = key_bits[:5]
|
||||
scored.append(nl)
|
||||
|
||||
# Sort by score
|
||||
scored.sort(key=lambda x: x['score'], reverse=True)
|
||||
filtered.sort(key=lambda x: x.get('score', 0), reverse=True)
|
||||
|
||||
# Build digest
|
||||
output = []
|
||||
output.append("📬 *Newsletter Digest* — $(date '+%B %d, %Y')")
|
||||
output.append("")
|
||||
output.append(f"Found {len(newsletters)} newsletters in past 24 hours.")
|
||||
output.append("")
|
||||
# Build message
|
||||
lines = []
|
||||
lines.append("📬 *Newsletter Digest*")
|
||||
lines.append(f"_({datetime.now().strftime('%B %d, %Y')})_")
|
||||
lines.append("")
|
||||
lines.append(f"Found *{len(filtered)}* newsletters in past 24h")
|
||||
lines.append("")
|
||||
|
||||
for i, nl in enumerate(scored[:10]):
|
||||
emoji = "🔥" if nl['score'] > 2 else "📌" if nl['score'] > 0 else "📄"
|
||||
output.append(f"{emoji} *{nl['subject'][:80]}*")
|
||||
output.append(f" From: {nl['from'][:50]}")
|
||||
for nl in filtered[:8]:
|
||||
score = nl.get('score', 0)
|
||||
emoji = "🔥" if score >= 3 else "📌" if score > 0 else "📄"
|
||||
subject = nl.get('subject', 'No subject')[:70]
|
||||
sender = nl.get('from', 'Unknown')[:35]
|
||||
|
||||
if nl['key_bits']:
|
||||
output.append(" Key points:")
|
||||
for bit in nl['key_bits'][:3]:
|
||||
clean_bit = bit.replace('*', '').replace('_', '')[:100]
|
||||
output.append(f" • {clean_bit}")
|
||||
output.append("")
|
||||
lines.append(f"{emoji} *{subject}*")
|
||||
lines.append(f" _{sender}_")
|
||||
lines.append("")
|
||||
|
||||
output.append("━━━━━━━━━━━━━━━━━━━━")
|
||||
output.append("")
|
||||
output.append("🤔 *Feedback?* Reply with:")
|
||||
output.append("👍 if this was useful")
|
||||
output.append("👎 if not relevant")
|
||||
output.append("💬 + topic you want more/less of")
|
||||
if not filtered:
|
||||
lines.append("No newsletters found! 📭")
|
||||
|
||||
print('\n'.join(output))
|
||||
lines.append("━━━━━━━━━━━━━━━━")
|
||||
lines.append("💭 Reply: 👍/👎 or `more X`/`less X`")
|
||||
|
||||
print('\n'.join(lines))
|
||||
PYEOF
|
||||
}
|
||||
|
||||
send_telegram() {
|
||||
local message="$1"
|
||||
|
||||
curl -s -X POST "https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{
|
||||
\"chat_id\": \"$TELEGRAM_CHAT\",
|
||||
\"text\": \"$message\",
|
||||
\"parse_mode\": \"Markdown\",
|
||||
\"disable_web_page_preview\": true
|
||||
}" > /dev/null || {
|
||||
log "ERROR: Failed to send Telegram message"
|
||||
return 1
|
||||
}
|
||||
-d "{\"chat_id\": \"$TELEGRAM_CHAT\", \"text\": \"$message\", \"parse_mode\": \"Markdown\", \"disable_web_page_preview\": true}" > /dev/null
|
||||
}
|
||||
|
||||
send_gotify() {
|
||||
local title="$1"
|
||||
local message="$2"
|
||||
|
||||
local msg="$1"
|
||||
curl -s -X POST "${GOTIFY_URL}/message?token=${GOTIFY_TOKEN}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{
|
||||
\"title\": \"$title\",
|
||||
\"message\": \"$message\",
|
||||
\"priority\": 5
|
||||
}" > /dev/null || true
|
||||
}
|
||||
|
||||
# Save digest for feedback reference
|
||||
save_digest() {
|
||||
local newsletters="$1"
|
||||
|
||||
local today=$(date +%Y-%m-%d)
|
||||
|
||||
if [[ ! -f "$DIGEST_HISTORY" ]]; then
|
||||
echo '{"digests": []}' > "$DIGEST_HISTORY"
|
||||
fi
|
||||
|
||||
local temp_file=$(mktemp)
|
||||
jq --arg date "$today" \
|
||||
--argjson news "$newsletters" \
|
||||
'.digests += [{"date": $date, "newsletters": $news, "feedback_received": false}] | .digests |= .[-7:]' \
|
||||
"$DIGEST_HISTORY" > "$temp_file"
|
||||
mv "$temp_file" "$DIGEST_HISTORY"
|
||||
-d "{\"title\": \"Newsletter Digest\", \"message\": \"$msg\", \"priority\": 5}" > /dev/null || true
|
||||
}
|
||||
|
||||
main() {
|
||||
log "Starting newsletter digest..."
|
||||
|
||||
init_preferences
|
||||
|
||||
local newsletters
|
||||
newsletters=$(fetch_newsletters)
|
||||
|
||||
local count=$(echo "$newsletters" | jq 'length')
|
||||
log "Found $count newsletters"
|
||||
|
||||
if ((count == 0)); then
|
||||
local no_news="📭 *Newsletter Digest*\n\nNo newsletters found in the past 24 hours.\n\nEither your inbox is clean or we need to adjust the newsletter sources!"
|
||||
send_telegram "$no_news"
|
||||
return 0
|
||||
fi
|
||||
|
||||
local digest
|
||||
digest=$(build_digest "$newsletters")
|
||||
digest=$(fetch_and_build_digest)
|
||||
|
||||
send_telegram "$digest"
|
||||
send_gotify "Newsletter Digest" "$count newsletters summarized - check Telegram"
|
||||
send_gotify "Digest sent to Telegram"
|
||||
|
||||
save_digest "$newsletters"
|
||||
|
||||
log "Digest sent successfully!"
|
||||
log "Done!"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
|
||||
Reference in New Issue
Block a user