AI Newsletter Digest improvements: fixed QP soft line break decoding, URL extraction, and content cleaning

automations/ai-newsletter-digest/summarize.py

@@ -0,0 +1,151 @@
+#!/usr/bin/env python3
+"""
+AI Newsletter Summarizer
+Uses LLM to synthesize and summarize newsletter content
+"""
+import json
+import sys
+import os
+import subprocess
+from pathlib import Path
+from datetime import datetime
+
+def call_llm(prompt, model="kilocode/kilo/auto-free"):
+    """Call LLM via OpenClaw CLI."""
+    
+    cmd = [
+        "openclaw",
+        "llm",
+        "--model", model,
+        "--prompt", prompt
+    ]
+    
+    try:
+        result = subprocess.run(
+            cmd,
+            capture_output=True,
+            text=True,
+            timeout=120,
+            cwd="/home/openclaw/.openclaw/workspace"
+        )
+        
+        if result.returncode == 0:
+            return result.stdout.strip()
+        else:
+            print(f"LLM error: {result.stderr}", file=sys.stderr)
+            return None
+    except Exception as e:
+        print(f"LLM call failed: {e}", file=sys.stderr)
+        return None
+
+def summarize_newsletters(newsletters):
+    """Use LLM to summarize newsletters into a proper digest."""
+    
+    # Prepare newsletter content
+    content_parts = []
+    for i, nl in enumerate(newsletters, 1):
+        source = nl.get('from', 'Unknown').split('<')[0].strip()
+        subject = nl.get('subject', 'No subject')
+        content = nl.get('content', '')[:1500]  # Limit to ~1500 chars per newsletter
+        
+        content_parts.append(f"""
+--- NEWSLETTER {i} ---
+Source: {source}
+Subject: {subject}
+Content: {content}
+""")
+    
+    combined = "\n".join(content_parts)
+    
+    prompt = f"""You are creating an AI newsletter digest for a tech-savvy reader. 
+
+Analyze these {len(newsletters)} AI newsletters and create a concise, informative digest.
+
+{combined}
+
+Create a digest with these sections:
+1. **TOP STORIES** (3-5 most important items) - Each with: headline, source, 2-3 sentence summary, why it matters
+2. **OTHER NOTABLE NEWS** - Brief mentions of other stories
+3. **KEY TAKEAWAYS** - 2-3 bullet points on patterns/trends
+
+Format rules:
+- Use markdown
+- Keep each story summary to 2-3 sentences max
+- Include the source newsletter name
+- Write for someone who follows AI but wants quick briefings
+- Prioritize news with real-world impact
+
+Today's date: {datetime.now().strftime('%A, %B %d, %Y')}
+
+Begin your digest:"""
+
+    # Try using the LLM
+    print("🧠 Using LLM to synthesize digest...", file=sys.stderr)
+    
+    result = call_llm(prompt)
+    
+    if result:
+        return result
+    else:
+        # Fallback: return basic formatted output
+        print("⚠️ LLM unavailable, using basic formatting", file=sys.stderr)
+        return None
+
+def create_basic_digest(newsletters):
+    """Create a basic digest without LLM (fallback)."""
+    
+    lines = [
+        f"🤖 **AI NEWSLETTER DIGEST** — {datetime.now().strftime('%A, %B %d, %Y')}",
+        "",
+        f"*{len(newsletters)} newsletters analyzed*",
+        "",
+        "═" * 50,
+        "",
+    ]
+    
+    for nl in newsletters:
+        source = nl.get('from', 'Unknown').split('<')[0].strip()
+        subject = nl.get('subject', 'No subject')
+        content = nl.get('content', '')[:300]
+        
+        # Clean up content
+        content = ' '.join(content.split())[:300]
+        
+        lines.append(f"📌 **{subject}**")
+        lines.append(f"   Source: {source}")
+        if content:
+            lines.append(f"   {content}...")
+        lines.append("")
+    
+    lines.append("═" * 50)
+    lines.append(f"🦀 Krilly the Crab | {datetime.now().strftime('%B %d, %Y')}")
+    
+    return "\n".join(lines)
+
+def main():
+    """Main entry point."""
+    
+    # Read newsletters from stdin or file
+    if len(sys.argv) > 1:
+        with open(sys.argv[1], 'r') as f:
+            newsletters = json.load(f)
+    else:
+        newsletters = json.load(sys.stdin)
+    
+    if not newsletters:
+        print("No newsletters to summarize")
+        return
+    
+    print(f"📊 Processing {len(newsletters)} newsletters...", file=sys.stderr)
+    
+    # Try LLM summarization first
+    digest = summarize_newsletters(newsletters)
+    
+    if not digest:
+        # Fallback to basic
+        digest = create_basic_digest(newsletters)
+    
+    print(digest)
+
+if __name__ == '__main__':
+    main()